mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-30 06:43:27 +00:00
refactor: move prompt_cache_options and the OpenAI drop_params strip to their own PRs
This PR now only carries prompt_cache_breakpoint to Claude cache_control on chat completions. The gpt-4.1 drop_params strip is #46453 and prompt_cache_options lands in a stacked follow-up
This commit is contained in:
ryan-crabbe-berri
parent
65bd696336
commit
66eefb1870
8 changed files with 23 additions and 281 deletions
|
|
@ -13,8 +13,6 @@ from pathlib import Path
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.router_utils.batch_utils import InMemoryFile
|
||||
|
|
@ -1807,33 +1805,6 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
|
|||
)
|
||||
|
||||
|
||||
_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
|
||||
_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _without_prompt_cache_breakpoint(value: object) -> object:
|
||||
if not isinstance(value, Mapping):
|
||||
return value
|
||||
fields: Final = _BREAKPOINT_FIELDS.validate_python(value)
|
||||
return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
|
||||
|
||||
|
||||
def _breakpoints_stripped_from(key: str, value: object) -> object:
|
||||
if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list):
|
||||
return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)]
|
||||
return _without_prompt_cache_breakpoint(value) if key == "function" else value
|
||||
|
||||
|
||||
def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
|
||||
"""A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts',
|
||||
its tool calls' and its ``function``'s."""
|
||||
stripped: Final = {
|
||||
key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint"
|
||||
}
|
||||
return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys
|
||||
|
||||
|
||||
LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
"""OpenAI's ``prompt_cache_breakpoint`` and ``prompt_cache_options`` on a chat request bound for Claude, rewritten as
|
||||
Claude's ``cache_control``."""
|
||||
"""OpenAI's ``prompt_cache_breakpoint`` on a chat request bound for Claude, rewritten as Claude's ``cache_control``."""
|
||||
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -31,7 +30,6 @@ _CLAUDE_TTL_MARKS: Final = MappingProxyType(
|
|||
class ClaudePromptCache:
|
||||
messages: tuple[object, ...]
|
||||
tools: tuple[object, ...] | None
|
||||
params: _Fields
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -75,31 +73,6 @@ def _breakpoint_mark(marker: object) -> ChatCompletionCachedContent | Unsupporte
|
|||
return _ttl_mark(fields.get("ttl"), "prompt_cache_breakpoint", marker)
|
||||
|
||||
|
||||
def _automatic_mark(
|
||||
options: object, accepts_automatic_caching: bool
|
||||
) -> ChatCompletionCachedContent | UnsupportedPromptCacheParam | None:
|
||||
if options is None:
|
||||
return None
|
||||
fields: Final = _fields(options)
|
||||
if fields is None or fields.get("mode") not in (None, "implicit", "explicit"):
|
||||
return UnsupportedPromptCacheParam(
|
||||
"prompt_cache_options", options, 'expected a "mode" of "implicit" or "explicit"', fallback=None
|
||||
)
|
||||
if fields.get("mode") != "implicit":
|
||||
return (
|
||||
None
|
||||
if fields.get("ttl") is None
|
||||
else UnsupportedPromptCacheParam(
|
||||
"prompt_cache_options", options, 'Claude applies a ttl only with "mode": "implicit"', fallback=None
|
||||
)
|
||||
)
|
||||
if not accepts_automatic_caching:
|
||||
return UnsupportedPromptCacheParam(
|
||||
"prompt_cache_options", options, "this route has no automatic caching", fallback=None
|
||||
)
|
||||
return _ttl_mark(fields.get("ttl"), "prompt_cache_options", options)
|
||||
|
||||
|
||||
def _indexed_fields(values: Sequence[object]) -> Iterator[tuple[int, _Fields]]:
|
||||
return ((index, fields) for index, fields in enumerate(map(_fields, values)) if fields is not None)
|
||||
|
||||
|
|
@ -216,43 +189,26 @@ def claude_prompt_cache(
|
|||
tools: Sequence[object] | None,
|
||||
params: _Fields,
|
||||
*,
|
||||
accepts_automatic_caching: bool,
|
||||
drop_params: bool,
|
||||
) -> ClaudePromptCache | UnsupportedPromptCacheParam:
|
||||
"""``prompt_cache_breakpoint`` marks renamed to ``cache_control``, and ``prompt_cache_options`` with
|
||||
``"mode": "implicit"`` turned into the request-level ``cache_control`` (Claude's automatic caching). A mark or
|
||||
option Claude cannot honor is reported unless ``drop_params``, which keeps what Claude can use and, when the
|
||||
request ends up with more than Claude's four breakpoints, the latest four. A request carrying neither param is
|
||||
returned as written."""
|
||||
"""``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless
|
||||
``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four
|
||||
breakpoints, the latest four. A request-level ``cache_control`` in ``params`` takes one of the four. A request
|
||||
carrying no breakpoint is returned as written."""
|
||||
listed_tools: Final = tools or ()
|
||||
options: Final = params.get("prompt_cache_options")
|
||||
blocks: Final = tuple(block for _, block in _markable(messages, listed_tools))
|
||||
if options is None and all(block.get("prompt_cache_breakpoint") is None for block in blocks):
|
||||
return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools), params)
|
||||
automatic: Final = _automatic_mark(options, accepts_automatic_caching)
|
||||
problem: Final = (
|
||||
automatic
|
||||
if isinstance(automatic, UnsupportedPromptCacheParam)
|
||||
else next(filter(None, map(_breakpoint_problem, blocks)), None)
|
||||
)
|
||||
if all(block.get("prompt_cache_breakpoint") is None for block in blocks):
|
||||
return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools))
|
||||
problem: Final = next(filter(None, map(_breakpoint_problem, blocks)), None)
|
||||
if problem is not None and not drop_params:
|
||||
return problem
|
||||
if problem is not None:
|
||||
verbose_logger.debug("Claude: adjusting %s=%r because %s", problem.param, problem.value, problem.reason)
|
||||
automatic_mark: Final = automatic.fallback if isinstance(automatic, UnsupportedPromptCacheParam) else automatic
|
||||
request_params: Final = {
|
||||
**{key: value for key, value in params.items() if key != "prompt_cache_options"},
|
||||
**(
|
||||
{"cache_control": automatic_mark}
|
||||
if automatic_mark is not None and params.get("cache_control") is None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
renamed_messages, renamed_tools = _marked_request(messages, listed_tools, _renamed)
|
||||
sites: Final = tuple(
|
||||
path for path, block in _markable(renamed_messages, renamed_tools) if block.get("cache_control") is not None
|
||||
)
|
||||
excess: Final = len(sites) + int(request_params.get("cache_control") is not None) - MAX_CACHE_CONTROL_BLOCKS
|
||||
excess: Final = len(sites) + int(params.get("cache_control") is not None) - MAX_CACHE_CONTROL_BLOCKS
|
||||
if excess > 0 and drop_params:
|
||||
verbose_logger.debug("Claude: dropping the earliest %d cache breakpoints", excess)
|
||||
trimmed_messages, trimmed_tools = (
|
||||
|
|
@ -260,7 +216,7 @@ def claude_prompt_cache(
|
|||
if excess > 0 and drop_params
|
||||
else (renamed_messages, renamed_tools)
|
||||
)
|
||||
return ClaudePromptCache(trimmed_messages, None if tools is None else trimmed_tools, request_params)
|
||||
return ClaudePromptCache(trimmed_messages, None if tools is None else trimmed_tools)
|
||||
|
||||
|
||||
def resolved_claude_prompt_cache(
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
hoist_images_from_tool_messages,
|
||||
system_messages_first,
|
||||
tool_with_sanitized_parameters,
|
||||
without_prompt_cache_breakpoints,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
|
|
@ -475,8 +474,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
|
||||
api.openai.com, or one opted in through ``prompt_cache_options`` or the cost
|
||||
map) gets each Anthropic ``cache_control`` mark rewritten in place. Any other
|
||||
endpoint keeps or strips the marks as before, and under drop_params also loses
|
||||
every ``prompt_cache_breakpoint`` it cannot read.
|
||||
endpoint keeps or strips the marks exactly as before.
|
||||
"""
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
|
||||
|
|
@ -487,24 +485,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
)
|
||||
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
|
||||
return messages, tools
|
||||
drops_breakpoints: Final = not reads_breakpoints and (
|
||||
litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
)
|
||||
renamed: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if reads_breakpoints
|
||||
else [without_prompt_cache_breakpoints(message) for message in messages]
|
||||
if drops_breakpoints
|
||||
else list(messages)
|
||||
)
|
||||
kept_tools: Final = (
|
||||
None
|
||||
if tools is None
|
||||
else [without_prompt_cache_breakpoints(tool) for tool in tools]
|
||||
if drops_breakpoints
|
||||
else list(tools)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(
|
||||
model=model, messages=renamed, tools=list(tools) if tools is not None else None
|
||||
)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
|
||||
|
||||
def _sanitized_tools_update_for_openai(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -211,7 +211,6 @@ from .litellm_core_utils.streaming_chunk_builder_utils import (
|
|||
)
|
||||
from .llms.anthropic.chat import AnthropicChatCompletion
|
||||
from .llms.anthropic.chat.prompt_cache_breakpoints import claude_prompt_cache, resolved_claude_prompt_cache
|
||||
from .llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from .llms.azure.audio_transcriptions import AzureAudioTranscription
|
||||
from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params
|
||||
from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
|
||||
|
|
@ -5567,7 +5566,6 @@ def completion(
|
|||
cast(list[object], messages), # cast-ok: completion types messages as a bare List
|
||||
cast(list[object] | None, tools), # cast-ok: completion types tools as a bare List
|
||||
non_default_params,
|
||||
accepts_automatic_caching=isinstance(provider_config, AnthropicConfig),
|
||||
drop_params=claude_drop_params,
|
||||
),
|
||||
model=model,
|
||||
|
|
@ -5652,9 +5650,7 @@ def completion(
|
|||
"allowed_openai_params": allowed_openai_params,
|
||||
"base_model": base_model,
|
||||
}
|
||||
optional_params = get_optional_params(
|
||||
**optional_param_args, **(non_default_params if claude_cache is None else claude_cache.params)
|
||||
)
|
||||
optional_params = get_optional_params(**optional_param_args, **non_default_params)
|
||||
processed_non_default_params: Final = pre_process_non_default_params(
|
||||
model=model,
|
||||
passed_params=optional_param_args,
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
|
|||
or beside a ``cache_control`` on the same system part, must write the cache
|
||||
on the first call and read it on the second. On
|
||||
anthropic and Converse, a 30m breakpoint and five breakpoints under
|
||||
drop_params must still cache. On anthropic, ``prompt_cache_options`` with
|
||||
``"mode": "implicit"`` must cache with no breakpoint at all.
|
||||
drop_params must still cache.
|
||||
|
||||
service_tier lives in test_provider_features_e2e.py.
|
||||
|
||||
|
|
@ -698,8 +697,8 @@ def _assert_claude_writes_then_reads_cache(client: PassthroughClient, key: str,
|
|||
class TestPromptCacheBreakpointOnClaude:
|
||||
"""OpenAI's ``prompt_cache_breakpoint`` sent to a Claude deployment caches the prefix it marks.
|
||||
|
||||
The 400s that need drop_params off (a 30m ttl, or implicit ``prompt_cache_options`` on Converse) happen before any
|
||||
provider call, so the wire tests in tests/integration prove them: the shared stack sets
|
||||
The 400s that need drop_params off (a 30m ttl or a malformed marker) happen before any provider call, so the wire
|
||||
tests in tests/integration prove them: the shared stack sets
|
||||
``litellm_settings.drop_params: true``, which no deployment can switch back off."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -755,22 +754,6 @@ class TestPromptCacheBreakpointOnClaude:
|
|||
)
|
||||
_assert_claude_writes_then_reads_cache(client, scoped_key, body)
|
||||
|
||||
@pytest.mark.parametrize("route", [_claude_route_param("anthropic")])
|
||||
def test_implicit_prompt_cache_options_cache_without_any_breakpoint(
|
||||
self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute
|
||||
) -> None:
|
||||
model: Final = _register_claude(client, resources, route)
|
||||
body: Final = ChatBody(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
prompt_cache_options=PromptCacheOptions(mode="implicit"),
|
||||
messages=[
|
||||
ChatMessage(role="system", content=_cacheable_prefix()),
|
||||
ChatMessage(role="user", content="Reply with one word."),
|
||||
],
|
||||
)
|
||||
_assert_claude_writes_then_reads_cache(client, scoped_key, body)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse", "bedrock_invoke", "vertex")]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import json
|
||||
from collections.abc import Iterator, Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
|
|
@ -54,7 +53,6 @@ _PROVIDERS: Final[tuple[Provider, ...]] = (
|
|||
"vertex_ai",
|
||||
"azure_ai",
|
||||
)
|
||||
_AUTOMATIC_CACHING_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if provider != "bedrock_converse")
|
||||
_SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "tool_result", "string_message")
|
||||
_FIVE_SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "string_message")
|
||||
_EXPLICIT: Final[dict[str, JsonValue]] = {"mode": "explicit"}
|
||||
|
|
@ -174,11 +172,6 @@ def _only_request(wire: Wire) -> Request:
|
|||
return received[0]
|
||||
|
||||
|
||||
def _automatic_mark(request: Request) -> JsonValue:
|
||||
body: Final = json.loads(request.body)
|
||||
return body.get("cache_control") if isinstance(body, dict) else None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("site", _SITES)
|
||||
@pytest.mark.parametrize("provider", _PROVIDERS)
|
||||
def test_breakpoint_becomes_a_cache_mark_on_the_same_block(gateway: Gateway, provider: Provider, site: Site) -> None:
|
||||
|
|
@ -309,56 +302,6 @@ def test_injection_points_stand_down_behind_the_latest_four_breakpoints(gateway:
|
|||
assert gateway_injected(response_id) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", _AUTOMATIC_CACHING_PROVIDERS)
|
||||
def test_implicit_prompt_cache_options_turn_on_claude_automatic_caching(gateway: Gateway, provider: Provider) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(gateway, scenario, wire, provider)
|
||||
status, _, text = post_chat(gateway, _body(model, marker, {}, prompt_cache_options={"mode": "implicit"}))
|
||||
assert status == 200, text
|
||||
request: Final = _only_request(wire)
|
||||
assert b"prompt_cache_options" not in request.body, request.body
|
||||
assert _automatic_mark(request) == EPHEMERAL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", _PROVIDERS)
|
||||
def test_explicit_prompt_cache_options_leave_only_the_breakpoints(gateway: Gateway, provider: Provider) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(gateway, scenario, wire, provider)
|
||||
status, _, text = post_chat(
|
||||
gateway, _body(model, marker, _marked(("user_part",)), prompt_cache_options=_EXPLICIT)
|
||||
)
|
||||
assert status == 200, text
|
||||
request: Final = _only_request(wire)
|
||||
assert b"prompt_cache_options" not in request.body, request.body
|
||||
assert _labels(provider, request) == [ASK_LABEL]
|
||||
|
||||
|
||||
def test_implicit_prompt_cache_options_are_a_400_where_claude_has_no_automatic_caching(gateway: Gateway) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(gateway, scenario, wire, "bedrock_converse")
|
||||
status, _, text = post_chat(gateway, _body(model, marker, {}, prompt_cache_options={"mode": "implicit"}))
|
||||
received: Final = wire.drain()
|
||||
assert status == 400, text
|
||||
assert "prompt_cache_options" in text and "drop_params" in text, text
|
||||
assert received == (), [request.target for request in received]
|
||||
|
||||
|
||||
def test_implicit_prompt_cache_options_are_dropped_where_claude_has_no_automatic_caching_under_drop_params(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(gateway, scenario, wire, "bedrock_converse", drop_params=True)
|
||||
status, _, text = post_chat(gateway, _body(model, marker, {}, prompt_cache_options={"mode": "implicit"}))
|
||||
assert status == 200, text
|
||||
request: Final = _only_request(wire)
|
||||
assert b"prompt_cache_options" not in request.body, request.body
|
||||
assert bedrock_labels(request) == []
|
||||
|
||||
|
||||
def _drop_params_config(directory: Path) -> Path:
|
||||
return owned_config(directory, [], litellm_settings={"drop_params": True})
|
||||
|
||||
|
|
|
|||
|
|
@ -19,16 +19,9 @@ def _cache(
|
|||
tools: list[object] | None = None,
|
||||
params: dict[str, object] | None = None,
|
||||
*,
|
||||
accepts_automatic_caching: bool = True,
|
||||
drop_params: bool = False,
|
||||
) -> ClaudePromptCache | UnsupportedPromptCacheParam:
|
||||
return claude_prompt_cache(
|
||||
messages,
|
||||
tools,
|
||||
params or {},
|
||||
accepts_automatic_caching=accepts_automatic_caching,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
return claude_prompt_cache(messages, tools, params or {}, drop_params=drop_params)
|
||||
|
||||
|
||||
def _resolved(
|
||||
|
|
@ -36,12 +29,9 @@ def _resolved(
|
|||
tools: list[object] | None = None,
|
||||
params: dict[str, object] | None = None,
|
||||
*,
|
||||
accepts_automatic_caching: bool = True,
|
||||
drop_params: bool = False,
|
||||
) -> ClaudePromptCache:
|
||||
outcome: Final = _cache(
|
||||
messages, tools, params, accepts_automatic_caching=accepts_automatic_caching, drop_params=drop_params
|
||||
)
|
||||
outcome: Final = _cache(messages, tools, params, drop_params=drop_params)
|
||||
assert isinstance(outcome, ClaudePromptCache), outcome
|
||||
return outcome
|
||||
|
||||
|
|
@ -82,13 +72,12 @@ def test_breakpoints_become_cache_control_on_every_block_that_carries_one():
|
|||
]
|
||||
|
||||
|
||||
def test_a_request_without_breakpoints_or_options_is_returned_as_written():
|
||||
def test_a_request_without_breakpoints_is_returned_as_written():
|
||||
messages: Final = [{"role": "user", "content": [_part("hi", cache_control=_EPHEMERAL)] * 5}]
|
||||
params: Final = {"temperature": 0.1}
|
||||
|
||||
result: Final = _resolved(messages, None, params, drop_params=True)
|
||||
result: Final = _resolved(messages, None, {"temperature": 0.1}, drop_params=True)
|
||||
|
||||
assert result == ClaudePromptCache(tuple(messages), None, params)
|
||||
assert result == ClaudePromptCache(tuple(messages), None)
|
||||
|
||||
|
||||
def test_a_client_cache_control_on_the_same_block_wins():
|
||||
|
|
@ -127,52 +116,6 @@ def test_an_unsupported_breakpoint_raises_a_400_naming_drop_params():
|
|||
assert "30m" in raised.value.message and "drop_params" in raised.value.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options, automatic",
|
||||
[({"mode": "implicit"}, _EPHEMERAL), ({"mode": "implicit", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"})],
|
||||
)
|
||||
def test_implicit_options_become_claude_automatic_caching(options: object, automatic: object):
|
||||
result: Final = _resolved([{"role": "user", "content": "hi"}], None, {"prompt_cache_options": options, "top_p": 1})
|
||||
|
||||
assert result.params == {"top_p": 1, "cache_control": automatic}
|
||||
|
||||
|
||||
def test_implicit_options_leave_a_client_top_level_cache_control_alone():
|
||||
hour: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
result: Final = _resolved([], None, {"prompt_cache_options": {"mode": "implicit"}, "cache_control": hour})
|
||||
|
||||
assert result.params == {"cache_control": hour}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("options", [{"mode": "explicit"}, {}])
|
||||
def test_options_without_implicit_mode_add_nothing_and_never_reach_claude(options: object):
|
||||
assert _resolved([], None, {"prompt_cache_options": options}).params == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options, accepts_automatic_caching, kept",
|
||||
[
|
||||
({"mode": "implicit"}, False, {}),
|
||||
({"mode": "implicit", "ttl": "30m"}, True, {"cache_control": _EPHEMERAL}),
|
||||
({"mode": "explicit", "ttl": "30m"}, True, {}),
|
||||
({"mode": "always"}, True, {}),
|
||||
("on", True, {}),
|
||||
],
|
||||
)
|
||||
def test_options_claude_cannot_honor_are_reported_or_adjusted_under_drop_params(
|
||||
options: object, accepts_automatic_caching: bool, kept: dict[str, object]
|
||||
):
|
||||
params: Final = {"prompt_cache_options": options}
|
||||
|
||||
problem: Final = _cache([], None, params, accepts_automatic_caching=accepts_automatic_caching)
|
||||
dropped: Final = _resolved([], None, params, accepts_automatic_caching=accepts_automatic_caching, drop_params=True)
|
||||
|
||||
assert isinstance(problem, UnsupportedPromptCacheParam)
|
||||
assert (problem.param, problem.value) == ("prompt_cache_options", options)
|
||||
assert dropped.params == kept
|
||||
|
||||
|
||||
def test_drop_params_keeps_the_latest_four_marks_in_the_order_claude_renders_them():
|
||||
tools: Final = [{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": _EXPLICIT}]
|
||||
messages: Final = [
|
||||
|
|
@ -180,14 +123,13 @@ def test_drop_params_keeps_the_latest_four_marks_in_the_order_claude_renders_the
|
|||
{"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(3)]},
|
||||
]
|
||||
|
||||
result: Final = _resolved(messages, tools, {"prompt_cache_options": {"mode": "implicit"}}, drop_params=True)
|
||||
result: Final = _resolved(messages, tools, {"cache_control": _EPHEMERAL}, drop_params=True)
|
||||
|
||||
assert list(result.tools or ()) == [{"type": "function", "function": {"name": "t"}}]
|
||||
assert list(result.messages) == [
|
||||
{"role": "system", "content": [_part("s")]},
|
||||
{"role": "user", "content": [_part(str(index), cache_control=_EPHEMERAL) for index in range(3)]},
|
||||
]
|
||||
assert result.params == {"cache_control": _EPHEMERAL}
|
||||
|
||||
|
||||
def test_without_drop_params_more_than_four_marks_are_sent_for_claude_to_answer():
|
||||
|
|
|
|||
|
|
@ -1592,43 +1592,6 @@ class TestOpenAIPromptCacheBreakpointChatPath:
|
|||
assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT}
|
||||
assert "prompt_cache_options" not in request
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "drop_params", "sent"),
|
||||
[("gpt-4.1", True, False), ("gpt-4.1", False, True), ("gpt-5.6", True, True)],
|
||||
ids=["unread-dropped", "unread-sent-through", "read-kept"],
|
||||
)
|
||||
def test_drop_params_strips_breakpoints_only_where_the_model_cannot_read_them(
|
||||
self, model: str, drop_params: bool, sent: bool
|
||||
):
|
||||
marked_part: Final = {"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
part: Final = {"type": "text", "text": "hi"}
|
||||
call: Final = {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}}
|
||||
function: Final = {"name": "f", "parameters": {"type": "object"}}
|
||||
marked: Final = {"prompt_cache_breakpoint": self.EXPLICIT}
|
||||
|
||||
request = OpenAIGPTConfig().transform_request(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "user", "content": [marked_part]},
|
||||
{"role": "assistant", "content": "ok", **marked, "tool_calls": [{**call, **marked}]},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": "done"},
|
||||
],
|
||||
optional_params={"tools": [{"type": "function", "function": {**function, **marked}}, {"type": "function", "function": function, **marked}]},
|
||||
litellm_params={"custom_llm_provider": "openai", "drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
mark: Final = marked if sent else {}
|
||||
assert request["messages"] == [
|
||||
{"role": "user", "content": [marked_part if sent else part]},
|
||||
{"role": "assistant", "content": "ok", **mark, "tool_calls": [{**call, **mark}]},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": "done"},
|
||||
]
|
||||
assert request["tools"] == [
|
||||
{"type": "function", "function": {**function, **mark}},
|
||||
{"type": "function", "function": function, **mark},
|
||||
]
|
||||
|
||||
|
||||
_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$'
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue