From 66eefb1870abe7280835e8015ac321f80e8c1fc5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 14 Oct 2026 09:16:29 -0700 Subject: [PATCH] refactor: move prompt_cache_options and the OpenAI drop_params strip to their own PRs This PR now only carries prompt_cache_breakpoint to Claude cache_control on chat completions. The gpt-4.1 drop_params strip is #46453 and prompt_cache_options lands in a stacked follow-up --- .../prompt_templates/common_utils.py | 29 -------- .../chat/prompt_cache_breakpoints.py | 64 +++-------------- .../llms/openai/chat/gpt_transformation.py | 18 +---- litellm/main.py | 6 +- .../e2e/llm_translation/test_cache_control.py | 23 +----- ...laude_chat_prompt_cache_breakpoint_wire.py | 57 --------------- .../chat/test_prompt_cache_breakpoints.py | 70 ++----------------- .../chat/test_openai_gpt_transformation.py | 37 ---------- 8 files changed, 23 insertions(+), 281 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f92230f5755..3c1b3cf964b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,8 +13,6 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from pydantic import TypeAdapter - import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -1807,33 +1805,6 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: ) -_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) -_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) -_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) - - -def _without_prompt_cache_breakpoint(value: object) -> object: - if not isinstance(value, Mapping): - return value - fields: Final = _BREAKPOINT_FIELDS.validate_python(value) - return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} - - -def _breakpoints_stripped_from(key: str, value: object) -> object: - if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list): - return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)] - return _without_prompt_cache_breakpoint(value) if key == "function" else value - - -def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT: - """A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts', - its tool calls' and its ``function``'s.""" - stripped: Final = { - key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint" - } - return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys - - LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"}) diff --git a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py index 3eb6d688bea..157c25da32f 100644 --- a/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py +++ b/litellm/llms/anthropic/chat/prompt_cache_breakpoints.py @@ -1,5 +1,4 @@ -"""OpenAI's ``prompt_cache_breakpoint`` and ``prompt_cache_options`` on a chat request bound for Claude, rewritten as -Claude's ``cache_control``.""" +"""OpenAI's ``prompt_cache_breakpoint`` on a chat request bound for Claude, rewritten as Claude's ``cache_control``.""" from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass @@ -31,7 +30,6 @@ _CLAUDE_TTL_MARKS: Final = MappingProxyType( class ClaudePromptCache: messages: tuple[object, ...] tools: tuple[object, ...] | None - params: _Fields @dataclass(frozen=True, slots=True) @@ -75,31 +73,6 @@ def _breakpoint_mark(marker: object) -> ChatCompletionCachedContent | Unsupporte return _ttl_mark(fields.get("ttl"), "prompt_cache_breakpoint", marker) -def _automatic_mark( - options: object, accepts_automatic_caching: bool -) -> ChatCompletionCachedContent | UnsupportedPromptCacheParam | None: - if options is None: - return None - fields: Final = _fields(options) - if fields is None or fields.get("mode") not in (None, "implicit", "explicit"): - return UnsupportedPromptCacheParam( - "prompt_cache_options", options, 'expected a "mode" of "implicit" or "explicit"', fallback=None - ) - if fields.get("mode") != "implicit": - return ( - None - if fields.get("ttl") is None - else UnsupportedPromptCacheParam( - "prompt_cache_options", options, 'Claude applies a ttl only with "mode": "implicit"', fallback=None - ) - ) - if not accepts_automatic_caching: - return UnsupportedPromptCacheParam( - "prompt_cache_options", options, "this route has no automatic caching", fallback=None - ) - return _ttl_mark(fields.get("ttl"), "prompt_cache_options", options) - - def _indexed_fields(values: Sequence[object]) -> Iterator[tuple[int, _Fields]]: return ((index, fields) for index, fields in enumerate(map(_fields, values)) if fields is not None) @@ -216,43 +189,26 @@ def claude_prompt_cache( tools: Sequence[object] | None, params: _Fields, *, - accepts_automatic_caching: bool, drop_params: bool, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: - """``prompt_cache_breakpoint`` marks renamed to ``cache_control``, and ``prompt_cache_options`` with - ``"mode": "implicit"`` turned into the request-level ``cache_control`` (Claude's automatic caching). A mark or - option Claude cannot honor is reported unless ``drop_params``, which keeps what Claude can use and, when the - request ends up with more than Claude's four breakpoints, the latest four. A request carrying neither param is - returned as written.""" + """``prompt_cache_breakpoint`` marks renamed to ``cache_control``. A mark Claude cannot honor is reported unless + ``drop_params``, which keeps what Claude can use and, when the request ends up with more than Claude's four + breakpoints, the latest four. A request-level ``cache_control`` in ``params`` takes one of the four. A request + carrying no breakpoint is returned as written.""" listed_tools: Final = tools or () - options: Final = params.get("prompt_cache_options") blocks: Final = tuple(block for _, block in _markable(messages, listed_tools)) - if options is None and all(block.get("prompt_cache_breakpoint") is None for block in blocks): - return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools), params) - automatic: Final = _automatic_mark(options, accepts_automatic_caching) - problem: Final = ( - automatic - if isinstance(automatic, UnsupportedPromptCacheParam) - else next(filter(None, map(_breakpoint_problem, blocks)), None) - ) + if all(block.get("prompt_cache_breakpoint") is None for block in blocks): + return ClaudePromptCache(tuple(messages), None if tools is None else tuple(tools)) + problem: Final = next(filter(None, map(_breakpoint_problem, blocks)), None) if problem is not None and not drop_params: return problem if problem is not None: verbose_logger.debug("Claude: adjusting %s=%r because %s", problem.param, problem.value, problem.reason) - automatic_mark: Final = automatic.fallback if isinstance(automatic, UnsupportedPromptCacheParam) else automatic - request_params: Final = { - **{key: value for key, value in params.items() if key != "prompt_cache_options"}, - **( - {"cache_control": automatic_mark} - if automatic_mark is not None and params.get("cache_control") is None - else {} - ), - } renamed_messages, renamed_tools = _marked_request(messages, listed_tools, _renamed) sites: Final = tuple( path for path, block in _markable(renamed_messages, renamed_tools) if block.get("cache_control") is not None ) - excess: Final = len(sites) + int(request_params.get("cache_control") is not None) - MAX_CACHE_CONTROL_BLOCKS + excess: Final = len(sites) + int(params.get("cache_control") is not None) - MAX_CACHE_CONTROL_BLOCKS if excess > 0 and drop_params: verbose_logger.debug("Claude: dropping the earliest %d cache breakpoints", excess) trimmed_messages, trimmed_tools = ( @@ -260,7 +216,7 @@ def claude_prompt_cache( if excess > 0 and drop_params else (renamed_messages, renamed_tools) ) - return ClaudePromptCache(trimmed_messages, None if tools is None else trimmed_tools, request_params) + return ClaudePromptCache(trimmed_messages, None if tools is None else trimmed_tools) def resolved_claude_prompt_cache( diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 584cecc0f36..a51704085bd 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -28,7 +28,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( hoist_images_from_tool_messages, system_messages_first, tool_with_sanitized_parameters, - without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, @@ -475,8 +474,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on api.openai.com, or one opted in through ``prompt_cache_options`` or the cost map) gets each Anthropic ``cache_control`` mark rewritten in place. Any other - endpoint keeps or strips the marks as before, and under drop_params also loses - every ``prompt_cache_breakpoint`` it cannot read. + endpoint keeps or strips the marks exactly as before. """ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -487,24 +485,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): return messages, tools - drops_breakpoints: Final = not reads_breakpoints and ( - litellm.drop_params is True or litellm_params.get("drop_params") is True - ) renamed: Final = ( [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] if reads_breakpoints - else [without_prompt_cache_breakpoints(message) for message in messages] - if drops_breakpoints else list(messages) ) - kept_tools: Final = ( - None - if tools is None - else [without_prompt_cache_breakpoints(tool) for tool in tools] - if drops_breakpoints - else list(tools) + return self.remove_cache_control_flag_from_messages_and_tools( + model=model, messages=renamed, tools=list(tools) if tools is not None else None ) - return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools) def _sanitized_tools_update_for_openai( self, diff --git a/litellm/main.py b/litellm/main.py index 59c973786be..bb68bfa324f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -211,7 +211,6 @@ from .litellm_core_utils.streaming_chunk_builder_utils import ( ) from .llms.anthropic.chat import AnthropicChatCompletion from .llms.anthropic.chat.prompt_cache_breakpoints import claude_prompt_cache, resolved_claude_prompt_cache -from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -5567,7 +5566,6 @@ def completion( cast(list[object], messages), # cast-ok: completion types messages as a bare List cast(list[object] | None, tools), # cast-ok: completion types tools as a bare List non_default_params, - accepts_automatic_caching=isinstance(provider_config, AnthropicConfig), drop_params=claude_drop_params, ), model=model, @@ -5652,9 +5650,7 @@ def completion( "allowed_openai_params": allowed_openai_params, "base_model": base_model, } - optional_params = get_optional_params( - **optional_param_args, **(non_default_params if claude_cache is None else claude_cache.params) - ) + optional_params = get_optional_params(**optional_param_args, **non_default_params) processed_non_default_params: Final = pre_process_non_default_params( model=model, passed_params=optional_param_args, diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index cc6696b2934..5c30d68a28b 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -37,8 +37,7 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix or beside a ``cache_control`` on the same system part, must write the cache on the first call and read it on the second. On anthropic and Converse, a 30m breakpoint and five breakpoints under - drop_params must still cache. On anthropic, ``prompt_cache_options`` with - ``"mode": "implicit"`` must cache with no breakpoint at all. + drop_params must still cache. service_tier lives in test_provider_features_e2e.py. @@ -698,8 +697,8 @@ def _assert_claude_writes_then_reads_cache(client: PassthroughClient, key: str, class TestPromptCacheBreakpointOnClaude: """OpenAI's ``prompt_cache_breakpoint`` sent to a Claude deployment caches the prefix it marks. - The 400s that need drop_params off (a 30m ttl, or implicit ``prompt_cache_options`` on Converse) happen before any - provider call, so the wire tests in tests/integration prove them: the shared stack sets + The 400s that need drop_params off (a 30m ttl or a malformed marker) happen before any provider call, so the wire + tests in tests/integration prove them: the shared stack sets ``litellm_settings.drop_params: true``, which no deployment can switch back off.""" @pytest.mark.parametrize( @@ -755,22 +754,6 @@ class TestPromptCacheBreakpointOnClaude: ) _assert_claude_writes_then_reads_cache(client, scoped_key, body) - @pytest.mark.parametrize("route", [_claude_route_param("anthropic")]) - def test_implicit_prompt_cache_options_cache_without_any_breakpoint( - self, client: PassthroughClient, resources: ResourceManager, scoped_key: str, route: ClaudeRoute - ) -> None: - model: Final = _register_claude(client, resources, route) - body: Final = ChatBody( - model=model, - max_tokens=64, - prompt_cache_options=PromptCacheOptions(mode="implicit"), - messages=[ - ChatMessage(role="system", content=_cacheable_prefix()), - ChatMessage(role="user", content="Reply with one word."), - ], - ) - _assert_claude_writes_then_reads_cache(client, scoped_key, body) - @pytest.mark.parametrize( "route", [_claude_route_param(route) for route in ("anthropic", "bedrock_converse", "bedrock_invoke", "vertex")] ) diff --git a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py index 0ebc93bf1bb..1500482ddc9 100644 --- a/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_claude_chat_prompt_cache_breakpoint_wire.py @@ -1,4 +1,3 @@ -import json from collections.abc import Iterator, Mapping from pathlib import Path from types import MappingProxyType @@ -54,7 +53,6 @@ _PROVIDERS: Final[tuple[Provider, ...]] = ( "vertex_ai", "azure_ai", ) -_AUTOMATIC_CACHING_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if provider != "bedrock_converse") _SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "tool_result", "string_message") _FIVE_SITES: Final[tuple[Site, ...]] = ("tool", "system", "user_part", "tool_call", "string_message") _EXPLICIT: Final[dict[str, JsonValue]] = {"mode": "explicit"} @@ -174,11 +172,6 @@ def _only_request(wire: Wire) -> Request: return received[0] -def _automatic_mark(request: Request) -> JsonValue: - body: Final = json.loads(request.body) - return body.get("cache_control") if isinstance(body, dict) else None - - @pytest.mark.parametrize("site", _SITES) @pytest.mark.parametrize("provider", _PROVIDERS) def test_breakpoint_becomes_a_cache_mark_on_the_same_block(gateway: Gateway, provider: Provider, site: Site) -> None: @@ -309,56 +302,6 @@ def test_injection_points_stand_down_behind_the_latest_four_breakpoints(gateway: assert gateway_injected(response_id) is False -@pytest.mark.parametrize("provider", _AUTOMATIC_CACHING_PROVIDERS) -def test_implicit_prompt_cache_options_turn_on_claude_automatic_caching(gateway: Gateway, provider: Provider) -> None: - marker: Final = new_marker() - with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario: - model: Final = _deployment(gateway, scenario, wire, provider) - status, _, text = post_chat(gateway, _body(model, marker, {}, prompt_cache_options={"mode": "implicit"})) - assert status == 200, text - request: Final = _only_request(wire) - assert b"prompt_cache_options" not in request.body, request.body - assert _automatic_mark(request) == EPHEMERAL - - -@pytest.mark.parametrize("provider", _PROVIDERS) -def test_explicit_prompt_cache_options_leave_only_the_breakpoints(gateway: Gateway, provider: Provider) -> None: - marker: Final = new_marker() - with wire_server(lambda request: _respond(provider, request)) as wire, gateway.scenario() as scenario: - model: Final = _deployment(gateway, scenario, wire, provider) - status, _, text = post_chat( - gateway, _body(model, marker, _marked(("user_part",)), prompt_cache_options=_EXPLICIT) - ) - assert status == 200, text - request: Final = _only_request(wire) - assert b"prompt_cache_options" not in request.body, request.body - assert _labels(provider, request) == [ASK_LABEL] - - -def test_implicit_prompt_cache_options_are_a_400_where_claude_has_no_automatic_caching(gateway: Gateway) -> None: - marker: Final = new_marker() - with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario: - model: Final = _deployment(gateway, scenario, wire, "bedrock_converse") - status, _, text = post_chat(gateway, _body(model, marker, {}, prompt_cache_options={"mode": "implicit"})) - received: Final = wire.drain() - assert status == 400, text - assert "prompt_cache_options" in text and "drop_params" in text, text - assert received == (), [request.target for request in received] - - -def test_implicit_prompt_cache_options_are_dropped_where_claude_has_no_automatic_caching_under_drop_params( - gateway: Gateway, -) -> None: - marker: Final = new_marker() - with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario: - model: Final = _deployment(gateway, scenario, wire, "bedrock_converse", drop_params=True) - status, _, text = post_chat(gateway, _body(model, marker, {}, prompt_cache_options={"mode": "implicit"})) - assert status == 200, text - request: Final = _only_request(wire) - assert b"prompt_cache_options" not in request.body, request.body - assert bedrock_labels(request) == [] - - def _drop_params_config(directory: Path) -> Path: return owned_config(directory, [], litellm_settings={"drop_params": True}) diff --git a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py index b1446573831..7a743071905 100644 --- a/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py +++ b/tests/unit/llms/anthropic/chat/test_prompt_cache_breakpoints.py @@ -19,16 +19,9 @@ def _cache( tools: list[object] | None = None, params: dict[str, object] | None = None, *, - accepts_automatic_caching: bool = True, drop_params: bool = False, ) -> ClaudePromptCache | UnsupportedPromptCacheParam: - return claude_prompt_cache( - messages, - tools, - params or {}, - accepts_automatic_caching=accepts_automatic_caching, - drop_params=drop_params, - ) + return claude_prompt_cache(messages, tools, params or {}, drop_params=drop_params) def _resolved( @@ -36,12 +29,9 @@ def _resolved( tools: list[object] | None = None, params: dict[str, object] | None = None, *, - accepts_automatic_caching: bool = True, drop_params: bool = False, ) -> ClaudePromptCache: - outcome: Final = _cache( - messages, tools, params, accepts_automatic_caching=accepts_automatic_caching, drop_params=drop_params - ) + outcome: Final = _cache(messages, tools, params, drop_params=drop_params) assert isinstance(outcome, ClaudePromptCache), outcome return outcome @@ -82,13 +72,12 @@ def test_breakpoints_become_cache_control_on_every_block_that_carries_one(): ] -def test_a_request_without_breakpoints_or_options_is_returned_as_written(): +def test_a_request_without_breakpoints_is_returned_as_written(): messages: Final = [{"role": "user", "content": [_part("hi", cache_control=_EPHEMERAL)] * 5}] - params: Final = {"temperature": 0.1} - result: Final = _resolved(messages, None, params, drop_params=True) + result: Final = _resolved(messages, None, {"temperature": 0.1}, drop_params=True) - assert result == ClaudePromptCache(tuple(messages), None, params) + assert result == ClaudePromptCache(tuple(messages), None) def test_a_client_cache_control_on_the_same_block_wins(): @@ -127,52 +116,6 @@ def test_an_unsupported_breakpoint_raises_a_400_naming_drop_params(): assert "30m" in raised.value.message and "drop_params" in raised.value.message -@pytest.mark.parametrize( - "options, automatic", - [({"mode": "implicit"}, _EPHEMERAL), ({"mode": "implicit", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"})], -) -def test_implicit_options_become_claude_automatic_caching(options: object, automatic: object): - result: Final = _resolved([{"role": "user", "content": "hi"}], None, {"prompt_cache_options": options, "top_p": 1}) - - assert result.params == {"top_p": 1, "cache_control": automatic} - - -def test_implicit_options_leave_a_client_top_level_cache_control_alone(): - hour: Final = {"type": "ephemeral", "ttl": "1h"} - - result: Final = _resolved([], None, {"prompt_cache_options": {"mode": "implicit"}, "cache_control": hour}) - - assert result.params == {"cache_control": hour} - - -@pytest.mark.parametrize("options", [{"mode": "explicit"}, {}]) -def test_options_without_implicit_mode_add_nothing_and_never_reach_claude(options: object): - assert _resolved([], None, {"prompt_cache_options": options}).params == {} - - -@pytest.mark.parametrize( - "options, accepts_automatic_caching, kept", - [ - ({"mode": "implicit"}, False, {}), - ({"mode": "implicit", "ttl": "30m"}, True, {"cache_control": _EPHEMERAL}), - ({"mode": "explicit", "ttl": "30m"}, True, {}), - ({"mode": "always"}, True, {}), - ("on", True, {}), - ], -) -def test_options_claude_cannot_honor_are_reported_or_adjusted_under_drop_params( - options: object, accepts_automatic_caching: bool, kept: dict[str, object] -): - params: Final = {"prompt_cache_options": options} - - problem: Final = _cache([], None, params, accepts_automatic_caching=accepts_automatic_caching) - dropped: Final = _resolved([], None, params, accepts_automatic_caching=accepts_automatic_caching, drop_params=True) - - assert isinstance(problem, UnsupportedPromptCacheParam) - assert (problem.param, problem.value) == ("prompt_cache_options", options) - assert dropped.params == kept - - def test_drop_params_keeps_the_latest_four_marks_in_the_order_claude_renders_them(): tools: Final = [{"type": "function", "function": {"name": "t"}, "prompt_cache_breakpoint": _EXPLICIT}] messages: Final = [ @@ -180,14 +123,13 @@ def test_drop_params_keeps_the_latest_four_marks_in_the_order_claude_renders_the {"role": "user", "content": [_part(str(index), prompt_cache_breakpoint=_EXPLICIT) for index in range(3)]}, ] - result: Final = _resolved(messages, tools, {"prompt_cache_options": {"mode": "implicit"}}, drop_params=True) + result: Final = _resolved(messages, tools, {"cache_control": _EPHEMERAL}, drop_params=True) assert list(result.tools or ()) == [{"type": "function", "function": {"name": "t"}}] assert list(result.messages) == [ {"role": "system", "content": [_part("s")]}, {"role": "user", "content": [_part(str(index), cache_control=_EPHEMERAL) for index in range(3)]}, ] - assert result.params == {"cache_control": _EPHEMERAL} def test_without_drop_params_more_than_four_marks_are_sent_for_claude_to_answer(): diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 76faf338307..a0a55df95e6 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1592,43 +1592,6 @@ class TestOpenAIPromptCacheBreakpointChatPath: assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT} assert "prompt_cache_options" not in request - @pytest.mark.parametrize( - ("model", "drop_params", "sent"), - [("gpt-4.1", True, False), ("gpt-4.1", False, True), ("gpt-5.6", True, True)], - ids=["unread-dropped", "unread-sent-through", "read-kept"], - ) - def test_drop_params_strips_breakpoints_only_where_the_model_cannot_read_them( - self, model: str, drop_params: bool, sent: bool - ): - marked_part: Final = {"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT} - part: Final = {"type": "text", "text": "hi"} - call: Final = {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}} - function: Final = {"name": "f", "parameters": {"type": "object"}} - marked: Final = {"prompt_cache_breakpoint": self.EXPLICIT} - - request = OpenAIGPTConfig().transform_request( - model=model, - messages=[ - {"role": "user", "content": [marked_part]}, - {"role": "assistant", "content": "ok", **marked, "tool_calls": [{**call, **marked}]}, - {"role": "tool", "tool_call_id": "c1", "content": "done"}, - ], - optional_params={"tools": [{"type": "function", "function": {**function, **marked}}, {"type": "function", "function": function, **marked}]}, - litellm_params={"custom_llm_provider": "openai", "drop_params": drop_params}, - headers={}, - ) - - mark: Final = marked if sent else {} - assert request["messages"] == [ - {"role": "user", "content": [marked_part if sent else part]}, - {"role": "assistant", "content": "ok", **mark, "tool_calls": [{**call, **mark}]}, - {"role": "tool", "tool_call_id": "c1", "content": "done"}, - ] - assert request["tools"] == [ - {"type": "function", "function": {**function, **mark}}, - {"type": "function", "function": function, **mark}, - ] - _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$'