diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 5cdddf93ddb..b93327eee5e 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -788,9 +788,13 @@ def _message_point_ordinal(point: CacheControlMessageInjectionPoint) -> int | No return None -def _placed_on_the_final_message_list(point: CacheControlMessageInjectionPoint) -> bool: +def _counts_back_from_the_end(point: CacheControlMessageInjectionPoint) -> bool: ordinal: Final = _message_point_ordinal(point) - return point.get("index") is None or (ordinal is not None and ordinal < 0) + return ordinal is not None and ordinal < 0 + + +def _placed_on_the_final_message_list(point: CacheControlMessageInjectionPoint) -> bool: + return point.get("index") is None or _counts_back_from_the_end(point) def _accepts_cache_control(part: object) -> bool: @@ -898,17 +902,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): # A provisional message list defers every role-targeted point to the pass holding # the final one: a role with no message here may have one there, and settling all # of them in one pass is what lets config order decide the shared breakpoint - # budget. A negative ordinal is deferred too, since the final list only grows at - # the front and its tail holds the tool calls and results missing here. A - # non-negative ordinal names a different message once a later layer prepends one, - # so it is placed here or not at all. + # budget. A negative ordinal is deferred too, since the final list's tail holds the + # tool calls and results missing here, and it keeps its slot in this pass so an + # ordinal after it in config order cannot take it. A non-negative ordinal names a + # different message once a later layer prepends one, so it is placed here or not at all. carried_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( tuple(point for point in message_points if _placed_on_the_final_message_list(point)) if carry_unmatched else () ) - applied_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( - tuple(point for point in message_points if not _placed_on_the_final_message_list(point)) + first_pass_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( + tuple(point for point in message_points if point.get("index") is not None) if carry_unmatched else tuple(message_points) ) @@ -924,11 +928,12 @@ class AnthropicCacheControlHook(CustomPromptManagement): processed_messages, answered_tool_ids=answered_tool_ids ) processed_messages = self._apply_message_injections( - points=applied_message_points, + points=first_pass_message_points, messages=processed_messages, max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks, openai_dialect=openai_dialect, answered_tool_ids=answered_tool_ids, + placed_by_a_later_pass=_counts_back_from_the_end if carry_unmatched else None, ) if ( openai_dialect @@ -1103,6 +1108,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): max_blocks: int, openai_dialect: bool = False, answered_tool_ids: frozenset[str] | None = None, + placed_by_a_later_pass: Callable[[CacheControlMessageInjectionPoint], bool] | None = None, ) -> list[AllMessageValues]: """Apply message-level cache control injection points in order. @@ -1111,7 +1117,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): limit, so we never inject onto a message that already carries cache_control (preserving the client's TTL) and we stop injecting once ``max_blocks`` is reached. Injection points are honored in config order, - so earlier points win when slots are scarce. + so earlier points win when slots are scarce. A point + ``placed_by_a_later_pass`` marks nothing here but keeps its slot. """ client_answers: Final = ( _client_tool_call_ids_when_a_server_tool_call_is_marked(messages) @@ -1128,6 +1135,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): limit_reached = True break + if placed_by_a_later_pass is not None and placed_by_a_later_pass(point): + used_blocks += 1 + continue + control: ChatCompletionCachedContent = point.get("control", None) or ChatCompletionCachedContent( type="ephemeral" ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 82b0377083a..deae6643300 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -100,6 +100,7 @@ from litellm.types.utils import ( all_litellm_params, ) from litellm.utils import ( + LITELLM_ONLY_PARAMS, ProviderConfigManager, client, ) @@ -643,6 +644,7 @@ def _chat_completion_request_fields(allowed_openai_params: Sequence[str] | None) *DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *all_litellm_params, *GenericLiteLLMParams.model_fields, + *LITELLM_ONLY_PARAMS, *(allowed_openai_params or ()), ) ).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED) diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 210488f0888..249cffc97e2 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -387,6 +387,43 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" + def test_bridge_marks_the_last_tool_result_for_an_end_relative_cache_point(self, respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://openrouter.ai/api/v1/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "gen-123", + "object": "chat.completion", + "created": 1677652288, + "model": "anthropic/claude-sonnet-5.5", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.responses( + model="openrouter/anthropic/claude-sonnet-5.5", + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "Read a.txt."}]}, + {"type": "function_call", "call_id": "call_a", "name": "read_file", "arguments": '{"path": "a.txt"}'}, + {"type": "function_call_output", "call_id": "call_a", "output": "contents of a"}, + ], + use_chat_completions_api=True, + cache_control_injection_points=[{"location": "message", "index": -1}], + api_key="fake-provider-api-key", + num_retries=0, + ) + + sent_messages: Final = json.loads(upstream.calls[0].request.read())["messages"] + assert [ + message.get("tool_call_id") or message["role"] + for message in sent_messages + if "cache_control" in json.dumps(message) + ] == ["call_a"] + def test_bridge_keeps_deployment_credentials_while_dropping_unknown_params(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post( "https://example-resource.openai.azure.com/openai/deployments/my-deployment/chat/completions", diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index e6966bbe5e6..47f8d37e667 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -6,6 +6,7 @@ in expected_responses_api_request/. import asyncio import copy +import itertools import json import logging import math @@ -1076,7 +1077,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo assert "cache_control_injection_points" not in body -async def _bridged_body(mock_post, *, points, input, instructions="You are a documentation assistant."): +async def _bridged_body(mock_post, *, points, input, instructions="You are a documentation assistant.", tools=None): injected_client = AsyncHTTPHandler() injected_client.post = mock_post @@ -1086,6 +1087,7 @@ async def _bridged_body(mock_post, *, points, input, instructions="You are a doc instructions=instructions, input=copy.deepcopy(input), cache_control_injection_points=copy.deepcopy(points), + tools=tools, client=injected_client, ) return _sent_body(mock_post) @@ -1141,7 +1143,7 @@ async def test_positional_points_address_the_input_item_the_caller_indexed(instr @pytest.mark.asyncio async def test_out_of_bounds_positional_points_are_not_revived_by_a_longer_list(): - """An ordinal addresses the list in front of the pass that reads it. + """A non-negative ordinal counts the caller's `input` messages. Carrying one forward would re-resolve it against the bridge's longer list, where an index that named nothing in the caller's `input` can land on a real message -- the @@ -1184,6 +1186,44 @@ async def test_end_relative_points_mark_the_tool_results_that_follow_the_last_me assert marked_blocks == ["call_b"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("indices", "expected"), + [ + pytest.param((-1, 0), ["call_a"], id="end-relative-point-first"), + pytest.param((0, -1), ["u0"], id="positional-point-first"), + ], +) +async def test_the_ordinal_point_listed_first_takes_the_last_breakpoint_slot(indices, expected): + mock_post: Final = AsyncMock(return_value=MockResponse(_ANTHROPIC_MESSAGES_PAYLOAD, 200)) + body: Final = await _bridged_body( + mock_post, + points=[{"location": "message", "index": index} for index in indices], + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "u0"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "u1"}]}, + {"type": "function_call", "call_id": "call_a", "name": "t0", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_a", "output": "out a"}, + ], + tools=[ + { + "type": "function", + "name": name, + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + for name in ("t0", "t1", "t2") + ], + ) + + marked_blocks: Final = [ + block.get("tool_use_id") or block.get("text") + for block in itertools.chain.from_iterable(message["content"] for message in body["messages"]) + if "cache_control" in block + ] + assert marked_blocks == expected + + @pytest.mark.asyncio async def test_a_non_numeric_index_marks_nothing_and_the_request_still_goes_out(): mock_post = AsyncMock(return_value=MockResponse(_ANTHROPIC_MESSAGES_PAYLOAD, 200))