diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index abdbfc79d41..eb634b99e8a 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -333,15 +333,31 @@ def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: # Set by a caller whose message list is not the one that goes upstream -- today the -# Responses API layer, whose `instructions` only becomes a system message further down. -# Tells this hook to hand role-targeted points to the pass holding the final messages -# rather than spending them on a list that is still missing some of their targets. +# Responses API layer, whose `instructions` only becomes a system message further down +# and whose tool calls and results only become messages there. Tells this hook to hand +# role-targeted and end-relative points to the pass holding the final messages rather +# than spending them on a list that is still missing some of their targets. CARRY_UNMATCHED_MESSAGE_POINTS: Final = "_litellm_carry_unmatched_cache_control_points" EXTERNAL_BREAKPOINTS_STAMP: Final = "_litellm_external_breakpoints" CUSTOM_LLM_PROVIDER_STAMP: Final = "_litellm_custom_llm_provider" +def _message_point_ordinal(point: CacheControlMessageInjectionPoint) -> int | None: + index: Final = point.get("index") + if not isinstance(index, str): + return index + try: + return int(index) + except ValueError: + return None + + +def _placed_on_the_final_message_list(point: CacheControlMessageInjectionPoint) -> bool: + ordinal: Final = _message_point_ordinal(point) + return point.get("index") is None or (ordinal is not None and ordinal < 0) + + class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _request_value(request_kwargs: object, key: str) -> object: @@ -428,13 +444,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): # A provisional message list defers every role-targeted point to the pass holding # the final one: a role with no message here may have one there, and settling all # of them in one pass is what lets config order decide the shared breakpoint - # budget. An ordinal names a different message once a later layer builds its own - # list, so it is placed here or not at all. + # budget. A negative ordinal is deferred too, since the final list only grows at + # the front and its tail holds the tool calls and results missing here. A + # non-negative ordinal names a different message once a later layer prepends one, + # so it is placed here or not at all. carried_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( - tuple(point for point in message_points if point.get("index") is None) if carry_unmatched else () + tuple(point for point in message_points if _placed_on_the_final_message_list(point)) + if carry_unmatched + else () ) applied_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( - tuple(point for point in message_points if point.get("index") is not None) + tuple(point for point in message_points if not _placed_on_the_final_message_list(point)) if carry_unmatched else tuple(message_points) ) @@ -661,15 +681,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues] ) -> list[int]: """Resolve which message indices an injection point targets.""" - _targetted_index: Final[int | str | None] = point.get("index", None) - targetted_index: int | None = None - if isinstance(_targetted_index, str): - try: - targetted_index = int(_targetted_index) - except ValueError: - pass - else: - targetted_index = _targetted_index + targetted_index: int | None = _message_point_ordinal(point) # Case 1: Target by specific index if targetted_index is not None: diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 1948597aaf1..9c3f85d2426 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -659,8 +659,9 @@ def _prompt_management_sees_a_provisional_message_list( """Tell the cache-control hook that this layer's messages are not the ones sent upstream. A Responses request keeps its system prompt in ``instructions``, which only becomes a - system message when the chat-completion bridge builds one, so a role-targeted point - is placed by the bridge's pass rather than this one. + system message when the chat-completion bridge builds one, and its ``function_call`` and + ``function_call_output`` items, which only become messages there too. So a role-targeted + point and one counting back from the end are placed by the bridge's pass rather than this one. Only raised for a request that will be bridged. A provider serving Responses natively gets no second pass, so this layer is the last one that can place anything and handing diff --git a/tests/integration/providers/test_cache_control_tool_call_marks_wire.py b/tests/integration/providers/test_cache_control_tool_call_marks_wire.py index 47d28860018..68e6bc2857b 100644 --- a/tests/integration/providers/test_cache_control_tool_call_marks_wire.py +++ b/tests/integration/providers/test_cache_control_tool_call_marks_wire.py @@ -527,6 +527,35 @@ def test_responses_bridge_points_skip_injection_when_function_call_marks_fill_th assert anthropic_labels(_only_request(wire)) == _CLIENT_MARKED +def _responses_body_ending_in_tool_results(model: str, marker: str) -> dict[str, JsonValue]: + items: Final[list[JsonValue]] = [ + {"role": "user", "content": [{"type": "input_text", "text": ASK}]}, + *( + { + "type": "function_call", + "call_id": call_id(city), + "name": "lookup_weather", + "arguments": json.dumps({"city": city}), + } + for city in CITIES + ), + *( + {"type": "function_call_output", "call_id": call_id(city), "output": f"sunny. marker-{marker}"} + for city in CITIES + ), + ] + return {**responses_body(model, marker), "input": items} + + +def test_responses_bridge_last_message_point_marks_the_trailing_tool_result(gateway: Gateway) -> None: + marker: Final = new_marker() + with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_deployment(scenario, wire, cache_control_injection_points=POINTS) + response: Final = gateway.request("POST", "/v1/responses", _responses_body_ending_in_tool_results(model, marker)) + assert response.status_code == 200, response.text + assert anthropic_labels(_only_request(wire)) == [SYSTEM_LABEL, f"user:tool_result:{call_id(CITIES[-1])}"] + + @pytest.mark.timeout(_OWNED_PROXY_TIMEOUT) def test_response_cache_serves_the_capped_request_once(uncapped_gateway: Gateway) -> None: marker: Final = new_marker() diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 22db24f6ffd..1416a1c4b8e 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4522,14 +4522,15 @@ class TestResponsesLayerReservesMarksOutsideTheProvisionalMessages: *earlier, last = self._items(marked_calls) items = [*earlier, *other_items, last] + provisional_messages = ResponsesAPIRequestUtils.responses_input_to_chat_messages(items) kwargs: dict = { - "cache_control_injection_points": [{"location": "message", "index": -1}], + "cache_control_injection_points": [{"location": "message", "index": len(provisional_messages) - 1}], CARRY_UNMATCHED_MESSAGE_POINTS: True, } _stamp_injection_points_for_the_hook(kwargs, self.MODEL, "anthropic", items, self._tools(marked_tools)) _, messages, _ = AnthropicCacheControlHook().get_chat_completion_prompt( model=self.MODEL, - messages=ResponsesAPIRequestUtils.responses_input_to_chat_messages(items), + messages=provisional_messages, non_default_params=kwargs, prompt_id=None, prompt_variables=None, diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 8c8b347be82..b5a7efd18fb 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -966,6 +966,44 @@ async def test_out_of_bounds_positional_points_are_not_revived_by_a_longer_list( assert "cache_control" not in json.dumps(body["messages"]) +@pytest.mark.asyncio +@pytest.mark.parametrize("index", [-1, "-1"], ids=["int", "str"]) +async def test_end_relative_points_mark_the_tool_results_that_follow_the_last_message(index): + mock_post = AsyncMock(return_value=MockResponse(_ANTHROPIC_MESSAGES_PAYLOAD, 200)) + body = await _bridged_body( + mock_post, + points=[{"location": "message", "index": index}], + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "Read a.txt and b.txt."}]}, + {"type": "function_call", "call_id": "call_a", "name": "read_file", "arguments": '{"path": "a.txt"}'}, + {"type": "function_call", "call_id": "call_b", "name": "read_file", "arguments": '{"path": "b.txt"}'}, + {"type": "function_call_output", "call_id": "call_a", "output": "contents of a"}, + {"type": "function_call_output", "call_id": "call_b", "output": "contents of b"}, + ], + ) + + marked_blocks = [ + block.get("tool_use_id") or block.get("id") or block.get("text") + for message in body["messages"] + for block in message["content"] + if "cache_control" in json.dumps(block) + ] + assert marked_blocks == ["call_b"] + + +@pytest.mark.asyncio +async def test_a_non_numeric_index_marks_nothing_and_the_request_still_goes_out(): + mock_post = AsyncMock(return_value=MockResponse(_ANTHROPIC_MESSAGES_PAYLOAD, 200)) + body = await _bridged_body( + mock_post, + points=[{"location": "message", "index": "last"}], + input=[{"role": "user", "content": [{"type": "input_text", "text": "only item"}]}], + ) + + assert "cache_control" not in json.dumps(body["system"]) + assert "cache_control" not in json.dumps(body["messages"]) + + def _four_user_turns() -> list: return [ item