From dd52d082590b6c79e6c47f69cc1c07159bfde980 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:04:43 -0800 Subject: [PATCH] fix(responses): keep config order for end-relative cache points and forward them on native-config bridge routes An end-relative point carried to the bridge's pass gave up its breakpoint slot on the Responses layer, so a non-negative point listed after it could take the last of the four slots. It now holds its slot in config order. Routes with their own Responses config (openrouter, azure and others with use_chat_completions_api) kept only chat completion request fields on the way into the bridge and dropped cache_control_injection_points, so every carried point marked nothing there. The bridge now forwards LITELLM_ONLY_PARAMS. Backport to lts/1.108.x of the main fix on litellm_responses_bridge_negative_cache_index. Cherry-picked from 8af15814f1ae4e72220028e26321af893ab0a50b, applied clean. --- .../anthropic_cache_control_hook.py | 31 ++++++++----- litellm/responses/main.py | 2 + .../test_responses_api_bridge_flag.py | 37 ++++++++++++++++ .../test_responses_api_request_body.py | 44 ++++++++++++++++++- 4 files changed, 102 insertions(+), 12 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index eb634b99e8a..663d1c5e05f 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -353,9 +353,13 @@ def _message_point_ordinal(point: CacheControlMessageInjectionPoint) -> int | No return None -def _placed_on_the_final_message_list(point: CacheControlMessageInjectionPoint) -> bool: +def _counts_back_from_the_end(point: CacheControlMessageInjectionPoint) -> bool: ordinal: Final = _message_point_ordinal(point) - return point.get("index") is None or (ordinal is not None and ordinal < 0) + return ordinal is not None and ordinal < 0 + + +def _placed_on_the_final_message_list(point: CacheControlMessageInjectionPoint) -> bool: + return point.get("index") is None or _counts_back_from_the_end(point) class AnthropicCacheControlHook(CustomPromptManagement): @@ -444,17 +448,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): # A provisional message list defers every role-targeted point to the pass holding # the final one: a role with no message here may have one there, and settling all # of them in one pass is what lets config order decide the shared breakpoint - # budget. A negative ordinal is deferred too, since the final list only grows at - # the front and its tail holds the tool calls and results missing here. A - # non-negative ordinal names a different message once a later layer prepends one, - # so it is placed here or not at all. + # budget. A negative ordinal is deferred too, since the final list's tail holds the + # tool calls and results missing here, and it keeps its slot in this pass so an + # ordinal after it in config order cannot take it. A non-negative ordinal names a + # different message once a later layer prepends one, so it is placed here or not at all. carried_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( tuple(point for point in message_points if _placed_on_the_final_message_list(point)) if carry_unmatched else () ) - applied_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( - tuple(point for point in message_points if not _placed_on_the_final_message_list(point)) + first_pass_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = ( + tuple(point for point in message_points if point.get("index") is not None) if carry_unmatched else tuple(message_points) ) @@ -470,11 +474,12 @@ class AnthropicCacheControlHook(CustomPromptManagement): processed_messages, answered_tool_ids=answered_tool_ids ) processed_messages = self._apply_message_injections( - points=applied_message_points, + points=first_pass_message_points, messages=processed_messages, max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks, openai_dialect=openai_dialect, answered_tool_ids=answered_tool_ids, + placed_by_a_later_pass=_counts_back_from_the_end if carry_unmatched else None, ) if ( openai_dialect @@ -621,6 +626,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): max_blocks: int, openai_dialect: bool = False, answered_tool_ids: frozenset[str] | None = None, + placed_by_a_later_pass: Callable[[CacheControlMessageInjectionPoint], bool] | None = None, ) -> list[AllMessageValues]: """Apply message-level cache control injection points in order. @@ -629,7 +635,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): limit, so we never inject onto a message that already carries cache_control (preserving the client's TTL) and we stop injecting once ``max_blocks`` is reached. Injection points are honored in config order, - so earlier points win when slots are scarce. + so earlier points win when slots are scarce. A point + ``placed_by_a_later_pass`` marks nothing here but keeps its slot. """ client_answers: Final = ( _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages) @@ -646,6 +653,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): limit_reached = True break + if placed_by_a_later_pass is not None and placed_by_a_later_pass(point): + used_blocks += 1 + continue + control: ChatCompletionCachedContent = point.get("control", None) or ChatCompletionCachedContent( type="ephemeral" ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 9c3f85d2426..7295d2be2f9 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -66,6 +66,7 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import all_litellm_params from litellm.utils import ( + LITELLM_ONLY_PARAMS, ProviderConfigManager, client, ) @@ -457,6 +458,7 @@ def _bridge_kwargs( *DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *all_litellm_params, *GenericLiteLLMParams.model_fields, + *LITELLM_ONLY_PARAMS, *(allowed_openai_params or ()), ) ).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED) diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 699144a31bb..74cfbe55cc8 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -404,6 +404,43 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" + def test_bridge_marks_the_last_tool_result_for_an_end_relative_cache_point(self, respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://openrouter.ai/api/v1/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "gen-123", + "object": "chat.completion", + "created": 1677652288, + "model": "anthropic/claude-sonnet-5.5", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.responses( + model="openrouter/anthropic/claude-sonnet-5.5", + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "Read a.txt."}]}, + {"type": "function_call", "call_id": "call_a", "name": "read_file", "arguments": '{"path": "a.txt"}'}, + {"type": "function_call_output", "call_id": "call_a", "output": "contents of a"}, + ], + use_chat_completions_api=True, + cache_control_injection_points=[{"location": "message", "index": -1}], + api_key="fake-provider-api-key", + num_retries=0, + ) + + sent_messages: Final = json.loads(upstream.calls[0].request.read())["messages"] + assert [ + message.get("tool_call_id") or message["role"] + for message in sent_messages + if "cache_control" in json.dumps(message) + ] == ["call_a"] + def test_bridge_keeps_deployment_credentials_while_dropping_unknown_params(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post( "https://example-resource.openai.azure.com/openai/deployments/my-deployment/chat/completions", diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index b5a7efd18fb..7a1e12a7beb 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -5,6 +5,7 @@ in expected_responses_api_request/. """ import copy +import itertools import json from pathlib import Path from importlib import import_module @@ -883,7 +884,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo assert "cache_control_injection_points" not in body -async def _bridged_body(mock_post, *, points, input, instructions="You are a documentation assistant."): +async def _bridged_body(mock_post, *, points, input, instructions="You are a documentation assistant.", tools=None): injected_client = AsyncHTTPHandler() injected_client.post = mock_post @@ -893,6 +894,7 @@ async def _bridged_body(mock_post, *, points, input, instructions="You are a doc instructions=instructions, input=copy.deepcopy(input), cache_control_injection_points=copy.deepcopy(points), + tools=tools, client=injected_client, ) return _sent_body(mock_post) @@ -948,7 +950,7 @@ async def test_positional_points_address_the_input_item_the_caller_indexed(instr @pytest.mark.asyncio async def test_out_of_bounds_positional_points_are_not_revived_by_a_longer_list(): - """An ordinal addresses the list in front of the pass that reads it. + """A non-negative ordinal counts the caller's `input` messages. Carrying one forward would re-resolve it against the bridge's longer list, where an index that named nothing in the caller's `input` can land on a real message -- the @@ -991,6 +993,44 @@ async def test_end_relative_points_mark_the_tool_results_that_follow_the_last_me assert marked_blocks == ["call_b"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("indices", "expected"), + [ + pytest.param((-1, 0), ["call_a"], id="end-relative-point-first"), + pytest.param((0, -1), ["u0"], id="positional-point-first"), + ], +) +async def test_the_ordinal_point_listed_first_takes_the_last_breakpoint_slot(indices, expected): + mock_post: Final = AsyncMock(return_value=MockResponse(_ANTHROPIC_MESSAGES_PAYLOAD, 200)) + body: Final = await _bridged_body( + mock_post, + points=[{"location": "message", "index": index} for index in indices], + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "u0"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "u1"}]}, + {"type": "function_call", "call_id": "call_a", "name": "t0", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_a", "output": "out a"}, + ], + tools=[ + { + "type": "function", + "name": name, + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + for name in ("t0", "t1", "t2") + ], + ) + + marked_blocks: Final = [ + block.get("tool_use_id") or block.get("text") + for block in itertools.chain.from_iterable(message["content"] for message in body["messages"]) + if "cache_control" in block + ] + assert marked_blocks == expected + + @pytest.mark.asyncio async def test_a_non_numeric_index_marks_nothing_and_the_request_still_goes_out(): mock_post = AsyncMock(return_value=MockResponse(_ANTHROPIC_MESSAGES_PAYLOAD, 200))