From edc40a731cd1310cb89e512f17eb12f5a065731e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:52:53 -0700 Subject: [PATCH] fix(cache_control): stamp the OpenAI dialect for a bare deployment name served by a flagged provider A deployment written as `model: openai.gpt-5.6-sol` with `custom_llm_provider: bedrock_mantle` has no cost-map row of its own and no openai row of the same name, so the dialect stamp's cheap gate (`supports_openai_prompt_cache_breakpoint`) returned early and the configured point was never stamped. On `/v1/responses` and on the chat seeding path the hook sees no provider, so it fell back to the Anthropic `cache_control` marker, which Bedrock Mantle strips. The gate now also passes a model whose serving provider is already known and whose provider-keyed row carries the flag, which is the same row the dialect resolution reads and costs no provider lookup. A bare name without a provider is still left alone, so models below GPT-5.6 keep their points untouched and resolve nothing. --- .../anthropic_cache_control_hook.py | 16 +++++- .../test_anthropic_cache_control_hook.py | 55 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 89c753271e4..faded73d0f3 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -770,7 +770,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): api_base: object, prompt_cache_options: object, ) -> Sequence[Mapping[str, object]]: - if not supports_openai_prompt_cache_breakpoint(model): + if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider): return points return AnthropicCacheControlHook._stamped( points, @@ -780,6 +780,20 @@ class AnthropicCacheControlHook(CustomPromptManagement): ), ) + @staticmethod + def _may_target_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool: + """Cheap gate before the dialect is resolved: the model's own row or version, or, when the + serving provider is already known, a row keyed for that provider (``openai.gpt-5.6-sol`` + served by ``bedrock_mantle``), which costs no provider lookup.""" + if supports_openai_prompt_cache_breakpoint(model): + return True + if custom_llm_provider is None: + return False + return ( + _hosted_openai_dialect_flag(model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider) + is not None + ) + @staticmethod def _stamped(points: Sequence[Mapping[str, object]], key: str, value: object) -> Sequence[Mapping[str, object]]: return [{**point, key: value} for point in points] diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index dd824d7316a..f7119346566 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3994,6 +3994,61 @@ class TestBedrockMantleGptShipsTheOpenAIDialect: assert messages[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] assert params["prompt_cache_options"] == {"mode": "implicit"} + BARE_MODEL = "openai.gpt-5.6-sol" + POINTS = [{"location": "message", "role": "system"}] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + MARKED_SYSTEM = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + + def test_a_bare_deployment_name_with_its_provider_is_stamped_on_the_openai_dialect(self): + """A deployment written as ``model: openai.gpt-5.6-sol`` plus ``custom_llm_provider: bedrock_mantle`` + has no row of its own and no openai row of the same name, so only the provider-keyed row can + answer; the stamp must read it the way the dialect resolution does.""" + stamped = AnthropicCacheControlHook._stamped_with_dialect( + copy.deepcopy(self.POINTS), self.BARE_MODEL, "bedrock_mantle", None, None + ) + assert stamped[0]["_litellm_openai_dialect"] is True + + def test_a_bare_deployment_name_without_its_provider_keeps_its_points_and_costs_no_lookup(self): + points = copy.deepcopy(self.POINTS) + with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: + assert AnthropicCacheControlHook._stamped_with_dialect(points, self.BARE_MODEL, None, None, None) is points + resolve.assert_not_called() + + def test_the_chat_seed_carries_the_resolved_provider_for_a_bare_deployment_name(self): + params: dict = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model=self.BARE_MODEL, + custom_llm_provider="bedrock_mantle", + ) + _, messages, out = AnthropicCacheControlHook().get_chat_completion_prompt( + model=self.BARE_MODEL, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert messages[0]["content"] == self.MARKED_SYSTEM + assert out["prompt_cache_options"] == {"mode": "implicit"} + + def test_the_responses_stamp_carries_the_resolved_provider_for_a_bare_deployment_name(self): + from litellm.responses.main import _stamp_injection_points_with_dialect + + kwargs: dict = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + _stamp_injection_points_with_dialect(kwargs, self.BARE_MODEL, "bedrock_mantle") + _, messages, out = AnthropicCacheControlHook().get_chat_completion_prompt( + model=self.BARE_MODEL, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=kwargs, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert messages[0]["content"] == self.MARKED_SYSTEM + assert out["prompt_cache_options"] == {"mode": "implicit"} + class TestRecordGatewayInjection: """The injection marker spend accounting gates prompt-caching savings on."""