From 45601ff1d288ddaf247a039e38d1920f310b5656 Mon Sep 17 00:00:00 2001 From: Priyansh Nandwana Date: Sat, 29 Aug 2026 10:46:02 +0530 Subject: [PATCH 1/2] fix(cache_control): let a hosted deployment opt into the OpenAI cache dialect _targets_openai_prompt_cache_breakpoint gated on custom_llm_provider == "openai" unconditionally, so an OpenAI-shaped model served by another provider could never qualify, even with supports_prompt_cache_breakpoint set explicitly on its own cost-map entry. bedrock_mantle honours prompt_cache_breakpoint end to end, and had no way to say so. model_cost is keyed per exact deployment string, so a flag on the deployment's own entry states the dialect more precisely than a provider name can. Consult it before the provider check, and require the entry's litellm_provider to match the serving provider so an openai entry cannot license another provider. Entries for the openai provider keep their api_base check, so an OpenAI-compatible third-party host is still not assumed to speak the dialect. Fixes #38666 --- .../anthropic_cache_control_hook.py | 32 +++++++- .../test_anthropic_cache_control_hook.py | 75 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4f9b18713d0..867100cc77b 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -12,7 +12,7 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and import copy import os import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -104,6 +104,27 @@ def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: return next((bool(flag) for flag in flags if flag is not None), None) +def _hosted_openai_dialect_flag(model: str, resolve_provider: Callable[[], str | None]) -> bool | None: + """ + Explicit opt-in for an OpenAI-shaped deployment served by another provider. + + ``model_cost`` is keyed per exact deployment string, so a flag on the deployment's + own entry states the dialect directly, which a provider name cannot express. Entries + for the openai provider are left to the caller's api_base check, so an + OpenAI-compatible third-party host is still not assumed to speak the dialect. + """ + import litellm + + entry: Final = litellm.model_cost.get(model) + if not isinstance(entry, dict): + return None + flag: Final = entry.get("supports_prompt_cache_breakpoint") + entry_provider: Final = entry.get("litellm_provider") + if flag is None or entry_provider is None or entry_provider == "openai": + return None + return bool(flag) if entry_provider == resolve_provider() else None + + def targets_openai_api(api_base: object) -> bool: import litellm @@ -269,7 +290,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): api_base: object = None, prompt_cache_options: object = None, ) -> bool: - if model is None or not supports_openai_prompt_cache_breakpoint(model): + if model is None: + return False + hosted_flag: Final = _hosted_openai_dialect_flag( + model, lambda: custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model) + ) + if hosted_flag is not None: + return hosted_flag + if not supports_openai_prompt_cache_breakpoint(model): return False if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 6b7780acd20..41cb371f0b9 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -3049,6 +3049,81 @@ class TestPromptCacheBreakpointCapability: assert supports_openai_prompt_cache_breakpoint(model) is expected +class TestHostedOpenAIDialectFlag: + """#38666: an OpenAI-shaped model served by another provider can opt in through its own + model-map entry, instead of being excluded by the openai-only provider check.""" + + MANTLE_MODEL = "bedrock_mantle/openai.gpt-5.6-sol" + + def _register(self, monkeypatch, key, provider, flag=True, **extra): + entry = {"litellm_provider": provider, "mode": "chat", **extra} + if flag is not None: + entry["supports_prompt_cache_breakpoint"] = flag + monkeypatch.setitem(litellm.model_cost, key, entry) + + def test_flagged_non_openai_deployment_is_eligible(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is True + ) + + def test_bedrock_api_base_does_not_veto_the_explicit_flag(self, monkeypatch): + """The api_base check exists to sniff for api.openai.com, which a Bedrock host never is.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + self.MANTLE_MODEL, + "bedrock_mantle", + api_base="https://bedrock-runtime.us-east-1.amazonaws.com", + ) + is True + ) + + def test_flag_set_false_keeps_the_deployment_ineligible(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False) + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is False + ) + + def test_unflagged_non_openai_deployment_stays_ineligible(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=None) + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is False + ) + + def test_entry_provider_must_match_the_request_provider(self, monkeypatch): + """A flagged entry does not license a different provider serving the same model string.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "azure") is False + ) + + def test_openai_entries_still_go_through_the_api_base_check(self, monkeypatch): + """gpt-5.6 is flagged and openai-provided, so it must not bypass the host gate.""" + assert litellm.model_cost["gpt-5.6"]["supports_prompt_cache_breakpoint"] is True + assert litellm.model_cost["gpt-5.6"]["litellm_provider"] == "openai" + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + "gpt-5.6", "openai", api_base="https://some-compatible-host.example.com" + ) + is False + ) + + def test_azure_hosted_gpt_5_6_remains_ineligible(self): + """Regression guard: the openai entry's flag must not leak to another provider.""" + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False + + def test_provider_is_not_resolved_when_no_flag_is_present(self): + """The cheap model-map lookup must short-circuit before the provider lookup.""" + with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False + resolve.assert_not_called() + + class TestRecordGatewayInjection: """The injection marker spend accounting gates prompt-caching savings on.""" From 79b24c70c9993418642cbe02a064fdfa80cdd7a3 Mon Sep 17 00:00:00 2001 From: Priyansh Nandwana Date: Sat, 29 Aug 2026 11:02:27 +0530 Subject: [PATCH 2/2] test: drop a laziness assertion the existing suite already makes test_provider_lookup_skipped_for_models_below_gpt_5_6 already pins that _resolve_provider is not called for an unflagged model, and it covers the new flag lookup unchanged. The duplicate patched a litellm internal for no added coverage, which the test-quality gate counts against TQ008. --- .../integrations/test_anthropic_cache_control_hook.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 41cb371f0b9..9146f988c47 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -3117,12 +3117,6 @@ class TestHostedOpenAIDialectFlag: assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False - def test_provider_is_not_resolved_when_no_flag_is_present(self): - """The cheap model-map lookup must short-circuit before the provider lookup.""" - with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False - resolve.assert_not_called() - class TestRecordGatewayInjection: """The injection marker spend accounting gates prompt-caching savings on."""