From 541d0f5cb1762ce5f2d97dc1daf5df1dba70acf8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:47:57 -0700 Subject: [PATCH] fix(cache_control): keep litellm_proxy routing when the chat path reads the provider prefix With USE_LITELLM_PROXY or litellm.use_litellm_proxy on, every call goes to the upstream proxy, so the provider prefix no longer decides the cache format there and resolution matches main. The plain OpenAI check reads the same provider value as the hosted check, so one function no longer mixes two lookups --- .../anthropic_cache_control_hook.py | 7 ++++-- .../test_anthropic_cache_control_hook.py | 22 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 3a92f3fcc65..8c29f82280c 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -127,6 +127,8 @@ def _served_provider_for_model(model: str) -> str | None: def _declared_provider(model: str) -> str | None: import litellm + if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(): + return None prefix, separator, _ = model.partition("/") return prefix if separator and prefix in litellm.provider_list else None @@ -517,14 +519,15 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> bool: if model is None: return False + declared_provider: Final = custom_llm_provider or _declared_provider(model) hosted_flag: Final = _hosted_openai_dialect_flag( - model, custom_llm_provider or _declared_provider(model), AnthropicCacheControlHook._resolve_provider + model, declared_provider, AnthropicCacheControlHook._resolve_provider ) if hosted_flag is not None: return hosted_flag if not supports_openai_prompt_cache_breakpoint(model): return False - if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + if (declared_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 962716df703..5d6b6bc60bd 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4001,6 +4001,7 @@ class TestChatPathProviderStamp: class TestChatPathResolvesTheDeploymentProvider: MODEL = "azure_ai/gpt-6-astra" AMBIENT_API_BASE = "https://other-deployment.openai.azure.com" + UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] @@ -4010,17 +4011,19 @@ class TestChatPathResolvesTheDeploymentProvider: else: monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) - def _chat(self, custom_llm_provider: str | None, api_base: str) -> tuple[list[AllMessageValues], dict[str, object]]: + def _chat( + self, custom_llm_provider: str | None, api_base: str, model: str = MODEL + ) -> tuple[list[AllMessageValues], dict[str, object]]: seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=seeded, messages=copy.deepcopy(self.MESSAGES), - model=self.MODEL, + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, ) _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( - model=self.MODEL, + model=model, messages=copy.deepcopy(self.MESSAGES), non_default_params=seeded, prompt_id=None, @@ -4042,6 +4045,19 @@ class TestChatPathResolvesTheDeploymentProvider: unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base) assert pinned_resolved == pinned_told == unset_resolved == unset_told assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1 + assert ("prompt_cache_options" in pinned_resolved[1]) == supports_openai_prompt_cache_breakpoint( + self.MODEL, "azure_ai" + ) + + @pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"]) + def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one( + self, monkeypatch: pytest.MonkeyPatch, model: str + ) -> None: + self._pin_ambient_api_base(monkeypatch, False) + monkeypatch.setenv("USE_LITELLM_PROXY", "True") + assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat( + "litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model + ) class TestClientBreakpointsCountedOnce: