From 4245a543009848e4c6b276292a3316960f4b29de Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:53:06 -0700 Subject: [PATCH 1/8] fix(cache_control): resolve the chat-path dialect with the deployment api_base --- .../anthropic_cache_control_hook.py | 6 +-- .../test_anthropic_cache_control_hook.py | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 22d76242f2f..15fe5716966 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -312,17 +312,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> bool: if model is None or not supports_openai_prompt_cache_breakpoint(model): return False - if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) @staticmethod - def _resolve_provider(model: str) -> str | None: + def _resolve_provider(model: str, api_base: object = None) -> str | None: from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider try: - _, provider, _, _ = get_llm_provider(model=model) + _, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None) except BadRequestError: return None return provider diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 8392cef8a7b..b4fad4490c8 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3558,6 +3558,59 @@ class TestChatPathProviderStamp: assert params["prompt_cache_options"] == {"mode": "explicit"} +class TestChatPathResolvesTheDeploymentProvider: + """The chat path reads the dialect off the provider the dispatch resolves for the deployment, so the + ambient AZURE_AI_API_BASE never speaks for a Foundry deployment that carries its own api_base.""" + + MODEL = "azure_ai/gpt-6-astra" + FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com" + AZURE_OPENAI_API_BASE = "https://other-deployment.openai.azure.com" + POINTS = [{"location": "message", "role": "system"}] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + + def _pin_ambient_api_base(self, monkeypatch, pinned): + if pinned: + monkeypatch.setenv("AZURE_AI_API_BASE", self.AZURE_OPENAI_API_BASE) + else: + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + + @pytest.mark.parametrize("ambient_pinned", [True, False]) + def test_deployment_api_base_decides_the_provider_over_the_ambient_one(self, monkeypatch, ambient_pinned): + self._pin_ambient_api_base(monkeypatch, ambient_pinned) + assert AnthropicCacheControlHook._resolve_provider(self.MODEL, self.FOUNDRY_API_BASE) == "azure_ai" + + def test_ambient_api_base_still_decides_for_a_deployment_without_one(self, monkeypatch): + self._pin_ambient_api_base(monkeypatch, True) + assert AnthropicCacheControlHook._resolve_provider(self.MODEL, None) == "azure" + + def _chat(self, custom_llm_provider): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model=self.MODEL, + custom_llm_provider=custom_llm_provider, + api_base=self.FOUNDRY_API_BASE, + ) + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=self.MODEL, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_chat_path_marks_the_foundry_deployment_the_same_whatever_the_ambient_api_base(self, monkeypatch): + self._pin_ambient_api_base(monkeypatch, True) + pinned_resolved, pinned_told = self._chat(None), self._chat("azure_ai") + self._pin_ambient_api_base(monkeypatch, False) + unset_resolved, unset_told = self._chat(None), self._chat("azure_ai") + assert pinned_resolved == pinned_told == unset_resolved == unset_told + assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1 + + class TestClientBreakpointsCountedOnce: def test_client_message_breakpoints_are_not_double_counted(self): messages = [{"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]}] + [ From 7a373a27653a761c29cdcff01231f8673046e8a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:36:07 -0700 Subject: [PATCH 2/8] test(cache_control): type the chat-path resolver helpers --- .../test_anthropic_cache_control_hook.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index b4fad4490c8..54d5798a502 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3559,34 +3559,33 @@ class TestChatPathProviderStamp: class TestChatPathResolvesTheDeploymentProvider: - """The chat path reads the dialect off the provider the dispatch resolves for the deployment, so the - ambient AZURE_AI_API_BASE never speaks for a Foundry deployment that carries its own api_base.""" - MODEL = "azure_ai/gpt-6-astra" FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com" AZURE_OPENAI_API_BASE = "https://other-deployment.openai.azure.com" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] - def _pin_ambient_api_base(self, monkeypatch, pinned): + def _pin_ambient_api_base(self, monkeypatch: pytest.MonkeyPatch, pinned: bool) -> None: if pinned: monkeypatch.setenv("AZURE_AI_API_BASE", self.AZURE_OPENAI_API_BASE) else: monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) @pytest.mark.parametrize("ambient_pinned", [True, False]) - def test_deployment_api_base_decides_the_provider_over_the_ambient_one(self, monkeypatch, ambient_pinned): + def test_deployment_api_base_decides_the_provider_over_the_ambient_one( + self, monkeypatch: pytest.MonkeyPatch, ambient_pinned: bool + ) -> None: self._pin_ambient_api_base(monkeypatch, ambient_pinned) assert AnthropicCacheControlHook._resolve_provider(self.MODEL, self.FOUNDRY_API_BASE) == "azure_ai" - def test_ambient_api_base_still_decides_for_a_deployment_without_one(self, monkeypatch): + def test_ambient_api_base_still_decides_for_a_deployment_without_one(self, monkeypatch: pytest.MonkeyPatch) -> None: self._pin_ambient_api_base(monkeypatch, True) assert AnthropicCacheControlHook._resolve_provider(self.MODEL, None) == "azure" - def _chat(self, custom_llm_provider): - params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + def _chat(self, custom_llm_provider: str | None) -> tuple[list[AllMessageValues], dict]: + seeded: Final[dict] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} AnthropicCacheControlHook.maybe_seed_default_injection_points( - non_default_params=params, + non_default_params=seeded, messages=copy.deepcopy(self.MESSAGES), model=self.MODEL, custom_llm_provider=custom_llm_provider, @@ -3595,14 +3594,16 @@ class TestChatPathResolvesTheDeploymentProvider: _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( model=self.MODEL, messages=copy.deepcopy(self.MESSAGES), - non_default_params=params, + non_default_params=seeded, prompt_id=None, prompt_variables=None, dynamic_callback_params={}, ) return out, params - def test_chat_path_marks_the_foundry_deployment_the_same_whatever_the_ambient_api_base(self, monkeypatch): + def test_chat_path_marks_the_foundry_deployment_the_same_whatever_the_ambient_api_base( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self._pin_ambient_api_base(monkeypatch, True) pinned_resolved, pinned_told = self._chat(None), self._chat("azure_ai") self._pin_ambient_api_base(monkeypatch, False) From 5fdc7321a934fd9b513d27b028249ed5623a00a5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:42:27 -0700 Subject: [PATCH 3/8] fix(cache_control): pick the chat-path dialect from the deployment's provider prefix Resolving with the deployment api_base followed the dispatch's azure_ai to azure flip, so chat on a non-Foundry Azure host sent Anthropic cache_control while /v1/messages, told azure_ai, sent the OpenAI breakpoint. Use the model's provider prefix when the caller does not name a provider, so chat matches the told path whatever AZURE_AI_API_BASE holds, and keep name-only resolution everywhere else. --- .../anthropic_cache_control_hook.py | 22 ++++++++++-------- .../test_anthropic_cache_control_hook.py | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 3036a320dfa..3a92f3fcc65 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -113,17 +113,24 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict)) -def _served_provider_for_model(model: str, api_base: str | None = None) -> str | None: +def _served_provider_for_model(model: str) -> str | None: from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider try: - _, provider, _, _ = get_llm_provider(model=model, api_base=api_base) + _, provider, _, _ = get_llm_provider(model=model) except BadRequestError: return None return provider +def _declared_provider(model: str) -> str | None: + import litellm + + prefix, separator, _ = model.partition("/") + return prefix if separator and prefix in litellm.provider_list else None + + def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: hosted_flag: Final = ( None @@ -510,23 +517,20 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> bool: if model is None: return False - deployment_api_base: Final = api_base if isinstance(api_base, str) else None hosted_flag: Final = _hosted_openai_dialect_flag( - model, - custom_llm_provider, - lambda name: AnthropicCacheControlHook._resolve_provider(name, deployment_api_base), + model, custom_llm_provider or _declared_provider(model), AnthropicCacheControlHook._resolve_provider ) if hosted_flag is not None: return hosted_flag if not supports_openai_prompt_cache_breakpoint(model): return False - if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, deployment_api_base)) != "openai": + if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) @staticmethod - def _resolve_provider(model: str, api_base: str | None = None) -> str | None: - return _served_provider_for_model(model, api_base) + def _resolve_provider(model: str) -> str | None: + return _served_provider_for_model(model) @staticmethod def count_request_cache_breakpoints( diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 9f59849d1c7..962716df703 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4000,25 +4000,24 @@ class TestChatPathProviderStamp: class TestChatPathResolvesTheDeploymentProvider: MODEL = "azure_ai/gpt-6-astra" - FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com" - AZURE_OPENAI_API_BASE = "https://other-deployment.openai.azure.com" + AMBIENT_API_BASE = "https://other-deployment.openai.azure.com" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] def _pin_ambient_api_base(self, monkeypatch: pytest.MonkeyPatch, pinned: bool) -> None: if pinned: - monkeypatch.setenv("AZURE_AI_API_BASE", self.AZURE_OPENAI_API_BASE) + monkeypatch.setenv("AZURE_AI_API_BASE", self.AMBIENT_API_BASE) else: monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) - def _chat(self, custom_llm_provider: str | None) -> tuple[list[AllMessageValues], dict]: - seeded: Final[dict] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + def _chat(self, custom_llm_provider: str | None, api_base: str) -> tuple[list[AllMessageValues], dict[str, object]]: + seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=seeded, messages=copy.deepcopy(self.MESSAGES), model=self.MODEL, custom_llm_provider=custom_llm_provider, - api_base=self.FOUNDRY_API_BASE, + api_base=api_base, ) _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( model=self.MODEL, @@ -4030,13 +4029,17 @@ class TestChatPathResolvesTheDeploymentProvider: ) return out, params - def test_chat_path_marks_the_foundry_deployment_the_same_whatever_the_ambient_api_base( - self, monkeypatch: pytest.MonkeyPatch + @pytest.mark.parametrize( + "api_base", + ["https://foundry.services.ai.azure.com", "https://resource.cognitiveservices.azure.com"], + ) + def test_chat_path_marks_the_deployment_the_same_as_when_told_its_provider( + self, monkeypatch: pytest.MonkeyPatch, api_base: str ) -> None: self._pin_ambient_api_base(monkeypatch, True) - pinned_resolved, pinned_told = self._chat(None), self._chat("azure_ai") + pinned_resolved, pinned_told = self._chat(None, api_base), self._chat("azure_ai", api_base) self._pin_ambient_api_base(monkeypatch, False) - unset_resolved, unset_told = self._chat(None), self._chat("azure_ai") + unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base) assert pinned_resolved == pinned_told == unset_resolved == unset_told assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1 From 541d0f5cb1762ce5f2d97dc1daf5df1dba70acf8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:47:57 -0700 Subject: [PATCH 4/8] fix(cache_control): keep litellm_proxy routing when the chat path reads the provider prefix With USE_LITELLM_PROXY or litellm.use_litellm_proxy on, every call goes to the upstream proxy, so the provider prefix no longer decides the cache format there and resolution matches main. The plain OpenAI check reads the same provider value as the hosted check, so one function no longer mixes two lookups --- .../anthropic_cache_control_hook.py | 7 ++++-- .../test_anthropic_cache_control_hook.py | 22 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 3a92f3fcc65..8c29f82280c 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -127,6 +127,8 @@ def _served_provider_for_model(model: str) -> str | None: def _declared_provider(model: str) -> str | None: import litellm + if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(): + return None prefix, separator, _ = model.partition("/") return prefix if separator and prefix in litellm.provider_list else None @@ -517,14 +519,15 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> bool: if model is None: return False + declared_provider: Final = custom_llm_provider or _declared_provider(model) hosted_flag: Final = _hosted_openai_dialect_flag( - model, custom_llm_provider or _declared_provider(model), AnthropicCacheControlHook._resolve_provider + model, declared_provider, AnthropicCacheControlHook._resolve_provider ) if hosted_flag is not None: return hosted_flag if not supports_openai_prompt_cache_breakpoint(model): return False - if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + if (declared_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 962716df703..5d6b6bc60bd 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4001,6 +4001,7 @@ class TestChatPathProviderStamp: class TestChatPathResolvesTheDeploymentProvider: MODEL = "azure_ai/gpt-6-astra" AMBIENT_API_BASE = "https://other-deployment.openai.azure.com" + UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] @@ -4010,17 +4011,19 @@ class TestChatPathResolvesTheDeploymentProvider: else: monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) - def _chat(self, custom_llm_provider: str | None, api_base: str) -> tuple[list[AllMessageValues], dict[str, object]]: + def _chat( + self, custom_llm_provider: str | None, api_base: str, model: str = MODEL + ) -> tuple[list[AllMessageValues], dict[str, object]]: seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=seeded, messages=copy.deepcopy(self.MESSAGES), - model=self.MODEL, + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, ) _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( - model=self.MODEL, + model=model, messages=copy.deepcopy(self.MESSAGES), non_default_params=seeded, prompt_id=None, @@ -4042,6 +4045,19 @@ class TestChatPathResolvesTheDeploymentProvider: unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base) assert pinned_resolved == pinned_told == unset_resolved == unset_told assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1 + assert ("prompt_cache_options" in pinned_resolved[1]) == supports_openai_prompt_cache_breakpoint( + self.MODEL, "azure_ai" + ) + + @pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"]) + def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one( + self, monkeypatch: pytest.MonkeyPatch, model: str + ) -> None: + self._pin_ambient_api_base(monkeypatch, False) + monkeypatch.setenv("USE_LITELLM_PROXY", "True") + assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat( + "litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model + ) class TestClientBreakpointsCountedOnce: From a5bf36f5d9c70f5e6482ebda15fecc69d1dd39bf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:36:56 -0700 Subject: [PATCH 5/8] fix(cache_control): resolve the chat-path dialect with the deployment api_base, as the call is routed The hosted cache-format check now resolves the provider with the deployment's own api_base when the caller names none, the same provider the call dispatches to and the Responses path passes, so AZURE_AI_API_BASE no longer decides the chat format. The plain OpenAI check keeps main's name-only lookup. --- .../anthropic_cache_control_hook.py | 22 +++------- .../test_anthropic_cache_control_hook.py | 41 +++++++++++-------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 8c29f82280c..a8c272d7476 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -113,26 +113,17 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict)) -def _served_provider_for_model(model: str) -> str | None: +def _served_provider_for_model(model: str, api_base: object = None) -> str | None: from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider try: - _, provider, _, _ = get_llm_provider(model=model) + _, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None) except BadRequestError: return None return provider -def _declared_provider(model: str) -> str | None: - import litellm - - if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(): - return None - prefix, separator, _ = model.partition("/") - return prefix if separator and prefix in litellm.provider_list else None - - def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: hosted_flag: Final = ( None @@ -519,21 +510,20 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) -> bool: if model is None: return False - declared_provider: Final = custom_llm_provider or _declared_provider(model) hosted_flag: Final = _hosted_openai_dialect_flag( - model, declared_provider, AnthropicCacheControlHook._resolve_provider + model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base) ) if hosted_flag is not None: return hosted_flag if not supports_openai_prompt_cache_breakpoint(model): return False - if (declared_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) @staticmethod - def _resolve_provider(model: str) -> str | None: - return _served_provider_for_model(model) + def _resolve_provider(model: str, api_base: object = None) -> str | None: + return _served_provider_for_model(model, api_base) @staticmethod def count_request_cache_breakpoints( diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 5d6b6bc60bd..fa03879c05a 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4001,6 +4001,7 @@ class TestChatPathProviderStamp: class TestChatPathResolvesTheDeploymentProvider: MODEL = "azure_ai/gpt-6-astra" AMBIENT_API_BASE = "https://other-deployment.openai.azure.com" + FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com" UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] @@ -4032,29 +4033,37 @@ class TestChatPathResolvesTheDeploymentProvider: ) return out, params - @pytest.mark.parametrize( - "api_base", - ["https://foundry.services.ai.azure.com", "https://resource.cognitiveservices.azure.com"], - ) - def test_chat_path_marks_the_deployment_the_same_as_when_told_its_provider( - self, monkeypatch: pytest.MonkeyPatch, api_base: str + def test_chat_path_stamps_the_breakpoint_on_a_foundry_deployment_while_the_ambient_base_is_azure_openai( + self, monkeypatch: pytest.MonkeyPatch ) -> None: self._pin_ambient_api_base(monkeypatch, True) - pinned_resolved, pinned_told = self._chat(None, api_base), self._chat("azure_ai", api_base) - self._pin_ambient_api_base(monkeypatch, False) - unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base) - assert pinned_resolved == pinned_told == unset_resolved == unset_told - assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1 - assert ("prompt_cache_options" in pinned_resolved[1]) == supports_openai_prompt_cache_breakpoint( - self.MODEL, "azure_ai" - ) + messages, params = self._chat(None, self.FOUNDRY_API_BASE) + assert (messages, params) == self._chat("azure_ai", self.FOUNDRY_API_BASE) + assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1 + assert ("prompt_cache_options" in params) == supports_openai_prompt_cache_breakpoint(self.MODEL, "azure_ai") + + @pytest.mark.parametrize("pinned", [True, False]) + @pytest.mark.parametrize( + "api_base", + [FOUNDRY_API_BASE, "https://resource.cognitiveservices.azure.com", "https://resource.openai.azure.com"], + ) + def test_chat_path_marks_the_deployment_like_the_provider_its_api_base_dispatches_to( + self, monkeypatch: pytest.MonkeyPatch, api_base: str, pinned: bool + ) -> None: + self._pin_ambient_api_base(monkeypatch, pinned) + dispatched_provider: Final = litellm.get_llm_provider(model=self.MODEL, api_base=api_base)[1] + assert self._chat(None, api_base) == self._chat(dispatched_provider, api_base) @pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"]) + @pytest.mark.parametrize("set_globally", ["env", "module"]) def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one( - self, monkeypatch: pytest.MonkeyPatch, model: str + self, monkeypatch: pytest.MonkeyPatch, model: str, set_globally: str ) -> None: self._pin_ambient_api_base(monkeypatch, False) - monkeypatch.setenv("USE_LITELLM_PROXY", "True") + if set_globally == "env": + monkeypatch.setenv("USE_LITELLM_PROXY", "True") + else: + monkeypatch.setattr(litellm, "use_litellm_proxy", True) assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat( "litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model ) From accfc9a6ccbce38e1bcf2f1af1e0148f38fbecbf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:01:42 -0700 Subject: [PATCH 6/8] fix(cache_control): send acompletion fallbacks the caller's request, not the primary's hooked kwargs acompletion handed async_completion_with_fallbacks the kwargs the cache seed and prompt hook had already rewritten for the first model, so a Claude fallback behind a Foundry GPT model received prompt_cache_options and got a 400. Fallbacks now get a copy of the caller's model, messages, tools, and kwargs taken before the hooks, as completion already does with its args. Both cache seeds also read base_url before api_base, since completion sends the call to base_url when both are set. --- litellm/main.py | 12 ++- .../litellm_core_utils/test_fallback_utils.py | 88 +++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 41c4302499e..cf95ed227a1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -570,6 +570,10 @@ async def acompletion( loop: Final = asyncio.get_event_loop() custom_llm_provider = kwargs.get("custom_llm_provider", None) _ = _resolve_control_options(kwargs, model) + caller_request: Final = cast( # cast-ok: acompletion types messages and tools as bare Lists + "Mapping[str, object]", {"model": model, "messages": messages, "tools": tools} + ) + caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs ## PROMPT MANAGEMENT HOOKS ## ######################################################### @@ -588,7 +592,7 @@ async def acompletion( custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs - api_base=kwargs.get("api_base") or base_url, + api_base=base_url or kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -701,7 +705,9 @@ async def acompletion( fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: - response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs}) + response = await async_completion_with_fallbacks( + **{**completion_kwargs, **caller_request}, kwargs={"fallbacks": fallbacks, **caller_kwargs} + ) if response is None: raise Exception( "No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details." @@ -5631,7 +5637,7 @@ def completion( custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs - api_base=kwargs.get("api_base") or base_url, + api_base=base_url or kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/tests/unit/litellm_core_utils/test_fallback_utils.py b/tests/unit/litellm_core_utils/test_fallback_utils.py index 1e3ca0ba90d..77290d48a5c 100644 --- a/tests/unit/litellm_core_utils/test_fallback_utils.py +++ b/tests/unit/litellm_core_utils/test_fallback_utils.py @@ -444,3 +444,91 @@ def test_completion_fallbacks_sync( assert response.choices[0].message.content == "sync fallback answer" assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("first", "second") + + +CACHE_POINTS: Final = ({"location": "message", "role": "system"},) +CACHED_MESSAGES: Final = ( + {"role": "system", "content": "You are a careful assistant."}, + {"role": "user", "content": "hello"}, +) +OPENAI_ANSWER: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +ANTHROPIC_ANSWER: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} + + +def _cached_request(**kwargs: object) -> dict[str, object]: + return { + "messages": [dict(message) for message in CACHED_MESSAGES], + "cache_control_injection_points": [dict(point) for point in CACHE_POINTS], + **kwargs, + } + + +@pytest.mark.asyncio +async def test_acompletion_fallback_sends_the_fallback_model_the_request_a_direct_call_sends( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 401, + json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}}, + ) + ) + anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_ANSWER) + ) + + await python_acompletion(**_cached_request(model="anthropic/claude-haiku-4-5", api_key="anthropic-key")) + await python_acompletion( + **_cached_request( + model="openai/gpt-6.1-sol", + api_key="openai-key", + fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}], + ) + ) + + assert "prompt_cache_options" in json.loads(openai_route.calls[0].request.content) + direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls) + assert fallback_body == direct_body + + +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_cache_dialect_follows_base_url_over_an_api_base_the_call_does_not_use( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, is_async: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=OPENAI_ANSWER) + ) + + async def send(**kwargs: object) -> None: + request: Final = _cached_request( + model="openai/gpt-6.1-sol", api_key="openai-key", base_url="https://api.openai.com/v1", **kwargs + ) + if is_async: + await python_acompletion(**request) + else: + python_completion(**request) + + await send() + await send(api_base="https://unused-gateway.example.com/v1") + + base_url_only, with_unused_api_base = (json.loads(call.request.content) for call in route.calls) + assert with_unused_api_base == base_url_only From 1bb5043d59d51ecba8195de1d2398439098d05a1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:14:45 -0700 Subject: [PATCH 7/8] fix(cache_control): keep the caller's tools list intact for acompletion fallbacks The vector-store hook pops recognized file_search tools from the request's tools list in place, so the fallback request built from the pre-hook snapshot lost the retrieved context. Snapshot a copy of the list instead. --- litellm/main.py | 2 +- .../litellm_core_utils/test_fallback_utils.py | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index cf95ed227a1..45d642ec15a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -571,7 +571,7 @@ async def acompletion( custom_llm_provider = kwargs.get("custom_llm_provider", None) _ = _resolve_control_options(kwargs, model) caller_request: Final = cast( # cast-ok: acompletion types messages and tools as bare Lists - "Mapping[str, object]", {"model": model, "messages": messages, "tools": tools} + "Mapping[str, object]", {"model": model, "messages": messages, "tools": None if tools is None else [*tools]} ) caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs diff --git a/tests/unit/litellm_core_utils/test_fallback_utils.py b/tests/unit/litellm_core_utils/test_fallback_utils.py index 77290d48a5c..6ae69439a84 100644 --- a/tests/unit/litellm_core_utils/test_fallback_utils.py +++ b/tests/unit/litellm_core_utils/test_fallback_utils.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.fallback_utils import ( ) from litellm.main import acompletion as python_acompletion from litellm.main import completion as python_completion +from litellm.vector_stores.vector_store_registry import LiteLLM_ManagedVectorStore, VectorStoreRegistry @pytest.mark.asyncio @@ -532,3 +533,58 @@ async def test_cache_dialect_follows_base_url_over_an_api_base_the_call_does_not base_url_only, with_unused_api_base = (json.loads(call.request.content) for call in route.calls) assert with_unused_api_base == base_url_only + + +@pytest.mark.asyncio +async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrieves( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + vector_stores=[ + LiteLLM_ManagedVectorStore( + vector_store_id="vs_docs", custom_llm_provider="openai", litellm_params={"api_key": "openai-key"} + ) + ] + ), + ) + respx_mock.post("https://api.openai.com/v1/vector_stores/vs_docs/search").mock( + return_value=httpx.Response( + 200, + json={ + "object": "vector_store.search_results.page", + "search_query": "what is litellm?", + "data": [{"score": 1.0, "content": [{"type": "text", "text": "litellm is an LLM gateway"}]}], + }, + ) + ) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 401, + json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}}, + ) + ) + anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_ANSWER) + ) + + async def send(**kwargs: object) -> None: + await python_acompletion( + messages=[{"role": "user", "content": "what is litellm?"}], + tools=[{"type": "file_search", "vector_store_ids": ["vs_docs"]}], + **kwargs, + ) + + await send(model="anthropic/claude-haiku-4-5", api_key="anthropic-key") + await send( + model="openai/gpt-6.1-sol", + api_key="openai-key", + fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}], + ) + + direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls) + assert "litellm is an LLM gateway" in json.dumps(direct_body) + assert fallback_body == direct_body From b25eb6f48de4653a11762e4a007b7fac895f3e15 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:45:53 -0700 Subject: [PATCH 8/8] fix(cache_control): skip acompletion's prompt hooks when fallbacks will run them Each fallback attempt runs the prompt hooks for its own model, so running them first in the outer call only cost a vector-store search whose result the hand-off threw away. --- litellm/main.py | 14 ++++++-------- .../unit/litellm_core_utils/test_fallback_utils.py | 5 +++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 45d642ec15a..2f301153dd1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -554,7 +554,7 @@ async def acompletion( - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop. - If `stream` is True, the function returns an async generator that yields completion lines. """ - fallbacks = kwargs.get("fallbacks", None) + fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks mock_timeout = kwargs.get("mock_timeout", None) if mock_timeout is True: @@ -570,9 +570,6 @@ async def acompletion( loop: Final = asyncio.get_event_loop() custom_llm_provider = kwargs.get("custom_llm_provider", None) _ = _resolve_control_options(kwargs, model) - caller_request: Final = cast( # cast-ok: acompletion types messages and tools as bare Lists - "Mapping[str, object]", {"model": model, "messages": messages, "tools": None if tools is None else [*tools]} - ) caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs ## PROMPT MANAGEMENT HOOKS ## @@ -595,8 +592,10 @@ async def acompletion( api_base=base_url or kwargs.get("api_base"), ) - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( - litellm_logging_obj.should_run_prompt_management_hooks( + if ( + fallbacks is None + and isinstance(litellm_logging_obj, LiteLLMLoggingObj) + and litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=kwargs.get("prompt_id", None), non_default_params=kwargs, tools=tools, @@ -703,10 +702,9 @@ async def acompletion( litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None), ) - fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: response = await async_completion_with_fallbacks( - **{**completion_kwargs, **caller_request}, kwargs={"fallbacks": fallbacks, **caller_kwargs} + **completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs} ) if response is None: raise Exception( diff --git a/tests/unit/litellm_core_utils/test_fallback_utils.py b/tests/unit/litellm_core_utils/test_fallback_utils.py index 6ae69439a84..8246cb7cff5 100644 --- a/tests/unit/litellm_core_utils/test_fallback_utils.py +++ b/tests/unit/litellm_core_utils/test_fallback_utils.py @@ -551,7 +551,7 @@ async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrie ] ), ) - respx_mock.post("https://api.openai.com/v1/vector_stores/vs_docs/search").mock( + search_route: Final = respx_mock.post("https://api.openai.com/v1/vector_stores/vs_docs/search").mock( return_value=httpx.Response( 200, json={ @@ -561,7 +561,7 @@ async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrie }, ) ) - respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( return_value=httpx.Response( 401, json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}}, @@ -588,3 +588,4 @@ async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrie direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls) assert "litellm is an LLM gateway" in json.dumps(direct_body) assert fallback_body == direct_body + assert search_route.call_count <= openai_route.call_count + anthropic_route.call_count