fix(cache_control): resolve the chat-path dialect with the deployment api_base, as the call is routed

The hosted cache-format check now resolves the provider with the deployment's own api_base when the caller names none, the same provider the call dispatches to and the Responses path passes, so AZURE_AI_API_BASE no longer decides the chat format. The plain OpenAI check keeps main's name-only lookup.
This commit is contained in:
mateo-berri 2026-10-29 06:36:56 -07:00
parent 541d0f5cb1
commit a5bf36f5d9
2 changed files with 31 additions and 32 deletions

View file

@ -113,26 +113,17 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
def _served_provider_for_model(model: str) -> str | None:
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
try:
_, provider, _, _ = get_llm_provider(model=model)
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
except BadRequestError:
return None
return provider
def _declared_provider(model: str) -> str | None:
import litellm
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default():
return None
prefix, separator, _ = model.partition("/")
return prefix if separator and prefix in litellm.provider_list else None
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
hosted_flag: Final = (
None
@ -519,21 +510,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
) -> bool:
if model is None:
return False
declared_provider: Final = custom_llm_provider or _declared_provider(model)
hosted_flag: Final = _hosted_openai_dialect_flag(
model, declared_provider, AnthropicCacheControlHook._resolve_provider
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
)
if hosted_flag is not None:
return hosted_flag
if not supports_openai_prompt_cache_breakpoint(model):
return False
if (declared_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
return False
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@staticmethod
def count_request_cache_breakpoints(

View file

@ -4001,6 +4001,7 @@ class TestChatPathProviderStamp:
class TestChatPathResolvesTheDeploymentProvider:
MODEL = "azure_ai/gpt-6-astra"
AMBIENT_API_BASE = "https://other-deployment.openai.azure.com"
FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com"
UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com"
POINTS = [{"location": "message", "role": "system"}]
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
@ -4032,29 +4033,37 @@ class TestChatPathResolvesTheDeploymentProvider:
)
return out, params
@pytest.mark.parametrize(
"api_base",
["https://foundry.services.ai.azure.com", "https://resource.cognitiveservices.azure.com"],
)
def test_chat_path_marks_the_deployment_the_same_as_when_told_its_provider(
self, monkeypatch: pytest.MonkeyPatch, api_base: str
def test_chat_path_stamps_the_breakpoint_on_a_foundry_deployment_while_the_ambient_base_is_azure_openai(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
self._pin_ambient_api_base(monkeypatch, True)
pinned_resolved, pinned_told = self._chat(None, api_base), self._chat("azure_ai", api_base)
self._pin_ambient_api_base(monkeypatch, False)
unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base)
assert pinned_resolved == pinned_told == unset_resolved == unset_told
assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1
assert ("prompt_cache_options" in pinned_resolved[1]) == supports_openai_prompt_cache_breakpoint(
self.MODEL, "azure_ai"
)
messages, params = self._chat(None, self.FOUNDRY_API_BASE)
assert (messages, params) == self._chat("azure_ai", self.FOUNDRY_API_BASE)
assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1
assert ("prompt_cache_options" in params) == supports_openai_prompt_cache_breakpoint(self.MODEL, "azure_ai")
@pytest.mark.parametrize("pinned", [True, False])
@pytest.mark.parametrize(
"api_base",
[FOUNDRY_API_BASE, "https://resource.cognitiveservices.azure.com", "https://resource.openai.azure.com"],
)
def test_chat_path_marks_the_deployment_like_the_provider_its_api_base_dispatches_to(
self, monkeypatch: pytest.MonkeyPatch, api_base: str, pinned: bool
) -> None:
self._pin_ambient_api_base(monkeypatch, pinned)
dispatched_provider: Final = litellm.get_llm_provider(model=self.MODEL, api_base=api_base)[1]
assert self._chat(None, api_base) == self._chat(dispatched_provider, api_base)
@pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"])
@pytest.mark.parametrize("set_globally", ["env", "module"])
def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one(
self, monkeypatch: pytest.MonkeyPatch, model: str
self, monkeypatch: pytest.MonkeyPatch, model: str, set_globally: str
) -> None:
self._pin_ambient_api_base(monkeypatch, False)
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
if set_globally == "env":
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
else:
monkeypatch.setattr(litellm, "use_litellm_proxy", True)
assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat(
"litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model
)