fix(cache_control): keep litellm_proxy routing when the chat path reads the provider prefix

With USE_LITELLM_PROXY or litellm.use_litellm_proxy on, every call goes to the
upstream proxy, so the provider prefix no longer decides the cache format there
and resolution matches main. The plain OpenAI check reads the same provider
value as the hosted check, so one function no longer mixes two lookups
This commit is contained in:
mateo-berri 2026-10-29 02:47:57 -07:00
parent 5fdc7321a9
commit 541d0f5cb1
2 changed files with 24 additions and 5 deletions

View file

@ -127,6 +127,8 @@ def _served_provider_for_model(model: str) -> str | None:
def _declared_provider(model: str) -> str | None:
import litellm
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default():
return None
prefix, separator, _ = model.partition("/")
return prefix if separator and prefix in litellm.provider_list else None
@ -517,14 +519,15 @@ class AnthropicCacheControlHook(CustomPromptManagement):
) -> bool:
if model is None:
return False
declared_provider: Final = custom_llm_provider or _declared_provider(model)
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider or _declared_provider(model), AnthropicCacheControlHook._resolve_provider
model, declared_provider, AnthropicCacheControlHook._resolve_provider
)
if hosted_flag is not None:
return hosted_flag
if not supports_openai_prompt_cache_breakpoint(model):
return False
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
if (declared_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
return False
return prompt_cache_options is not None or targets_openai_api(api_base)

View file

@ -4001,6 +4001,7 @@ class TestChatPathProviderStamp:
class TestChatPathResolvesTheDeploymentProvider:
MODEL = "azure_ai/gpt-6-astra"
AMBIENT_API_BASE = "https://other-deployment.openai.azure.com"
UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com"
POINTS = [{"location": "message", "role": "system"}]
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
@ -4010,17 +4011,19 @@ class TestChatPathResolvesTheDeploymentProvider:
else:
monkeypatch.delenv("AZURE_AI_API_BASE", raising=False)
def _chat(self, custom_llm_provider: str | None, api_base: str) -> tuple[list[AllMessageValues], dict[str, object]]:
def _chat(
self, custom_llm_provider: str | None, api_base: str, model: str = MODEL
) -> tuple[list[AllMessageValues], dict[str, object]]:
seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
AnthropicCacheControlHook.maybe_seed_default_injection_points(
non_default_params=seeded,
messages=copy.deepcopy(self.MESSAGES),
model=self.MODEL,
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
model=self.MODEL,
model=model,
messages=copy.deepcopy(self.MESSAGES),
non_default_params=seeded,
prompt_id=None,
@ -4042,6 +4045,19 @@ class TestChatPathResolvesTheDeploymentProvider:
unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base)
assert pinned_resolved == pinned_told == unset_resolved == unset_told
assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1
assert ("prompt_cache_options" in pinned_resolved[1]) == supports_openai_prompt_cache_breakpoint(
self.MODEL, "azure_ai"
)
@pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"])
def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one(
self, monkeypatch: pytest.MonkeyPatch, model: str
) -> None:
self._pin_ambient_api_base(monkeypatch, False)
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat(
"litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model
)
class TestClientBreakpointsCountedOnce: