mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(cache_control): resolve the chat-path dialect with the deployment api_base, as the call is routed
The hosted cache-format check now resolves the provider with the deployment's own api_base when the caller names none, the same provider the call dispatches to and the Responses path passes, so AZURE_AI_API_BASE no longer decides the chat format. The plain OpenAI check keeps main's name-only lookup.
This commit is contained in:
parent
541d0f5cb1
commit
a5bf36f5d9
2 changed files with 31 additions and 32 deletions
|
|
@ -113,26 +113,17 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
|
|||
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
|
||||
|
||||
|
||||
def _served_provider_for_model(model: str) -> str | None:
|
||||
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
try:
|
||||
_, provider, _, _ = get_llm_provider(model=model)
|
||||
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
|
||||
except BadRequestError:
|
||||
return None
|
||||
return provider
|
||||
|
||||
|
||||
def _declared_provider(model: str) -> str | None:
|
||||
import litellm
|
||||
|
||||
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default():
|
||||
return None
|
||||
prefix, separator, _ = model.partition("/")
|
||||
return prefix if separator and prefix in litellm.provider_list else None
|
||||
|
||||
|
||||
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
hosted_flag: Final = (
|
||||
None
|
||||
|
|
@ -519,21 +510,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
) -> bool:
|
||||
if model is None:
|
||||
return False
|
||||
declared_provider: Final = custom_llm_provider or _declared_provider(model)
|
||||
hosted_flag: Final = _hosted_openai_dialect_flag(
|
||||
model, declared_provider, AnthropicCacheControlHook._resolve_provider
|
||||
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
|
||||
)
|
||||
if hosted_flag is not None:
|
||||
return hosted_flag
|
||||
if not supports_openai_prompt_cache_breakpoint(model):
|
||||
return False
|
||||
if (declared_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
|
||||
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
|
||||
return False
|
||||
return prompt_cache_options is not None or targets_openai_api(api_base)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: str) -> str | None:
|
||||
return _served_provider_for_model(model)
|
||||
def _resolve_provider(model: str, api_base: object = None) -> str | None:
|
||||
return _served_provider_for_model(model, api_base)
|
||||
|
||||
@staticmethod
|
||||
def count_request_cache_breakpoints(
|
||||
|
|
|
|||
|
|
@ -4001,6 +4001,7 @@ class TestChatPathProviderStamp:
|
|||
class TestChatPathResolvesTheDeploymentProvider:
|
||||
MODEL = "azure_ai/gpt-6-astra"
|
||||
AMBIENT_API_BASE = "https://other-deployment.openai.azure.com"
|
||||
FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com"
|
||||
UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com"
|
||||
POINTS = [{"location": "message", "role": "system"}]
|
||||
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
|
||||
|
|
@ -4032,29 +4033,37 @@ class TestChatPathResolvesTheDeploymentProvider:
|
|||
)
|
||||
return out, params
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
["https://foundry.services.ai.azure.com", "https://resource.cognitiveservices.azure.com"],
|
||||
)
|
||||
def test_chat_path_marks_the_deployment_the_same_as_when_told_its_provider(
|
||||
self, monkeypatch: pytest.MonkeyPatch, api_base: str
|
||||
def test_chat_path_stamps_the_breakpoint_on_a_foundry_deployment_while_the_ambient_base_is_azure_openai(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
self._pin_ambient_api_base(monkeypatch, True)
|
||||
pinned_resolved, pinned_told = self._chat(None, api_base), self._chat("azure_ai", api_base)
|
||||
self._pin_ambient_api_base(monkeypatch, False)
|
||||
unset_resolved, unset_told = self._chat(None, api_base), self._chat("azure_ai", api_base)
|
||||
assert pinned_resolved == pinned_told == unset_resolved == unset_told
|
||||
assert AnthropicCacheControlHook.count_request_cache_breakpoints(pinned_resolved[0]) == 1
|
||||
assert ("prompt_cache_options" in pinned_resolved[1]) == supports_openai_prompt_cache_breakpoint(
|
||||
self.MODEL, "azure_ai"
|
||||
)
|
||||
messages, params = self._chat(None, self.FOUNDRY_API_BASE)
|
||||
assert (messages, params) == self._chat("azure_ai", self.FOUNDRY_API_BASE)
|
||||
assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1
|
||||
assert ("prompt_cache_options" in params) == supports_openai_prompt_cache_breakpoint(self.MODEL, "azure_ai")
|
||||
|
||||
@pytest.mark.parametrize("pinned", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
[FOUNDRY_API_BASE, "https://resource.cognitiveservices.azure.com", "https://resource.openai.azure.com"],
|
||||
)
|
||||
def test_chat_path_marks_the_deployment_like_the_provider_its_api_base_dispatches_to(
|
||||
self, monkeypatch: pytest.MonkeyPatch, api_base: str, pinned: bool
|
||||
) -> None:
|
||||
self._pin_ambient_api_base(monkeypatch, pinned)
|
||||
dispatched_provider: Final = litellm.get_llm_provider(model=self.MODEL, api_base=api_base)[1]
|
||||
assert self._chat(None, api_base) == self._chat(dispatched_provider, api_base)
|
||||
|
||||
@pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"])
|
||||
@pytest.mark.parametrize("set_globally", ["env", "module"])
|
||||
def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one(
|
||||
self, monkeypatch: pytest.MonkeyPatch, model: str
|
||||
self, monkeypatch: pytest.MonkeyPatch, model: str, set_globally: str
|
||||
) -> None:
|
||||
self._pin_ambient_api_base(monkeypatch, False)
|
||||
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
|
||||
if set_globally == "env":
|
||||
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
|
||||
else:
|
||||
monkeypatch.setattr(litellm, "use_litellm_proxy", True)
|
||||
assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat(
|
||||
"litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue