This commit is contained in:
devin-ai-integration[bot] 2026-10-30 03:40:27 +00:00 • committed by GitHub
commit 90a5d01d78
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 232 additions and 12 deletions

View file

@ -115,12 +115,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
def _served_provider_for_model(model: str) -> str | None:
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
try:
_, provider, _, _ = get_llm_provider(model=model)
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
except BadRequestError:
return None
return provider
@ -527,7 +527,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if model is None:
return False
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
)
if hosted_flag is not None:
return hosted_flag
@ -538,8 +538,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@staticmethod
def count_request_cache_breakpoints(

View file

@ -681,7 +681,7 @@ async def acompletion(
- The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
- If `stream` is True, the function returns an async generator that yields completion lines.
"""
fallbacks = kwargs.get("fallbacks", None)
fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
mock_timeout = kwargs.get("mock_timeout", None)
if mock_timeout is True:
@ -697,6 +697,7 @@ async def acompletion(
loop: Final = asyncio.get_event_loop()
custom_llm_provider = kwargs.get("custom_llm_provider", None)
_ = _resolve_control_options(kwargs, model)
caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs
## PROMPT MANAGEMENT HOOKS ##
#########################################################
@ -715,11 +716,13 @@ async def acompletion(
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
if (
fallbacks is None
and isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=kwargs.get("prompt_id", None),
non_default_params=kwargs,
tools=tools,
@ -826,9 +829,10 @@ async def acompletion(
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
response = await async_completion_with_fallbacks(
**completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
)
if response is None:
raise Exception(
"No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
@ -5765,7 +5769,7 @@ def completion(
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (

View file

@ -3998,6 +3998,77 @@ class TestChatPathProviderStamp:
assert params["prompt_cache_options"] == {"mode": "explicit"}
class TestChatPathResolvesTheDeploymentProvider:
MODEL = "azure_ai/gpt-6-astra"
AMBIENT_API_BASE = "https://other-deployment.openai.azure.com"
FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com"
UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com"
POINTS = [{"location": "message", "role": "system"}]
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
def _pin_ambient_api_base(self, monkeypatch: pytest.MonkeyPatch, pinned: bool) -> None:
if pinned:
monkeypatch.setenv("AZURE_AI_API_BASE", self.AMBIENT_API_BASE)
else:
monkeypatch.delenv("AZURE_AI_API_BASE", raising=False)
def _chat(
self, custom_llm_provider: str | None, api_base: str, model: str = MODEL
) -> tuple[list[AllMessageValues], dict[str, object]]:
seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
AnthropicCacheControlHook.maybe_seed_default_injection_points(
non_default_params=seeded,
messages=copy.deepcopy(self.MESSAGES),
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
model=model,
messages=copy.deepcopy(self.MESSAGES),
non_default_params=seeded,
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
return out, params
def test_chat_path_stamps_the_breakpoint_on_a_foundry_deployment_while_the_ambient_base_is_azure_openai(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
self._pin_ambient_api_base(monkeypatch, True)
messages, params = self._chat(None, self.FOUNDRY_API_BASE)
assert (messages, params) == self._chat("azure_ai", self.FOUNDRY_API_BASE)
assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1
assert ("prompt_cache_options" in params) == supports_openai_prompt_cache_breakpoint(self.MODEL, "azure_ai")
@pytest.mark.parametrize("pinned", [True, False])
@pytest.mark.parametrize(
"api_base",
[FOUNDRY_API_BASE, "https://resource.cognitiveservices.azure.com", "https://resource.openai.azure.com"],
)
def test_chat_path_marks_the_deployment_like_the_provider_its_api_base_dispatches_to(
self, monkeypatch: pytest.MonkeyPatch, api_base: str, pinned: bool
) -> None:
self._pin_ambient_api_base(monkeypatch, pinned)
dispatched_provider: Final = litellm.get_llm_provider(model=self.MODEL, api_base=api_base)[1]
assert self._chat(None, api_base) == self._chat(dispatched_provider, api_base)
@pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"])
@pytest.mark.parametrize("set_globally", ["env", "module"])
def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one(
self, monkeypatch: pytest.MonkeyPatch, model: str, set_globally: str
) -> None:
self._pin_ambient_api_base(monkeypatch, False)
if set_globally == "env":
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
else:
monkeypatch.setattr(litellm, "use_litellm_proxy", True)
assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat(
"litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model
)
class TestClientBreakpointsCountedOnce:
def test_client_message_breakpoints_are_not_double_counted(self):
messages = [

View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.fallback_utils import (
)
from litellm.main import acompletion as python_acompletion
from litellm.main import completion as python_completion
from litellm.vector_stores.vector_store_registry import LiteLLM_ManagedVectorStore, VectorStoreRegistry
@pytest.mark.asyncio
@ -444,3 +445,147 @@ def test_completion_fallbacks_sync(
assert response.choices[0].message.content == "sync fallback answer"
assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("first", "second")
CACHE_POINTS: Final = ({"location": "message", "role": "system"},)
CACHED_MESSAGES: Final = (
{"role": "system", "content": "You are a careful assistant."},
{"role": "user", "content": "hello"},
)
OPENAI_ANSWER: Final = {
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "gpt-6.1-sol",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
ANTHROPIC_ANSWER: Final = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
}
def _cached_request(**kwargs: object) -> dict[str, object]:
return {
"messages": [dict(message) for message in CACHED_MESSAGES],
"cache_control_injection_points": [dict(point) for point in CACHE_POINTS],
**kwargs,
}
@pytest.mark.asyncio
async def test_acompletion_fallback_sends_the_fallback_model_the_request_a_direct_call_sends(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(
401,
json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}},
)
)
anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(200, json=ANTHROPIC_ANSWER)
)
await python_acompletion(**_cached_request(model="anthropic/claude-haiku-4-5", api_key="anthropic-key"))
await python_acompletion(
**_cached_request(
model="openai/gpt-6.1-sol",
api_key="openai-key",
fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}],
)
)
assert "prompt_cache_options" in json.loads(openai_route.calls[0].request.content)
direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls)
assert fallback_body == direct_body
@pytest.mark.parametrize("is_async", [True, False])
@pytest.mark.asyncio
async def test_cache_dialect_follows_base_url_over_an_api_base_the_call_does_not_use(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, is_async: bool
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(200, json=OPENAI_ANSWER)
)
async def send(**kwargs: object) -> None:
request: Final = _cached_request(
model="openai/gpt-6.1-sol", api_key="openai-key", base_url="https://api.openai.com/v1", **kwargs
)
if is_async:
await python_acompletion(**request)
else:
python_completion(**request)
await send()
await send(api_base="https://unused-gateway.example.com/v1")
base_url_only, with_unused_api_base = (json.loads(call.request.content) for call in route.calls)
assert with_unused_api_base == base_url_only
@pytest.mark.asyncio
async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrieves(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(
litellm,
"vector_store_registry",
VectorStoreRegistry(
vector_stores=[
LiteLLM_ManagedVectorStore(
vector_store_id="vs_docs", custom_llm_provider="openai", litellm_params={"api_key": "openai-key"}
)
]
),
)
search_route: Final = respx_mock.post("https://api.openai.com/v1/vector_stores/vs_docs/search").mock(
return_value=httpx.Response(
200,
json={
"object": "vector_store.search_results.page",
"search_query": "what is litellm?",
"data": [{"score": 1.0, "content": [{"type": "text", "text": "litellm is an LLM gateway"}]}],
},
)
)
openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(
401,
json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}},
)
)
anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(200, json=ANTHROPIC_ANSWER)
)
async def send(**kwargs: object) -> None:
await python_acompletion(
messages=[{"role": "user", "content": "what is litellm?"}],
tools=[{"type": "file_search", "vector_store_ids": ["vs_docs"]}],
**kwargs,
)
await send(model="anthropic/claude-haiku-4-5", api_key="anthropic-key")
await send(
model="openai/gpt-6.1-sol",
api_key="openai-key",
fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}],
)
direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls)
assert "litellm is an LLM gateway" in json.dumps(direct_body)
assert fallback_body == direct_body
assert search_route.call_count <= openai_route.call_count + anthropic_route.call_count