mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(cache_control): send acompletion fallbacks the caller's request, not the primary's hooked kwargs
acompletion handed async_completion_with_fallbacks the kwargs the cache seed and prompt hook had already rewritten for the first model, so a Claude fallback behind a Foundry GPT model received prompt_cache_options and got a 400. Fallbacks now get a copy of the caller's model, messages, tools, and kwargs taken before the hooks, as completion already does with its args. Both cache seeds also read base_url before api_base, since completion sends the call to base_url when both are set.
This commit is contained in:
parent
a5bf36f5d9
commit
accfc9a6cc
2 changed files with 97 additions and 3 deletions
|
|
@ -570,6 +570,10 @@ async def acompletion(
|
|||
loop: Final = asyncio.get_event_loop()
|
||||
custom_llm_provider = kwargs.get("custom_llm_provider", None)
|
||||
_ = _resolve_control_options(kwargs, model)
|
||||
caller_request: Final = cast( # cast-ok: acompletion types messages and tools as bare Lists
|
||||
"Mapping[str, object]", {"model": model, "messages": messages, "tools": tools}
|
||||
)
|
||||
caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs
|
||||
|
||||
## PROMPT MANAGEMENT HOOKS ##
|
||||
#########################################################
|
||||
|
|
@ -588,7 +592,7 @@ async def acompletion(
|
|||
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base") or base_url,
|
||||
api_base=base_url or kwargs.get("api_base"),
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
@ -701,7 +705,9 @@ async def acompletion(
|
|||
|
||||
fallbacks = fallbacks or litellm.model_fallbacks
|
||||
if fallbacks is not None:
|
||||
response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
|
||||
response = await async_completion_with_fallbacks(
|
||||
**{**completion_kwargs, **caller_request}, kwargs={"fallbacks": fallbacks, **caller_kwargs}
|
||||
)
|
||||
if response is None:
|
||||
raise Exception(
|
||||
"No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
|
||||
|
|
@ -5631,7 +5637,7 @@ def completion(
|
|||
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base") or base_url,
|
||||
api_base=base_url or kwargs.get("api_base"),
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
|
|||
|
|
@ -444,3 +444,91 @@ def test_completion_fallbacks_sync(
|
|||
|
||||
assert response.choices[0].message.content == "sync fallback answer"
|
||||
assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("first", "second")
|
||||
|
||||
|
||||
CACHE_POINTS: Final = ({"location": "message", "role": "system"},)
|
||||
CACHED_MESSAGES: Final = (
|
||||
{"role": "system", "content": "You are a careful assistant."},
|
||||
{"role": "user", "content": "hello"},
|
||||
)
|
||||
OPENAI_ANSWER: Final = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-6.1-sol",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
ANTHROPIC_ANSWER: Final = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-4-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
def _cached_request(**kwargs: object) -> dict[str, object]:
|
||||
return {
|
||||
"messages": [dict(message) for message in CACHED_MESSAGES],
|
||||
"cache_control_injection_points": [dict(point) for point in CACHE_POINTS],
|
||||
**kwargs,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_fallback_sends_the_fallback_model_the_request_a_direct_call_sends(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
401,
|
||||
json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}},
|
||||
)
|
||||
)
|
||||
anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(200, json=ANTHROPIC_ANSWER)
|
||||
)
|
||||
|
||||
await python_acompletion(**_cached_request(model="anthropic/claude-haiku-4-5", api_key="anthropic-key"))
|
||||
await python_acompletion(
|
||||
**_cached_request(
|
||||
model="openai/gpt-6.1-sol",
|
||||
api_key="openai-key",
|
||||
fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}],
|
||||
)
|
||||
)
|
||||
|
||||
assert "prompt_cache_options" in json.loads(openai_route.calls[0].request.content)
|
||||
direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls)
|
||||
assert fallback_body == direct_body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_dialect_follows_base_url_over_an_api_base_the_call_does_not_use(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, is_async: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=OPENAI_ANSWER)
|
||||
)
|
||||
|
||||
async def send(**kwargs: object) -> None:
|
||||
request: Final = _cached_request(
|
||||
model="openai/gpt-6.1-sol", api_key="openai-key", base_url="https://api.openai.com/v1", **kwargs
|
||||
)
|
||||
if is_async:
|
||||
await python_acompletion(**request)
|
||||
else:
|
||||
python_completion(**request)
|
||||
|
||||
await send()
|
||||
await send(api_base="https://unused-gateway.example.com/v1")
|
||||
|
||||
base_url_only, with_unused_api_base = (json.loads(call.request.content) for call in route.calls)
|
||||
assert with_unused_api_base == base_url_only
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue