mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge a539b90db3 into e16d79d635
This commit is contained in:
commit
fcd27c7513
2 changed files with 182 additions and 1 deletions
|
|
@ -8569,7 +8569,9 @@ class Router:
|
|||
e,
|
||||
(litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError),
|
||||
)
|
||||
_request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id")
|
||||
_request_team_id: Final[str | None] = get_request_team_id(
|
||||
kwargs # pyright: ignore[reportUnknownArgumentType] # legacy kwargs lack type arguments
|
||||
)
|
||||
# Use wildcard-aware lookup so order-based fallback also works for model
|
||||
# groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`).
|
||||
order_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from itertools import product
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -1006,3 +1007,181 @@ async def test_unknown_default_fallback_raises_after_primary_failure(
|
|||
|
||||
assert route.call_count == 1
|
||||
assert "missing" in str(exc_info.value)
|
||||
|
||||
|
||||
def _team_ordered_fallback_router(public_model: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": f"{team_id}-{public_model}-{order}",
|
||||
"litellm_params": {
|
||||
"model": f"openai/{public_model}-order-{order}",
|
||||
"api_key": f"test-{team_id}",
|
||||
"api_base": "https://ordered-fallback.local/v1",
|
||||
"order": order,
|
||||
},
|
||||
"model_info": {
|
||||
"id": f"{team_id}-{public_model}-{order}",
|
||||
"team_id": team_id,
|
||||
"team_public_model_name": public_model,
|
||||
},
|
||||
}
|
||||
for team_id, order in product(("env-dev", "env-preview", "env-prod"), (1, 2))
|
||||
],
|
||||
num_retries=0,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"public_model, surface, metadata, team_id",
|
||||
(
|
||||
*(
|
||||
pytest.param(
|
||||
"general-model", "responses", {"litellm_metadata": {"user_api_key_team_id": team}}, team, id=team
|
||||
)
|
||||
for team in ("env-dev", "env-preview", "env-prod")
|
||||
),
|
||||
pytest.param(
|
||||
"compact-model",
|
||||
"responses",
|
||||
{"litellm_metadata": {"user_api_key_team_id": "env-prod"}},
|
||||
"env-prod",
|
||||
id="second-alias",
|
||||
),
|
||||
pytest.param(
|
||||
"general-model",
|
||||
"responses",
|
||||
{"metadata": {}, "litellm_metadata": {"user_api_key_team_id": "env-prod"}},
|
||||
"env-prod",
|
||||
id="empty-metadata",
|
||||
),
|
||||
pytest.param(
|
||||
"general-model",
|
||||
"responses",
|
||||
{"metadata": {"purpose": "test"}, "litellm_metadata": {"user_api_key_team_id": "env-prod"}},
|
||||
"env-prod",
|
||||
id="business-metadata",
|
||||
),
|
||||
pytest.param(
|
||||
"general-model",
|
||||
"responses",
|
||||
{
|
||||
"metadata": {"user_api_key_team_id": "env-prod"},
|
||||
"litellm_metadata": {"user_api_key_team_id": "env-prod"},
|
||||
},
|
||||
"env-prod",
|
||||
id="matching-metadata",
|
||||
),
|
||||
pytest.param(
|
||||
"general-model",
|
||||
"responses",
|
||||
{"metadata": {"user_api_key_team_id": "env-dev"}, "litellm_metadata": {"user_api_key_team_id": "env-prod"}},
|
||||
"env-dev",
|
||||
id="metadata-takes-precedence",
|
||||
),
|
||||
pytest.param(
|
||||
"general-model",
|
||||
"responses",
|
||||
{"metadata": {"user_api_key_team_id": ""}, "litellm_metadata": {"user_api_key_team_id": "env-prod"}},
|
||||
"env-prod",
|
||||
id="empty-team",
|
||||
),
|
||||
pytest.param(
|
||||
"general-model",
|
||||
"chat/completions",
|
||||
{"metadata": {"user_api_key_team_id": "env-prod"}},
|
||||
"env-prod",
|
||||
id="chat-control",
|
||||
),
|
||||
),
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_ordered_fallback_preserves_request_team(
|
||||
public_model: str,
|
||||
surface: Literal["responses", "chat/completions"],
|
||||
metadata: dict[str, dict[str, str]],
|
||||
team_id: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = _team_ordered_fallback_router(public_model)
|
||||
answer: Final = "served by the same team's backup"
|
||||
backup_model: Final = f"{public_model}-order-2"
|
||||
route: Final = respx_mock.post(f"https://ordered-fallback.local/v1/{surface}").mock(
|
||||
side_effect=(
|
||||
httpx.Response(400, json={"error": {"message": "invalid input", "type": "invalid_request_error"}}),
|
||||
httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_team_fallback",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": backup_model,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_team_fallback",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": answer, "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
if surface == "responses"
|
||||
else _fallback_chat_response(backup_model, answer),
|
||||
)
|
||||
)
|
||||
|
||||
if surface == "responses":
|
||||
responses_result: Final = await router.aresponses(model=public_model, input="fallback prompt", **metadata)
|
||||
assert responses_result.output[0].content[0].text == answer
|
||||
assert responses_result._hidden_params["model_id"] == f"{team_id}-{public_model}-2"
|
||||
else:
|
||||
chat_result: Final = await router.acompletion(
|
||||
model=public_model, messages=[{"role": "user", "content": "fallback prompt"}], **metadata
|
||||
)
|
||||
assert chat_result.choices[0].message.content == answer
|
||||
assert chat_result._hidden_params["model_id"] == f"{team_id}-{public_model}-2"
|
||||
|
||||
outbound: Final = tuple(
|
||||
(json.loads(call.request.content)["model"], call.request.headers["authorization"]) for call in route.calls
|
||||
)
|
||||
assert outbound == (
|
||||
(f"{public_model}-order-1", f"Bearer test-{team_id}"),
|
||||
(backup_model, f"Bearer test-{team_id}"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata", ({}, {"litellm_metadata": {"user_api_key_team_id": "unknown-team"}}))
|
||||
@pytest.mark.asyncio
|
||||
async def test_ordered_fallback_without_matching_team_never_sends(
|
||||
metadata: dict[str, dict[str, str]], respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
router: Final = _team_ordered_fallback_router("general-model")
|
||||
error: Final = litellm.BadRequestError(message="invalid input", model="general-model", llm_provider="openai")
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as exc_info:
|
||||
await router.async_function_with_fallbacks_common_utils(
|
||||
e=error,
|
||||
disable_fallbacks=False,
|
||||
fallbacks=None,
|
||||
context_window_fallbacks=None,
|
||||
content_policy_fallbacks=None,
|
||||
model_group="general-model",
|
||||
args=(),
|
||||
kwargs={
|
||||
"model": "general-model",
|
||||
"original_function": router._ageneric_api_call_with_fallbacks_helper,
|
||||
"original_generic_function": litellm.aresponses,
|
||||
"input": "fallback prompt",
|
||||
**metadata,
|
||||
},
|
||||
)
|
||||
|
||||
assert exc_info.value is error
|
||||
assert tuple(respx_mock.calls) == ()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue