From 8c5fa0c0f9ec195636372bc8861f2f88402717f4 Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Sat, 15 Aug 2026 20:05:43 +0200 Subject: [PATCH 01/34] feat: add show budget window usage --- .../key_management_endpoints.py | 62 ++++++ .../test_key_management_endpoints.py | 206 ++++++++++++++++++ 2 files changed, 268 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 7e190e8b19d..22edbf33f94 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -106,6 +106,7 @@ from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, ) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper +from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start from litellm.proxy.spend_tracking.spend_tracking_utils import _is_master_key from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( get_ui_settings_cached, @@ -3516,6 +3517,61 @@ async def _build_model_max_budget_usage( return result +async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None: + """ + Attach current-window spend to each entry in key_info["budget_limits"], in place. + + Per-window spend is not persisted in the DB; it lives in the cross-pod spend + counters (spend:key:{hashed_token}:window:{budget_duration}) that + _virtual_key_multi_budget_check enforces against, so we read the same + counters via get_current_spend. Passing max_budget + window_start makes the + read re-check against the authoritative spend-log aggregate when the counter + is stale-low (e.g. after a Redis flush), same as the enforcement path. + """ + from litellm.proxy.proxy_server import get_current_spend + + budget_limits: Any = key_info.get("budget_limits") + if isinstance(budget_limits, str): + try: + budget_limits = json.loads(budget_limits) + except (TypeError, ValueError): + return + key_info["budget_limits"] = budget_limits + if not isinstance(budget_limits, list): + return + + for idx, window in enumerate(budget_limits): + w: dict | None = None + if isinstance(window, dict): + w = window + elif hasattr(window, "model_dump"): + try: + w = window.model_dump() + except Exception: # noqa: BLE001 + continue + budget_limits[idx] = w + if not w: + continue + duration: Any = w.get("budget_duration") + max_budget: Any = w.get("max_budget") + if not duration: + continue + try: + max_budget = float(max_budget) if max_budget is not None else None + except (TypeError, ValueError): + max_budget = None + counter_key: Final = f"spend:key:{api_key_hash}:window:{duration}" + spend: Final = await get_current_spend( + counter_key=counter_key, + fallback_spend=0.0, + max_budget=max_budget, + window_entity_type="Key", + window_entity_id=api_key_hash, + window_start=get_budget_window_start(w), + ) + w["current_spend"] = round(spend, 4) + + @router.post( "/v2/key/info", tags=["key management"], @@ -3597,6 +3653,8 @@ async def info_key_fn_v2( model_max_budget=model_max_budget, user_api_key_cache=user_api_key_cache, ) + if k_token_hash: + await _attach_budget_limits_usage(key_info=k_dict, api_key_hash=k_token_hash) filtered_key_info.append(k_dict) return {"key": data.keys, "info": filtered_key_info} @@ -3633,6 +3691,9 @@ async def info_key_fn( - model_max_budget: dict - Per-model budgets, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}} - model_max_budget_usage: dict | None - Current-window spend per model, present only when the key has per-model budgets + - budget_limits: list | None - Concurrent budget windows. Each entry includes + current_spend: spend accumulated in the window so far (read from the same cross-pod + spend counter the budget enforcement uses) - models: list - Model_name's the key is allowed to call - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} @@ -3709,6 +3770,7 @@ async def info_key_fn( model_max_budget=model_max_budget, user_api_key_cache=user_api_key_cache, ) + await _attach_budget_limits_usage(key_info=key_info, api_key_hash=key_token_hash) # Attach object_permission if object_permission_id is set key_info = await attach_object_permission_to_dict(key_info, prisma_client) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index bdf09a95e4b..59f7a2f1e8e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -13789,6 +13789,212 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch): mock_prisma_client.db.query_raw.assert_not_awaited() +@pytest.mark.asyncio +async def test_info_key_fn_budget_limits_includes_current_spend(monkeypatch): + """ + /key/info should attach current_spend to each budget_limits window, read from + the same spend counter (spend:key:{token}:window:{duration}) that budget + enforcement uses. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import LiteLLM_VerificationToken + from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn + + test_key_token = "hashed_token_window_test" + budget_limits = [ + { + "reset_at": "2026-08-15T18:00:00+00:00", + "max_budget": 2.0, + "budget_duration": "1h", + } + ] + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_user_api_key_cache = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + mock_get_current_spend = AsyncMock(return_value=0.73) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + mock_key_info = MagicMock(spec=LiteLLM_VerificationToken) + mock_key_info.token = test_key_token + mock_key_info.object_permission_id = None + mock_key_info.user_id = "user-w" + mock_key_info.team_id = None + mock_key_info.litellm_budget_table = None + mock_key_info.model_dump.return_value = { + "token": test_key_token, + "budget_limits": [dict(w) for w in budget_limits], + "user_id": "user-w", + "team_id": None, + "object_permission_id": None, + "litellm_budget_table": None, + } + mock_key_info.dict.return_value = mock_key_info.model_dump.return_value + + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key_info + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test-window-key", + ) + + result = await info_key_fn( + key="sk-test-window-key", + user_api_key_dict=user_api_key_dict, + ) + + windows = result["info"]["budget_limits"] + assert len(windows) == 1 + assert windows[0]["current_spend"] == 0.73 + assert windows[0]["max_budget"] == 2.0 + assert windows[0]["budget_duration"] == "1h" + + mock_get_current_spend.assert_awaited_once() + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == f"spend:key:{test_key_token}:window:1h" + assert call_kwargs["max_budget"] == 2.0 + assert call_kwargs["window_entity_type"] == "Key" + assert call_kwargs["window_entity_id"] == test_key_token + assert call_kwargs["window_start"] is not None + + +@pytest.mark.asyncio +async def test_info_key_fn_no_budget_limits_skips_spend_lookup(monkeypatch): + """Keys without budget_limits should not trigger window spend lookups.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import LiteLLM_VerificationToken + from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn + + test_key_token = "hashed_token_no_windows" + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_user_api_key_cache = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + mock_get_current_spend = AsyncMock(return_value=0.0) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + mock_key_info = MagicMock(spec=LiteLLM_VerificationToken) + mock_key_info.token = test_key_token + mock_key_info.object_permission_id = None + mock_key_info.user_id = "user-nw" + mock_key_info.team_id = None + mock_key_info.litellm_budget_table = None + mock_key_info.model_dump.return_value = { + "token": test_key_token, + "budget_limits": None, + "user_id": "user-nw", + "team_id": None, + "object_permission_id": None, + "litellm_budget_table": None, + } + mock_key_info.dict.return_value = mock_key_info.model_dump.return_value + + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key_info + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test-no-window-key", + ) + + result = await info_key_fn( + key="sk-test-no-window-key", + user_api_key_dict=user_api_key_dict, + ) + + assert result["info"]["budget_limits"] is None + mock_get_current_spend.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): + """/v2/key/info should attach current_spend to each budget_limits window.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import KeyRequest, LiteLLM_VerificationToken + from litellm.proxy.management_endpoints.key_management_endpoints import ( + info_key_fn_v2, + ) + + test_key_token = "hashed_token_v2_window_test" + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_user_api_key_cache = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + mock_get_current_spend = AsyncMock(return_value=1.25) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + mock_key = MagicMock(spec=LiteLLM_VerificationToken) + mock_key.token = test_key_token + mock_key.user_id = "user-v2-w" + mock_key.team_id = None + mock_key.model_dump.return_value = { + "token": test_key_token, + "budget_limits": [ + { + "reset_at": "2026-08-15T18:00:00+00:00", + "max_budget": 2.0, + "budget_duration": "1h", + }, + { + "reset_at": "2026-08-16T00:00:00+00:00", + "max_budget": 20.0, + "budget_duration": "1d", + }, + ], + "user_id": "user-v2-w", + "team_id": None, + "litellm_budget_table": None, + } + mock_key.dict.return_value = mock_key.model_dump.return_value + + mock_prisma_client.get_data = AsyncMock(return_value=[mock_key]) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin-v2-w", + ) + + result = await info_key_fn_v2( + data=KeyRequest(keys=[test_key_token]), + user_api_key_dict=user_api_key_dict, + ) + + assert len(result["info"]) == 1 + windows = result["info"][0]["budget_limits"] + assert len(windows) == 2 + assert windows[0]["current_spend"] == 1.25 + assert windows[1]["current_spend"] == 1.25 + assert mock_get_current_spend.await_count == 2 + counter_keys = { + call.kwargs["counter_key"] for call in mock_get_current_spend.await_args_list + } + assert counter_keys == { + f"spend:key:{test_key_token}:window:1h", + f"spend:key:{test_key_token}:window:1d", + } + + @pytest.mark.asyncio async def test_info_key_fn_provider_prefix_spend_fallback(monkeypatch): """Cached spend for 'gpt-4o' matches budget key 'openai/gpt-4o' via suffix match.""" From 34bc22fffd3459954afc42ede850f622d1344520 Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Sat, 15 Aug 2026 21:10:12 +0200 Subject: [PATCH 02/34] fix: address pr comments --- .../key_management_endpoints.py | 25 +++-- .../test_key_management_endpoints.py | 103 ++++++++++++++++++ 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 22edbf33f94..5e4642d66a9 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3517,6 +3517,20 @@ async def _build_model_max_budget_usage( return result +def _budget_window_to_dict(window: object) -> dict | None: + """Coerce a budget_limits entry to a dict; None when the entry is unusable.""" + if isinstance(window, dict): + return window + model_dump = getattr(window, "model_dump", None) + if callable(model_dump): + try: + dumped: Any = model_dump() + except Exception: # noqa: BLE001 + return None + return dumped if isinstance(dumped, dict) else None + return None + + async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None: """ Attach current-window spend to each entry in key_info["budget_limits"], in place. @@ -3541,17 +3555,10 @@ async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None return for idx, window in enumerate(budget_limits): - w: dict | None = None - if isinstance(window, dict): - w = window - elif hasattr(window, "model_dump"): - try: - w = window.model_dump() - except Exception: # noqa: BLE001 - continue - budget_limits[idx] = w + w: Final = _budget_window_to_dict(window) if not w: continue + budget_limits[idx] = w duration: Any = w.get("budget_duration") max_budget: Any = w.get("max_budget") if not duration: diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 59f7a2f1e8e..5df7c1cb28d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -13995,6 +13995,109 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): } +@pytest.mark.asyncio +async def test_attach_budget_limits_usage_json_string_input(monkeypatch): + """budget_limits stored as a JSON string should be parsed and annotated.""" + import json as json_module + from unittest.mock import AsyncMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _attach_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=0.5) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + key_info = { + "budget_limits": json_module.dumps( + [{"budget_duration": "1h", "max_budget": 2.0, "reset_at": None}] + ) + } + await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + + assert isinstance(key_info["budget_limits"], list) + assert key_info["budget_limits"][0]["current_spend"] == 0.5 + + +@pytest.mark.asyncio +async def test_attach_budget_limits_usage_skips_unusable_inputs(monkeypatch): + """Invalid JSON strings, non-list values, and malformed windows are skipped.""" + from unittest.mock import AsyncMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _attach_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=0.0) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + # invalid JSON string + key_info = {"budget_limits": "{not json"} + await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + assert key_info["budget_limits"] == "{not json" + + # non-list value + key_info = {"budget_limits": {"budget_duration": "1h"}} + await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + + # windows that are falsy, missing budget_duration, or not dict-like + key_info = { + "budget_limits": [ + {}, + {"max_budget": 2.0}, + {"budget_duration": "1h", "max_budget": "not-a-number"}, + 42, + ] + } + await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + + # only the well-formed window (with unparseable max_budget coerced to None) + # triggers a spend lookup + mock_get_current_spend.assert_awaited_once() + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == "spend:key:hash-1:window:1h" + assert call_kwargs["max_budget"] is None + assert key_info["budget_limits"][2]["current_spend"] == 0.0 + assert key_info["budget_limits"][3] == 42 + + +@pytest.mark.asyncio +async def test_attach_budget_limits_usage_pydantic_windows(monkeypatch): + """Window objects with model_dump() are converted to dicts in place.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _attach_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=1.0) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + good_window = MagicMock() + good_window.model_dump.return_value = { + "budget_duration": "7d", + "max_budget": 10.0, + "reset_at": None, + } + bad_window = MagicMock() + bad_window.model_dump.side_effect = ValueError("boom") + + key_info = {"budget_limits": [good_window, bad_window]} + await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-2") + + # good window converted to dict and annotated; failing window left as-is + assert isinstance(key_info["budget_limits"][0], dict) + assert key_info["budget_limits"][0]["current_spend"] == 1.0 + assert key_info["budget_limits"][1] is bad_window + mock_get_current_spend.assert_awaited_once() + + @pytest.mark.asyncio async def test_info_key_fn_provider_prefix_spend_fallback(monkeypatch): """Cached spend for 'gpt-4o' matches budget key 'openai/gpt-4o' via suffix match.""" From db67bafd59a0e131d70cfc4a5b1a3e494ca63373 Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Sat, 15 Aug 2026 21:26:55 +0200 Subject: [PATCH 03/34] fix: drop Final from loop locals to stay within the basedpyright budget --- .../proxy/management_endpoints/key_management_endpoints.py | 6 +++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 5e4642d66a9..51b9ebf91d8 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3555,7 +3555,7 @@ async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None return for idx, window in enumerate(budget_limits): - w: Final = _budget_window_to_dict(window) + w = _budget_window_to_dict(window) if not w: continue budget_limits[idx] = w @@ -3567,8 +3567,8 @@ async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None max_budget = float(max_budget) if max_budget is not None else None except (TypeError, ValueError): max_budget = None - counter_key: Final = f"spend:key:{api_key_hash}:window:{duration}" - spend: Final = await get_current_spend( + counter_key = f"spend:key:{api_key_hash}:window:{duration}" + spend = await get_current_spend( counter_key=counter_key, fallback_spend=0.0, max_budget=max_budget, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3b4b96bdd4e..f4703202b18 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7020,6 +7020,9 @@ export interface paths { * - model_max_budget: dict - Per-model budgets, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}} * - model_max_budget_usage: dict | None - Current-window spend per model, present only when * the key has per-model budgets + * - budget_limits: list | None - Concurrent budget windows. Each entry includes + * current_spend: spend accumulated in the window so far (read from the same cross-pod + * spend counter the budget enforcement uses) * - models: list - Model_name's the key is allowed to call * - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits * - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} From 31130036c0013b8c0ad34e6cffbd58b24ee8b78a Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Mon, 17 Aug 2026 16:21:31 +0200 Subject: [PATCH 04/34] refactor: build budget window usage without in-place mutation Replace _attach_budget_limits_usage, which rewrote the caller's key_info dict, with _budget_limits_with_usage returning a new list. Callers assign the result once. Keeps the response shape and spend-counter read path identical while following the repo's no-mutation rule. --- .../key_management_endpoints.py | 119 +++++++++++------- .../test_key_management_endpoints.py | 83 ++++++------ 2 files changed, 121 insertions(+), 81 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 51b9ebf91d8..f6f739f3eae 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3517,23 +3517,44 @@ async def _build_model_max_budget_usage( return result -def _budget_window_to_dict(window: object) -> dict | None: +def _budget_window_to_dict(window: object) -> Mapping[str, object] | None: """Coerce a budget_limits entry to a dict; None when the entry is unusable.""" if isinstance(window, dict): return window - model_dump = getattr(window, "model_dump", None) - if callable(model_dump): + model_dump: Final = getattr(window, "model_dump", None) + if not callable(model_dump): + return None + try: + dumped: Final = model_dump() + except Exception: # noqa: BLE001 # model_dump implementations can raise arbitrary errors + return None + return dumped if isinstance(dumped, dict) else None + + +def _coerce_budget_limits(budget_limits: object) -> Sequence[object] | None: + """Coerce budget_limits to a sequence of windows, parsing JSON strings; None when unusable.""" + if isinstance(budget_limits, str): try: - dumped: Any = model_dump() - except Exception: # noqa: BLE001 + parsed: Final = json.loads(budget_limits) + except (TypeError, ValueError): + return None + return parsed if isinstance(parsed, list) else None + return budget_limits if isinstance(budget_limits, list) else None + + +def _parse_window_max_budget(value: object) -> float | None: + """Coerce a window's max_budget to float; None when absent or unparseable.""" + if isinstance(value, (int, float, str)): + try: + return float(value) + except (TypeError, ValueError): return None - return dumped if isinstance(dumped, dict) else None return None -async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None: +async def _budget_window_with_usage(window: Mapping[str, object], api_key_hash: str) -> Mapping[str, object]: """ - Attach current-window spend to each entry in key_info["budget_limits"], in place. + Return a copy of a budget window with current-window spend attached. Per-window spend is not persisted in the DB; it lives in the cross-pod spend counters (spend:key:{hashed_token}:window:{budget_duration}) that @@ -3544,39 +3565,43 @@ async def _attach_budget_limits_usage(key_info: dict, api_key_hash: str) -> None """ from litellm.proxy.proxy_server import get_current_spend - budget_limits: Any = key_info.get("budget_limits") - if isinstance(budget_limits, str): - try: - budget_limits = json.loads(budget_limits) - except (TypeError, ValueError): - return - key_info["budget_limits"] = budget_limits - if not isinstance(budget_limits, list): - return + duration: Final = window.get("budget_duration") + if not duration: + return dict(window) # mutable-ok: per-window response copy, built once per window + spend: Final = await get_current_spend( + counter_key=f"spend:key:{api_key_hash}:window:{duration}", + fallback_spend=0.0, + max_budget=_parse_window_max_budget(window.get("max_budget")), + window_entity_type="Key", + window_entity_id=api_key_hash, + window_start=get_budget_window_start(window), + ) + return {**window, "current_spend": round(spend, 4)} # mutable-ok: per-window response copy, built once per window - for idx, window in enumerate(budget_limits): - w = _budget_window_to_dict(window) - if not w: - continue - budget_limits[idx] = w - duration: Any = w.get("budget_duration") - max_budget: Any = w.get("max_budget") - if not duration: - continue - try: - max_budget = float(max_budget) if max_budget is not None else None - except (TypeError, ValueError): - max_budget = None - counter_key = f"spend:key:{api_key_hash}:window:{duration}" - spend = await get_current_spend( - counter_key=counter_key, - fallback_spend=0.0, - max_budget=max_budget, - window_entity_type="Key", - window_entity_id=api_key_hash, - window_start=get_budget_window_start(w), - ) - w["current_spend"] = round(spend, 4) + +async def _budget_limits_entry_with_usage(window: object, api_key_hash: str) -> object: + """Return the window as an enriched dict when dict-coercible; the original entry otherwise.""" + coerced: Final = _budget_window_to_dict(window) + if not coerced: + return window + return await _budget_window_with_usage(window=coerced, api_key_hash=api_key_hash) + + +async def _budget_limits_with_usage(budget_limits: object, api_key_hash: str) -> Sequence[object] | None: + """ + Return budget_limits as window dicts with current-window spend attached. + + None when budget_limits is not a usable (possibly JSON-encoded) list; the + caller keeps the original value then. Entries that are not dict-coercible + are preserved as-is. + """ + windows: Final = _coerce_budget_limits(budget_limits) + if windows is None: + return None + return [ # mutable-ok: entries are awaited, so they cannot be built inside a frozen wrapper + await _budget_limits_entry_with_usage(window=window, api_key_hash=api_key_hash) + for window in windows + ] @router.post( @@ -3661,7 +3686,12 @@ async def info_key_fn_v2( user_api_key_cache=user_api_key_cache, ) if k_token_hash: - await _attach_budget_limits_usage(key_info=k_dict, api_key_hash=k_token_hash) + budget_limits_usage = await _budget_limits_with_usage( + budget_limits=k_dict.get("budget_limits"), + api_key_hash=k_token_hash, + ) + if budget_limits_usage is not None: + k_dict["budget_limits"] = budget_limits_usage filtered_key_info.append(k_dict) return {"key": data.keys, "info": filtered_key_info} @@ -3777,7 +3807,12 @@ async def info_key_fn( model_max_budget=model_max_budget, user_api_key_cache=user_api_key_cache, ) - await _attach_budget_limits_usage(key_info=key_info, api_key_hash=key_token_hash) + budget_limits_usage: Final = await _budget_limits_with_usage( + budget_limits=key_info.get("budget_limits"), + api_key_hash=key_token_hash, + ) + if budget_limits_usage is not None: + key_info["budget_limits"] = budget_limits_usage # Attach object_permission if object_permission_id is set key_info = await attach_object_permission_to_dict(key_info, prisma_client) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 5df7c1cb28d..1f3b9042ed7 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -13996,13 +13996,13 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): @pytest.mark.asyncio -async def test_attach_budget_limits_usage_json_string_input(monkeypatch): +async def test_budget_limits_with_usage_json_string_input(monkeypatch): """budget_limits stored as a JSON string should be parsed and annotated.""" import json as json_module from unittest.mock import AsyncMock from litellm.proxy.management_endpoints.key_management_endpoints import ( - _attach_budget_limits_usage, + _budget_limits_with_usage, ) mock_get_current_spend = AsyncMock(return_value=0.5) @@ -14010,24 +14010,29 @@ async def test_attach_budget_limits_usage_json_string_input(monkeypatch): "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend ) - key_info = { - "budget_limits": json_module.dumps( - [{"budget_duration": "1h", "max_budget": 2.0, "reset_at": None}] - ) - } - await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + raw = json_module.dumps( + [{"budget_duration": "1h", "max_budget": 2.0, "reset_at": None}] + ) + result = await _budget_limits_with_usage(budget_limits=raw, api_key_hash="hash-1") - assert isinstance(key_info["budget_limits"], list) - assert key_info["budget_limits"][0]["current_spend"] == 0.5 + assert result == [ + { + "budget_duration": "1h", + "max_budget": 2.0, + "reset_at": None, + "current_spend": 0.5, + } + ] + mock_get_current_spend.assert_awaited_once() @pytest.mark.asyncio -async def test_attach_budget_limits_usage_skips_unusable_inputs(monkeypatch): +async def test_budget_limits_with_usage_skips_unusable_inputs(monkeypatch): """Invalid JSON strings, non-list values, and malformed windows are skipped.""" from unittest.mock import AsyncMock from litellm.proxy.management_endpoints.key_management_endpoints import ( - _attach_budget_limits_usage, + _budget_limits_with_usage, ) mock_get_current_spend = AsyncMock(return_value=0.0) @@ -14035,25 +14040,18 @@ async def test_attach_budget_limits_usage_skips_unusable_inputs(monkeypatch): "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend ) - # invalid JSON string - key_info = {"budget_limits": "{not json"} - await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") - assert key_info["budget_limits"] == "{not json" - - # non-list value - key_info = {"budget_limits": {"budget_duration": "1h"}} - await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + # invalid JSON string and non-list values return None: callers keep the original + assert await _budget_limits_with_usage(budget_limits="{not json", api_key_hash="hash-1") is None + assert await _budget_limits_with_usage(budget_limits={"budget_duration": "1h"}, api_key_hash="hash-1") is None # windows that are falsy, missing budget_duration, or not dict-like - key_info = { - "budget_limits": [ - {}, - {"max_budget": 2.0}, - {"budget_duration": "1h", "max_budget": "not-a-number"}, - 42, - ] - } - await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-1") + windows = [ + {}, + {"max_budget": 2.0}, + {"budget_duration": "1h", "max_budget": "not-a-number"}, + 42, + ] + result = await _budget_limits_with_usage(budget_limits=windows, api_key_hash="hash-1") # only the well-formed window (with unparseable max_budget coerced to None) # triggers a spend lookup @@ -14061,17 +14059,22 @@ async def test_attach_budget_limits_usage_skips_unusable_inputs(monkeypatch): call_kwargs = mock_get_current_spend.await_args.kwargs assert call_kwargs["counter_key"] == "spend:key:hash-1:window:1h" assert call_kwargs["max_budget"] is None - assert key_info["budget_limits"][2]["current_spend"] == 0.0 - assert key_info["budget_limits"][3] == 42 + assert result is not None + assert result[0] == {} + assert result[1] == {"max_budget": 2.0} + assert result[2] == {"budget_duration": "1h", "max_budget": "not-a-number", "current_spend": 0.0} + assert result[3] == 42 + # input is not mutated + assert windows[2] == {"budget_duration": "1h", "max_budget": "not-a-number"} @pytest.mark.asyncio -async def test_attach_budget_limits_usage_pydantic_windows(monkeypatch): - """Window objects with model_dump() are converted to dicts in place.""" +async def test_budget_limits_with_usage_pydantic_windows(monkeypatch): + """Window objects with model_dump() are converted to dicts; failing windows pass through.""" from unittest.mock import AsyncMock, MagicMock from litellm.proxy.management_endpoints.key_management_endpoints import ( - _attach_budget_limits_usage, + _budget_limits_with_usage, ) mock_get_current_spend = AsyncMock(return_value=1.0) @@ -14088,13 +14091,15 @@ async def test_attach_budget_limits_usage_pydantic_windows(monkeypatch): bad_window = MagicMock() bad_window.model_dump.side_effect = ValueError("boom") - key_info = {"budget_limits": [good_window, bad_window]} - await _attach_budget_limits_usage(key_info=key_info, api_key_hash="hash-2") + result = await _budget_limits_with_usage( + budget_limits=[good_window, bad_window], api_key_hash="hash-2" + ) # good window converted to dict and annotated; failing window left as-is - assert isinstance(key_info["budget_limits"][0], dict) - assert key_info["budget_limits"][0]["current_spend"] == 1.0 - assert key_info["budget_limits"][1] is bad_window + assert result is not None + assert isinstance(result[0], dict) + assert result[0]["current_spend"] == 1.0 + assert result[1] is bad_window mock_get_current_spend.assert_awaited_once() From 9b30e73d07ebccdf4e6ff24981110e238c080d4d Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Mon, 17 Aug 2026 20:48:12 +0200 Subject: [PATCH 05/34] fix: cap /v2/key/info batch size to bound spend-log query fan-out --- .../key_management_endpoints.py | 16 +++++++++ .../test_key_management_endpoints.py | 33 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index f6f739f3eae..46d7f617352 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3604,6 +3604,11 @@ async def _budget_limits_with_usage(budget_limits: object, api_key_hash: str) -> ] +# Caps per-request fan-out: each key with budget windows costs one spend-counter +# read (worst case a SpendLogs aggregation) per window. +MAX_KEY_INFO_KEYS_PER_REQUEST: Final = 100 + + @router.post( "/v2/key/info", tags=["key management"], @@ -3643,6 +3648,17 @@ async def info_key_fn_v2( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={"message": "Malformed request. No keys passed in."}, ) + requested_key_count: Final = len(data.keys or []) + len(data.key_aliases or []) + if requested_key_count > MAX_KEY_INFO_KEYS_PER_REQUEST: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={ + "message": ( + f"Too many keys requested: {requested_key_count}. " + f"At most {MAX_KEY_INFO_KEYS_PER_REQUEST} keys and key_aliases combined per request." + ) + }, + ) # Resolve key_aliases to tokens so we never pass token=None (unbounded query) tokens_to_query: Final = list(data.keys) if data.keys else [] diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 1f3b9042ed7..9194ffb603b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -13995,6 +13995,39 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): } +@pytest.mark.asyncio +async def test_info_key_fn_v2_rejects_oversized_batch(monkeypatch): + """/v2/key/info must reject over-cap batches before doing any DB work.""" + from unittest.mock import AsyncMock + + from litellm.proxy._types import KeyRequest, ProxyException + from litellm.proxy.management_endpoints.key_management_endpoints import ( + MAX_KEY_INFO_KEYS_PER_REQUEST, + info_key_fn_v2, + ) + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", AsyncMock()) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin-batch-cap", + ) + + with pytest.raises(ProxyException) as exc_info: + await info_key_fn_v2( + data=KeyRequest( + keys=[f"hash-{i}" for i in range(MAX_KEY_INFO_KEYS_PER_REQUEST)], + key_aliases=["alias-over-cap"], + ), + user_api_key_dict=user_api_key_dict, + ) + + assert exc_info.value.code == "422" + mock_prisma_client.get_data.assert_not_awaited() + + @pytest.mark.asyncio async def test_budget_limits_with_usage_json_string_input(monkeypatch): """budget_limits stored as a JSON string should be parsed and annotated.""" From ffd23ab1319d0caf9c870be5849006cbc931f3d1 Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Mon, 17 Aug 2026 21:00:24 +0200 Subject: [PATCH 06/34] style: apply ruff format to budget limits comprehension --- litellm/proxy/management_endpoints/key_management_endpoints.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 46d7f617352..170b431c1e2 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3599,8 +3599,7 @@ async def _budget_limits_with_usage(budget_limits: object, api_key_hash: str) -> if windows is None: return None return [ # mutable-ok: entries are awaited, so they cannot be built inside a frozen wrapper - await _budget_limits_entry_with_usage(window=window, api_key_hash=api_key_hash) - for window in windows + await _budget_limits_entry_with_usage(window=window, api_key_hash=api_key_hash) for window in windows ] From 4e3e7d8cf6c0afe6171f784a1bba1b0080831edf Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Mon, 17 Aug 2026 21:12:36 +0200 Subject: [PATCH 07/34] test: cover soft budget row creation and windows without max_budget --- .../test_key_management_endpoints.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 9194ffb603b..c05380dd054 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -540,6 +540,57 @@ async def test_key_generation_with_object_permission(monkeypatch): assert key_insert_calls[0]["data"].get("object_permission_id") == "objperm123" +@pytest.mark.asyncio +async def test_generate_key_with_soft_budget_creates_budget_row(monkeypatch): + """soft_budget on /key/generate must create a budget table row and link its budget_id to the key.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.jsonify_object = lambda data: data + mock_prisma_client.db = MagicMock() + mock_budget_create = AsyncMock(return_value=MagicMock(budget_id="budget-soft-123")) + mock_prisma_client.db.litellm_budgettable = MagicMock() + mock_prisma_client.db.litellm_budgettable.create = mock_budget_create + + async def _insert_data_side_effect(*args, **kwargs): + if kwargs.get("table_name") == "user": + return MagicMock(models=[], spend=0) + return MagicMock( + token="hashed_token_soft", + litellm_budget_table=None, + object_permission=None, + ) + + mock_prisma_client.insert_data = AsyncMock(side_effect=_insert_data_side_effect) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles + from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + generate_key_fn, + ) + + await generate_key_fn( + data=GenerateKeyRequest(soft_budget=5.0), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="admin-1", + ), + ) + + mock_budget_create.assert_awaited_once() + created_budget = mock_budget_create.call_args.kwargs["data"] + assert created_budget["soft_budget"] == 5.0 + assert created_budget["created_by"] == "admin-1" + + key_insert_calls = [ + call.kwargs + for call in mock_prisma_client.insert_data.call_args_list + if call.kwargs.get("table_name") == "key" + ] + assert len(key_insert_calls) == 1 + assert key_insert_calls[0]["data"].get("budget_id") == "budget-soft-123" + + @pytest.mark.asyncio async def test_generate_key_debug_log_never_contains_raw_token(monkeypatch, caplog): """Regression for LIT-4356: /key/generate must never emit the raw virtual key @@ -14101,6 +14152,30 @@ async def test_budget_limits_with_usage_skips_unusable_inputs(monkeypatch): assert windows[2] == {"budget_duration": "1h", "max_budget": "not-a-number"} +@pytest.mark.asyncio +async def test_budget_limits_with_usage_window_without_max_budget(monkeypatch): + """A window with only budget_duration still gets current_spend, read without a budget ceiling.""" + from unittest.mock import AsyncMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _budget_limits_with_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=0.75) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + result = await _budget_limits_with_usage( + budget_limits=[{"budget_duration": "2d"}], api_key_hash="hash-no-max" + ) + + assert result == [{"budget_duration": "2d", "current_spend": 0.75}] + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == "spend:key:hash-no-max:window:2d" + assert call_kwargs["max_budget"] is None + + @pytest.mark.asyncio async def test_budget_limits_with_usage_pydantic_windows(monkeypatch): """Window objects with model_dump() are converted to dicts; failing windows pass through.""" From f4406b4060cddb60731f804513455bea321e7407 Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Mon, 24 Aug 2026 08:52:18 +0200 Subject: [PATCH 08/34] fix(proxy): clear LIT002 in /v2/key/info batch-cap guard The two `or []` fallbacks only feed len(), so tuples do the job without a mutable literal. The detail dict mirrors the sibling HTTPException payload above and is never mutated, so it carries a mutable-ok reason. --- .../proxy/management_endpoints/key_management_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2b37d0bc8fc..85f6d25f6dc 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3675,11 +3675,11 @@ async def info_key_fn_v2( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={"message": "Malformed request. No keys passed in."}, ) - requested_key_count: Final = len(data.keys or []) + len(data.key_aliases or []) + requested_key_count: Final = len(data.keys or ()) + len(data.key_aliases or ()) if requested_key_count > MAX_KEY_INFO_KEYS_PER_REQUEST: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ + detail={ # mutable-ok: one-shot HTTPException payload matching the sibling detail dict above; never mutated after construction "message": ( f"Too many keys requested: {requested_key_count}. " f"At most {MAX_KEY_INFO_KEYS_PER_REQUEST} keys and key_aliases combined per request." From 41ab7e57e8ac4f222fa6191d1940d7e636debffd Mon Sep 17 00:00:00 2001 From: Thijmen Stavenuiter Date: Mon, 24 Aug 2026 09:09:30 +0200 Subject: [PATCH 09/34] ci(test-unit): raise job timeout to 60m for the three 55m shards caching-local, proxy-extras and enterprise-package gave pytest 20m but capped the job at 55m; with a 35m setup ceiling plus 5m of runner overhead the job deadline could preempt pytest itself. --- .github/workflows/test-unit.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a7c67f2b35d..2dfca3d308f 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -211,7 +211,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: proxy-extras artifact-name: proxy-extras @@ -219,7 +219,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: enterprise-package artifact-name: enterprise-package @@ -227,7 +227,7 @@ jobs: workers: 4 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: responses-caching-types artifact-name: responses-caching-types From 4e2574ce08701e89d6f61a5df8557e48d3c5d4bc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:44:43 -0700 Subject: [PATCH 10/34] fix(proxy): match /v1/audio/speech content-type to the returned audio format --- .../litellm_core_utils/audio_utils/utils.py | 30 ++++++++++++++- litellm/proxy/proxy_server.py | 15 +++----- .../audio_utils/test_utils.py | 30 +++++++++++++++ .../proxy/proxy_server/test_routes_audio.py | 38 ++++++++++++++++++- .../test_audio_speech_prometheus_hooks.py | 2 + 5 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index 3b3775a8fe6..4d75d5dc8f5 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -7,7 +7,12 @@ import os from dataclasses import dataclass from typing import Final -from litellm.types.files import get_file_mime_type_from_extension +from litellm.types.files import ( + AUDIO_FILE_TYPES, + FILE_EXTENSIONS, + FILE_MIME_TYPES, + get_file_mime_type_from_extension, +) from litellm.types.utils import FileTypes @@ -323,3 +328,26 @@ def calculate_request_duration(file: FileTypes) -> float | None: except Exception: # Silently fail if duration extraction fails return None + + +DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg" + + +def _speech_media_type_for_response_format(response_format: str) -> str | None: + file_type: Final = next( + (candidate for candidate, extensions in FILE_EXTENSIONS.items() if response_format.lower() in extensions), + None, + ) + if file_type is None or file_type not in AUDIO_FILE_TYPES: + return None + return FILE_MIME_TYPES[file_type] + + +def resolve_speech_media_type(upstream_content_type: str | None, response_format: str | None) -> str: + upstream_media_type: Final = (upstream_content_type or "").split(";", 1)[0].strip().lower() + if upstream_media_type.startswith("audio/"): + return upstream_media_type + requested_media_type: Final = ( + None if response_format is None else _speech_media_type_for_response_format(response_format) + ) + return requested_media_type or DEFAULT_SPEECH_MEDIA_TYPE diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af0aa9743bc..96ff4f0daef 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -263,6 +263,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, @@ -10877,15 +10878,11 @@ async def audio_speech( if callback_headers: custom_headers.update(callback_headers) - # Determine media type based on model type - media_type = "audio/mpeg" # Default for OpenAI TTS - request_model: Final = data.get("model", "") - if request_model: - request_model_lower: Final = request_model.lower() - if "gemini" in request_model_lower and ( - "tts" in request_model_lower or "preview-tts" in request_model_lower - ): - media_type = "audio/wav" # Gemini TTS returns WAV format after conversion + requested_format: Final = data.get("response_format") + media_type: Final = resolve_speech_media_type( + upstream_content_type=response.response.headers.get("content-type"), + response_format=requested_format if isinstance(requested_format, str) else None, + ) return StreamingResponse( _audio_speech_chunk_generator(response), diff --git a/tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py b/tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py new file mode 100644 index 00000000000..87207588ba5 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py @@ -0,0 +1,30 @@ +import pytest + +from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type + + +@pytest.mark.parametrize( + ("upstream_content_type", "response_format", "expected"), + [ + ("audio/wav", None, "audio/wav"), + ("AUDIO/WAV", None, "audio/wav"), + ("audio/flac; charset=binary", "mp3", "audio/flac"), + ("application/json", "flac", "audio/flac"), + ("application/octet-stream", "pcm", "audio/pcm"), + (None, "wav", "audio/wav"), + (None, "WAV", "audio/wav"), + (None, "opus", "audio/opus"), + (None, "aac", "audio/aac"), + (None, "mp3", "audio/mpeg"), + (None, "mp4", "audio/mpeg"), + (None, "bogus", "audio/mpeg"), + (None, None, "audio/mpeg"), + ("", None, "audio/mpeg"), + ], +) +def test_resolve_speech_media_type(upstream_content_type, response_format, expected): + resolved = resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=response_format, + ) + assert resolved == expected diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index 74542a3eaf6..b99affc2ac3 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -12,13 +12,15 @@ from __future__ import annotations import io from unittest.mock import AsyncMock, MagicMock +import httpx import pytest from litellm.proxy import proxy_server @pytest.fixture -def patched_speech(monkeypatch): +def patched_speech(monkeypatch, request): + upstream_content_type = getattr(request, "param", "audio/mpeg") monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) monkeypatch.setattr( proxy_server, @@ -37,6 +39,11 @@ def patched_speech(monkeypatch): monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) class _FakeBinaryResp: + response = httpx.Response( + status_code=200, + headers={} if upstream_content_type is None else {"content-type": upstream_content_type}, + ) + async def aiter_bytes(self, chunk_size: int = 8192): async def _gen(): yield b"\x00\x01\x02" @@ -152,6 +159,35 @@ def test_audio_speech_happy_path(client, auth_as, patched_speech, path): } +@pytest.mark.parametrize( + ("patched_speech", "response_format", "expected_content_type"), + [ + ("audio/wav", "wav", "audio/wav"), + ("audio/flac", "flac", "audio/flac"), + ("audio/pcm", "pcm", "audio/pcm"), + ("audio/wav", "mp3", "audio/wav"), + ("application/json", "flac", "audio/flac"), + (None, "wav", "audio/wav"), + (None, None, "audio/mpeg"), + ], + indirect=["patched_speech"], +) +def test_audio_speech_content_type_matches_audio_format( + client, auth_as, patched_speech, response_format, expected_content_type +): + """Regression for LIT-6482: /v1/audio/speech mislabeled wav/flac/pcm as audio/mpeg.""" + payload = { + "model": "tts-1", + "input": "Hi", + "voice": "alloy", + **({} if response_format is None else {"response_format": response_format}), + } + with auth_as(): + response = client.post("/v1/audio/speech", json=payload) + assert response.status_code == 200 + assert response.headers.get("content-type", "").split(";")[0] == expected_content_type + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_error(client, auth_as, patched_speech_error, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (error).""" diff --git a/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py b/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py index 99f6f3a9b72..959cb2b1e89 100644 --- a/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py +++ b/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py @@ -2,6 +2,7 @@ import asyncio import os from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi.testclient import TestClient @@ -29,6 +30,7 @@ def _make_mock_tts_response(): inner = MagicMock() inner.aiter_bytes = _aiter_bytes inner._hidden_params = {} + inner.response = httpx.Response(status_code=200, headers={"content-type": "audio/mpeg"}) async def _resolver(): return inner From c251d6d609e5a85ae8df28411304de9dc7d84b06 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:11:51 -0700 Subject: [PATCH 11/34] fix(vertex_ai): label TTS audio bytes with their real content-type --- .../litellm_core_utils/audio_utils/utils.py | 22 ++++++++ .../text_to_speech/transformation.py | 8 ++- .../audio_utils/test_utils.py | 30 ---------- .../litellm_core_utils/test_audio_utils.py | 56 +++++++++++++++++++ .../text_to_speech/test_transformation.py | 43 ++++++++++++++ 5 files changed, 126 insertions(+), 33 deletions(-) delete mode 100644 tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index 4d75d5dc8f5..89222bf8107 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -11,6 +11,7 @@ from litellm.types.files import ( AUDIO_FILE_TYPES, FILE_EXTENSIONS, FILE_MIME_TYPES, + FileType, get_file_mime_type_from_extension, ) from litellm.types.utils import FileTypes @@ -351,3 +352,24 @@ def resolve_speech_media_type(upstream_content_type: str | None, response_format None if response_format is None else _speech_media_type_for_response_format(response_format) ) return requested_media_type or DEFAULT_SPEECH_MEDIA_TYPE + + +_OGG_OPUS_HEAD_WINDOW: Final = 64 +_MPEG_FRAME_SYNC_MASK: Final = 0xE0 +_MPEG_FRAME_LAYER_MASK: Final = 0x06 + + +def speech_media_type_from_audio_bytes(audio: bytes) -> str | None: + if audio[:4] == b"RIFF" and audio[8:12] == b"WAVE": + return FILE_MIME_TYPES[FileType.WAV] + if audio[:4] == b"fLaC": + return FILE_MIME_TYPES[FileType.FLAC] + if audio[:4] == b"OggS": + is_opus: Final = b"OpusHead" in audio[:_OGG_OPUS_HEAD_WINDOW] + return FILE_MIME_TYPES[FileType.OPUS if is_opus else FileType.OGG] + if audio[:3] == b"ID3": + return FILE_MIME_TYPES[FileType.MP3] + if len(audio) < 2 or audio[0] != 0xFF or (audio[1] & _MPEG_FRAME_SYNC_MASK) != _MPEG_FRAME_SYNC_MASK: + return None + is_adts_aac: Final = (audio[1] & _MPEG_FRAME_LAYER_MASK) == 0 + return FILE_MIME_TYPES[FileType.AAC if is_adts_aac else FileType.MP3] diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index cf14ab88751..a5ff7eca021 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -11,6 +11,9 @@ from typing import TYPE_CHECKING, Any, Final, Union import httpx +from litellm.litellm_core_utils.audio_utils.utils import ( + speech_media_type_from_audio_bytes, +) from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, @@ -457,12 +460,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): if not response_content: raise ValueError("No audioContent in Vertex AI TTS response") - # Decode base64 to get binary content binary_data: Final = base64.b64decode(response_content) - - # Create an httpx.Response object with the binary data + media_type: Final = speech_media_type_from_audio_bytes(binary_data) response: Final = httpx.Response( status_code=200, + headers={} if media_type is None else {"content-type": media_type}, content=binary_data, ) diff --git a/tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py b/tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py deleted file mode 100644 index 87207588ba5..00000000000 --- a/tests/test_litellm/litellm_core_utils/audio_utils/test_utils.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type - - -@pytest.mark.parametrize( - ("upstream_content_type", "response_format", "expected"), - [ - ("audio/wav", None, "audio/wav"), - ("AUDIO/WAV", None, "audio/wav"), - ("audio/flac; charset=binary", "mp3", "audio/flac"), - ("application/json", "flac", "audio/flac"), - ("application/octet-stream", "pcm", "audio/pcm"), - (None, "wav", "audio/wav"), - (None, "WAV", "audio/wav"), - (None, "opus", "audio/opus"), - (None, "aac", "audio/aac"), - (None, "mp3", "audio/mpeg"), - (None, "mp4", "audio/mpeg"), - (None, "bogus", "audio/mpeg"), - (None, None, "audio/mpeg"), - ("", None, "audio/mpeg"), - ], -) -def test_resolve_speech_media_type(upstream_content_type, response_format, expected): - resolved = resolve_speech_media_type( - upstream_content_type=upstream_content_type, - response_format=response_format, - ) - assert resolved == expected diff --git a/tests/test_litellm/litellm_core_utils/test_audio_utils.py b/tests/test_litellm/litellm_core_utils/test_audio_utils.py index 0e8176fffce..693ff760e2d 100644 --- a/tests/test_litellm/litellm_core_utils/test_audio_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_audio_utils.py @@ -347,3 +347,59 @@ class TestNormalizeTranscriptionLanguageToBcp47: ) assert normalize_transcription_language_to_bcp47(language) == expected + + +class TestResolveSpeechMediaType: + @pytest.mark.parametrize( + ("upstream_content_type", "response_format", "expected"), + [ + ("audio/wav", None, "audio/wav"), + ("AUDIO/WAV", None, "audio/wav"), + ("audio/flac; charset=binary", "mp3", "audio/flac"), + ("application/json", "flac", "audio/flac"), + ("application/octet-stream", "pcm", "audio/pcm"), + (None, "wav", "audio/wav"), + (None, "WAV", "audio/wav"), + (None, "opus", "audio/opus"), + (None, "aac", "audio/aac"), + (None, "mp3", "audio/mpeg"), + (None, "mp4", "audio/mpeg"), + (None, "bogus", "audio/mpeg"), + (None, None, "audio/mpeg"), + ("", None, "audio/mpeg"), + ], + ) + def test_resolution(self, upstream_content_type, response_format, expected): + from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type + + resolved = resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=response_format, + ) + assert resolved == expected + + +class TestSpeechMediaTypeFromAudioBytes: + @pytest.mark.parametrize( + ("audio", "expected"), + [ + (b"RIFF\x24\x00\x00\x00WAVEfmt ", "audio/wav"), + (b"fLaC\x00\x00\x00\x22", "audio/flac"), + (b"OggS" + b"\x00" * 24 + b"OpusHead", "audio/opus"), + (b"OggS" + b"\x00" * 24 + b"\x01vorbis", "audio/ogg"), + (b"ID3\x04\x00\x00\x00\x00\x00\x00", "audio/mpeg"), + (b"\xff\xfb\x90\x64", "audio/mpeg"), + (b"\xff\xf3\x80\x00", "audio/mpeg"), + (b"\xff\xf1\x50\x80", "audio/aac"), + (b"\xff\xf9\x50\x80", "audio/aac"), + (b"RIFF\x24\x00\x00\x00AVI LIST", None), + (b"\xff\x00\x00\x00", None), + (b"\x00\x01\x02\x03\x04\x05", None), + (b"\xff", None), + (b"", None), + ], + ) + def test_sniffing(self, audio, expected): + from litellm.litellm_core_utils.audio_utils.utils import speech_media_type_from_audio_bytes + + assert speech_media_type_from_audio_bytes(audio) == expected diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 05da22a73fd..fba337b5f2c 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,3 +1,4 @@ +import base64 from unittest.mock import MagicMock, Mock, patch import httpx @@ -126,6 +127,48 @@ class TestVertexAITextToSpeechConfig: assert voice_dict == voice_input +@pytest.mark.parametrize( + ("audio", "expected_content_type"), + [ + (b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00", "audio/wav"), + (b"\xff\xfb\x90\x64\x00\x00\x00\x00", "audio/mpeg"), + (b"OggS" + b"\x00" * 24 + b"OpusHead", "audio/opus"), + (b"fLaC\x00\x00\x00\x22", "audio/flac"), + ], +) +def test_transform_text_to_speech_response_labels_content_type(audio, expected_content_type): + raw_response = httpx.Response( + status_code=200, + json={"audioContent": base64.b64encode(audio).decode()}, + ) + + result = VertexAITextToSpeechConfig().transform_text_to_speech_response( + model="vertex_ai/chirp", + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert result.response.headers["content-type"] == expected_content_type + assert result.response.content == audio + + +def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): + raw_pcm = b"\x00\x01\x02\x03\x04\x05\x06\x07" + raw_response = httpx.Response( + status_code=200, + json={"audioContent": base64.b64encode(raw_pcm).decode()}, + ) + + result = VertexAITextToSpeechConfig().transform_text_to_speech_response( + model="vertex_ai/chirp", + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert "content-type" not in result.response.headers + assert result.response.content == raw_pcm + + @patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") From 4cd11e8b19405c192b380be2f6c9a1a61518d1f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:20:40 -0700 Subject: [PATCH 12/34] fix(audio_utils): validate MPEG frame headers before labeling sniffed audio --- .../litellm_core_utils/audio_utils/utils.py | 38 ++++++++++++++++--- .../litellm_core_utils/test_audio_utils.py | 6 +++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index 89222bf8107..dab3e48f91a 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -355,8 +355,35 @@ def resolve_speech_media_type(upstream_content_type: str | None, response_format _OGG_OPUS_HEAD_WINDOW: Final = 64 -_MPEG_FRAME_SYNC_MASK: Final = 0xE0 -_MPEG_FRAME_LAYER_MASK: Final = 0x06 +_ADTS_SYNC_AND_LAYER_MASK: Final = 0xF6 +_ADTS_SYNC_AND_LAYER: Final = 0xF0 +_ADTS_SAMPLE_RATE_INDEX_LIMIT: Final = 13 +_MPEG_SYNC_MASK: Final = 0xE0 +_MPEG_LAYER_MASK: Final = 0x06 +_MPEG_RESERVED_VERSION: Final = 0x01 +_MPEG_INVALID_BITRATE_INDEX: Final = 0x0F +_MPEG_RESERVED_SAMPLE_RATE_INDEX: Final = 0x03 + + +def _adts_aac_frame_media_type(header: bytes) -> str | None: + sample_rate_index: Final = (header[2] >> 2) & 0x0F + return FILE_MIME_TYPES[FileType.AAC] if sample_rate_index < _ADTS_SAMPLE_RATE_INDEX_LIMIT else None + + +def _mpeg_audio_frame_media_type(header: bytes) -> str | None: + version: Final = (header[1] >> 3) & 0x03 + layer: Final = header[1] & _MPEG_LAYER_MASK + bitrate_index: Final = header[2] >> 4 + sample_rate_index: Final = (header[2] >> 2) & 0x03 + if ( + (header[1] & _MPEG_SYNC_MASK) != _MPEG_SYNC_MASK + or version == _MPEG_RESERVED_VERSION + or layer == 0 + or bitrate_index == _MPEG_INVALID_BITRATE_INDEX + or sample_rate_index == _MPEG_RESERVED_SAMPLE_RATE_INDEX + ): + return None + return FILE_MIME_TYPES[FileType.MP3] def speech_media_type_from_audio_bytes(audio: bytes) -> str | None: @@ -369,7 +396,8 @@ def speech_media_type_from_audio_bytes(audio: bytes) -> str | None: return FILE_MIME_TYPES[FileType.OPUS if is_opus else FileType.OGG] if audio[:3] == b"ID3": return FILE_MIME_TYPES[FileType.MP3] - if len(audio) < 2 or audio[0] != 0xFF or (audio[1] & _MPEG_FRAME_SYNC_MASK) != _MPEG_FRAME_SYNC_MASK: + if len(audio) < 3 or audio[0] != 0xFF: return None - is_adts_aac: Final = (audio[1] & _MPEG_FRAME_LAYER_MASK) == 0 - return FILE_MIME_TYPES[FileType.AAC if is_adts_aac else FileType.MP3] + if (audio[1] & _ADTS_SYNC_AND_LAYER_MASK) == _ADTS_SYNC_AND_LAYER: + return _adts_aac_frame_media_type(audio) + return _mpeg_audio_frame_media_type(audio) diff --git a/tests/test_litellm/litellm_core_utils/test_audio_utils.py b/tests/test_litellm/litellm_core_utils/test_audio_utils.py index 693ff760e2d..155f6680416 100644 --- a/tests/test_litellm/litellm_core_utils/test_audio_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_audio_utils.py @@ -393,8 +393,14 @@ class TestSpeechMediaTypeFromAudioBytes: (b"\xff\xf1\x50\x80", "audio/aac"), (b"\xff\xf9\x50\x80", "audio/aac"), (b"RIFF\x24\x00\x00\x00AVI LIST", None), + (b"\xff\xff\xff\xff\xff\xff", None), + (b"\xff\xfb\xf0\x00", None), + (b"\xff\xfb\x9c\x00", None), + (b"\xff\xeb\x90\x00", None), + (b"\xff\xf1\xf4\x80", None), (b"\xff\x00\x00\x00", None), (b"\x00\x01\x02\x03\x04\x05", None), + (b"\xff\xfb", None), (b"\xff", None), (b"", None), ], From 608603ee63e4544a73a433464ee54ca124c83fa6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:36:34 -0700 Subject: [PATCH 13/34] fix(speech): honor pcm response_format for Gemini TTS and reject unsupported containers --- .../speech_to_completion_bridge/handler.py | 2 + .../transformation.py | 54 ++++++++++++------ .../test_transformation.py | 57 ++++++++++++++++++- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index 9e949db625a..6c33621ec89 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -115,9 +115,11 @@ class SpeechToCompletionBridgeHandler: **request_data, ) + requested_response_format: Final = optional_params.get("response_format") if isinstance(result, ModelResponse): return self.transformation_handler.transform_response( model_response=result, + response_format=requested_response_format if isinstance(requested_response_format, str) else None, ) else: raise Exception(f"Unmapped response type. Got type: {type(result)}") diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index 9b757ce86be..e2d3fadf852 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -21,6 +21,8 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None: GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16" +GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm" +GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT}) class ChatAudioParam(TypedDict): @@ -29,6 +31,26 @@ class ChatAudioParam(TypedDict): class SpeechToCompletionBridgeTransformationHandler: + def _validate_response_format( + self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object] + ) -> None: + if not self._is_gemini_tts_model(model): + return + response_format: Final = optional_params.get("response_format") + if not isinstance(response_format, str) or response_format in GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: + return + from litellm.exceptions import BadRequestError + + supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS)) + raise BadRequestError( + message=( + f"Gemini TTS only produces raw PCM16 audio, so response_format='{response_format}'" + f" is not supported. Supported response formats: {supported}." + ), + model=model, + llm_provider=custom_llm_provider, + ) + def _chat_completion_params(self, optional_params: Mapping[str, object]) -> Mapping[str, object]: return MappingProxyType( { @@ -67,6 +89,7 @@ class SpeechToCompletionBridgeTransformationHandler: litellm_logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, ) -> dict: + self._validate_response_format(model, custom_llm_provider, optional_params) user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": input} return_kwargs: Final = { "model": model, @@ -125,7 +148,14 @@ class SpeechToCompletionBridgeTransformationHandler: """Check if the model is a Gemini TTS model that returns PCM16 data.""" return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower()) - def transform_response(self, model_response: "ModelResponse") -> "HttpxBinaryResponseContent": + def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]: + if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT: + return decoded_audio, "audio/pcm" + return self._convert_pcm16_to_wav(decoded_audio), "audio/wav" + + def transform_response( + self, model_response: "ModelResponse", response_format: str | None + ) -> "HttpxBinaryResponseContent": import base64 import httpx @@ -136,23 +166,15 @@ class SpeechToCompletionBridgeTransformationHandler: audio_part: Final = cast(Choices, model_response.choices[0]).message.audio if audio_part is None: raise ValueError("No audio part found in the response") - audio_content: Final = audio_part.data + decoded_audio: Final = base64.b64decode(audio_part.data) - # Decode base64 to get binary content - binary_data = base64.b64decode(audio_content) - - # Check if this is a Gemini TTS model that returns raw PCM16 data model: Final = getattr(model_response, "model", "") - headers: Final = {} - if self._is_gemini_tts_model(model): - # Convert PCM16 to WAV format for proper audio file playback - binary_data = self._convert_pcm16_to_wav(binary_data) - headers["Content-Type"] = "audio/wav" - else: - headers["Content-Type"] = "audio/mpeg" - - # Create an httpx.Response object - response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers) + content, content_type = ( + self._gemini_tts_response_body(decoded_audio, response_format) + if self._is_gemini_tts_model(model) + else (decoded_audio, "audio/mpeg") + ) + response: Final = httpx.Response(status_code=200, content=content, headers={"Content-Type": content_type}) binary_response: Final = HttpxBinaryResponseContent(response) binary_response.set_response_cost(_completion_response_cost(model_response)) return binary_response diff --git a/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py index c0c720bbaf6..953f028af3c 100644 --- a/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py +++ b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -1,3 +1,4 @@ +import base64 from typing import Final from unittest.mock import MagicMock @@ -8,8 +9,17 @@ from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ( SpeechToCompletionBridgeTransformationHandler, ) +from litellm.types.utils import ChatCompletionAudioResponse, Choices, Message, ModelResponse GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview" +PCM_BYTES: Final = b"\x01\x02\x03\x04" * 6 + + +def _model_response(model: str, pcm: bytes) -> ModelResponse: + audio: Final = ChatCompletionAudioResponse( + data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello" + ) + return ModelResponse(model=model, choices=[Choices(message=Message(content=None, audio=audio))]) def _bridge_request(response_format: str | None) -> dict: @@ -28,7 +38,7 @@ def _bridge_request(response_format: str | None) -> dict: ) -@pytest.mark.parametrize("response_format", ["wav", "mp3", "pcm", None]) +@pytest.mark.parametrize("response_format", ["wav", "pcm", None]) def test_gemini_tts_request_keeps_speech_response_format_out_of_chat_params(response_format: str | None) -> None: request: Final = _bridge_request(response_format) @@ -60,3 +70,48 @@ def test_non_gemini_request_forwards_speech_response_format_as_audio_format() -> assert "response_format" not in request assert request["audio"] == {"voice": "alloy", "format": "wav"} + + +@pytest.mark.parametrize("response_format", ["mp3", "flac", "opus", "aac"]) +def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_format: str) -> None: + with pytest.raises(litellm.BadRequestError) as excinfo: + _bridge_request(response_format) + + assert excinfo.value.status_code == 400 + assert response_format in str(excinfo.value) + assert "pcm" in str(excinfo.value) + assert "wav" in str(excinfo.value) + + +def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + response_format="pcm", + ) + + assert response.response.content == PCM_BYTES + assert response.response.headers["content-type"] == "audio/pcm" + + +@pytest.mark.parametrize("response_format", ["wav", None]) +def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: str | None) -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + response_format=response_format, + ) + + body: Final = response.response.content + assert body[:4] == b"RIFF" + assert body[8:12] == b"WAVE" + assert body[44:] == PCM_BYTES + assert response.response.headers["content-type"] == "audio/wav" + + +def test_non_gemini_response_keeps_original_bytes_and_mpeg_content_type() -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response("gpt-4o-audio-preview", PCM_BYTES), + response_format="mp3", + ) + + assert response.response.content == PCM_BYTES + assert response.response.headers["content-type"] == "audio/mpeg" From b67b44bdaaa747d6d40b24c2b9583b9caf332a38 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:10:14 -0700 Subject: [PATCH 14/34] fix(proxy): map audio_speech errors to their status codes instead of a blanket 500 --- litellm/proxy/proxy_server.py | 10 ++++++- .../proxy/proxy_server/test_routes_audio.py | 30 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ee1eb876b87..37034ea9a62 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10996,7 +10996,15 @@ async def audio_speech( ) verbose_proxy_logger.error("litellm.proxy.proxy_server.audio_speech(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) - raise e + if isinstance(e, (ProxyException, HTTPException)): + raise e + raise ProxyException( + message=getattr(e, "message", f"{e}"), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + openai_code=getattr(e, "code", None), + code=getattr(e, "status_code", 500), + ) @router.post( diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index b99affc2ac3..522a20cd34b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -86,6 +86,24 @@ def patched_speech_error(monkeypatch): yield +@pytest.fixture +def patched_speech_provider_rejection(monkeypatch, patched_speech_error): + import litellm + + async def _raise(*args, **kwargs): + raise litellm.BadRequestError( + message=( + "Gemini TTS only produces raw PCM16 audio, so response_format='mp3' is not supported." + " Supported response formats: pcm, wav." + ), + model="gemini-3.1-flash-tts-preview", + llm_provider="gemini", + ) + + monkeypatch.setattr(proxy_server, "route_request", _raise) + yield + + @pytest.fixture def patched_transcription(monkeypatch): router = MagicMock() @@ -198,6 +216,18 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path): assert len(response.content) > 0 +def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection): + """Regression for LIT-6501: a BadRequestError from the speech path surfaced as a generic 500.""" + payload = {"model": "gemini-tts", "input": "Hi", "voice": "Kore", "response_format": "mp3"} + with auth_as(): + response = client.post("/v1/audio/speech", json=payload) + assert response.status_code == 400 + error = response.json()["error"] + assert "response_format='mp3'" in error["message"] + assert "pcm" in error["message"] + assert "wav" in error["message"] + + @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) def test_audio_transcription_happy_path(client, auth_as, patched_transcription, path): """Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (happy).""" From 35a375e26f99f6246fd2e68e4c002532fef111a0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:10:15 -0700 Subject: [PATCH 15/34] fix(speech): stop vertex gemini tts from dropping response_format in cloud tts param mapping --- litellm/utils.py | 4 ++++ tests/test_litellm/test_utils.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index 5e9e115ed54..e7028301dab 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9409,6 +9409,10 @@ class ProviderConfigManager: return RunwayMLTextToSpeechConfig() elif litellm.LlmProviders.VERTEX_AI == provider: + if "gemini" in model: + # Gemini TTS uses the speech_to_completion bridge, and Google Cloud TTS param + # mapping would drop response_format before the bridge sees it (LIT-6501) + return None from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAITextToSpeechConfig, ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6524353aa48..935e4c61535 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1416,6 +1416,26 @@ def test_get_provider_rerank_config(): assert isinstance(config, HostedVLLMRerankConfig) +def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts(): + """Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS + dropped response_format before the speech_to_completion bridge could honor it.""" + from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig + from litellm.utils import LlmProviders + + assert ( + ProviderConfigManager.get_provider_text_to_speech_config( + model="gemini-2.5-flash-preview-tts", provider=LlmProviders.VERTEX_AI + ) + is None + ) + assert isinstance( + ProviderConfigManager.get_provider_text_to_speech_config( + model="en-US-Studio-O", provider=LlmProviders.VERTEX_AI + ), + VertexAITextToSpeechConfig, + ) + + # Models that should be skipped during testing OLD_PROVIDERS = ["aleph_alpha", "palm"] SKIP_MODELS = [ From 99a6dd02af50e6f822cf57d438a75097232bde15 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:46:11 -0700 Subject: [PATCH 16/34] fix(proxy): narrow audio_speech response before reading upstream content-type --- litellm/proxy/proxy_server.py | 5 ++++- .../proxy/proxy_server/test_routes_audio.py | 21 +++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 70057201af8..c996735ddbb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11068,8 +11068,11 @@ async def audio_speech( custom_headers.update(callback_headers) requested_format: Final = data.get("response_format") + upstream_content_type: Final = ( + response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None + ) media_type: Final = resolve_speech_media_type( - upstream_content_type=response.response.headers.get("content-type"), + upstream_content_type=upstream_content_type, response_format=requested_format if isinstance(requested_format, str) else None, ) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index 522a20cd34b..de76c7257cf 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -16,6 +16,7 @@ import httpx import pytest from litellm.proxy import proxy_server +from litellm.types.llms.openai import HttpxBinaryResponseContent @pytest.fixture @@ -38,20 +39,14 @@ def patched_speech(monkeypatch, request): monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - class _FakeBinaryResp: - response = httpx.Response( - status_code=200, - headers={} if upstream_content_type is None else {"content-type": upstream_content_type}, - ) - - async def aiter_bytes(self, chunk_size: int = 8192): - async def _gen(): - yield b"\x00\x01\x02" - - return _gen() - async def _llm_call(): - return _FakeBinaryResp() + return HttpxBinaryResponseContent( + httpx.Response( + status_code=200, + headers={} if upstream_content_type is None else {"content-type": upstream_content_type}, + content=b"\x00\x01\x02", + ) + ) async def _fake_route_request(*args, **kwargs): return _llm_call() From 849269d52d9f3b9d566627a6756b3eb0c2f16672 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:40:18 -0700 Subject: [PATCH 17/34] build(rust): configure native extension profiles --- .github/actions/cache-cargo-build/action.yml | 23 ++++++++++---------- litellm-rust/Cargo.toml | 9 ++++++++ litellm-rust/crates/python-bridge/Cargo.toml | 3 ++- pyproject.toml | 5 ++++- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index 36c6c790b84..0ccb58c012c 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -4,17 +4,16 @@ description: >- so only the first job on a given Cargo.lock compiles the bridge from scratch. litellm builds through maturin, which compiles litellm-rust/crates/python-bridge - in release mode before it can produce a wheel. `uv sync` therefore pays a full - build in every job that installs the workspace: measured at 2m40s per unit shard - on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught - it, because the uv cache holds wheels uv downloads rather than wheels it builds, - and a path dependency whose source moves every commit could never hit that cache - anyway. Cargo rebuilds only what changed when its target directory survives, so a - warm job pays for the bridge crate alone. + in the dev profile for editable installs. `uv sync` therefore pays a full build + in every job that installs the workspace. Nothing caught it, because the uv cache + holds wheels uv downloads rather than wheels it builds, and a path dependency + whose source moves every commit could never hit that cache anyway. Cargo rebuilds + only what changed when its target directory survives, so a warm job pays for the + bridge crate alone. - The key namespace is separate from test-rust.yml's. Both cache the same directory, - but that workflow fills it with debug and clippy artifacts, which a release build - cannot reuse, and a shared key would let whichever ran first deny the other a save. + The key namespace is separate from test-rust.yml's check and release caches. They + cache the same directory for different workloads, and a shared key would let + whichever ran first deny the others a save. runs: using: composite @@ -26,6 +25,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-cargo-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo-release- + ${{ runner.os }}-cargo-dev- diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 481ea3f8f66..c17a0605fc7 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -30,3 +30,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "unwind" +debug = false +incremental = false +strip = "symbols" diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 0c4a753f762..d461a483ae0 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,8 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["extension-module"] +default = ["abi3"] +abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] [dependencies] diff --git a/pyproject.toml b/pyproject.toml index 34c1fec1c11..23e96e475f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -262,7 +262,7 @@ healthcheck = [ ] [build-system] -requires = ["maturin==1.9.4"] +requires = ["maturin==1.15.0"] build-backend = "maturin" [tool.maturin] @@ -270,6 +270,9 @@ manifest-path = "litellm-rust/crates/python-bridge/Cargo.toml" module-name = "litellm.rust_bridge._native" python-source = "." bindings = "pyo3" +features = ["extension-module"] +profile = "release" +editable-profile = "dev" include = ["litellm/proxy/_experimental/out/**"] exclude = [ "litellm/proxy/enterprise", From 9b774d3dcff85654431f30c075ca1985321c6885 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:50:07 -0700 Subject: [PATCH 18/34] fix(ci): isolate editable Cargo cache namespace --- .github/actions/cache-cargo-build/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index 0ccb58c012c..c3b8ce22c68 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -25,6 +25,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo-dev- + ${{ runner.os }}-maturin-dev- From 33cc9c1c48d7d8551828d0137a97300833294582 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:55:53 -0700 Subject: [PATCH 19/34] fix(ui): keep litellm_credential_name from LiteLLM Params JSON when no credential is selected (#39005) * fix(ui): keep litellm_credential_name from LiteLLM Params JSON when no credential is selected Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): drop null litellm_credential_name from AddModelPanel payload fixture Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../panels/AddModelPanel.integration.test.tsx | 1 - .../handle_add_model_submit.test.tsx | 24 +++++++++++++++++++ .../add_model/handle_add_model_submit.tsx | 5 +++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 19e1e3aa8bd..efba26734ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -87,7 +87,6 @@ const alwaysMounted = { api_key: undefined, api_base: undefined, custom_llm_provider: "openai", - litellm_credential_name: null, model: "gpt-4o", }; diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx index 7ef09d34924..2aa8f93b72b 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx @@ -73,4 +73,28 @@ describe("prepareModelAddRequest", () => { expect(deployment.litellmParamsObj.litellm_credential_name).toBe("selected-credential"); expect(deployment.litellmParamsObj.timeout).toBe(5); }); + + it("keeps litellm_credential_name from LiteLLM Params JSON when no credential is selected", async () => { + const formValues = { + model_mappings: [ + { + public_name: "Public Model", + litellm_model: "litellm/public", + }, + ], + model_name: "custom-model-name", + litellm_extra_params: JSON.stringify({ + litellm_credential_name: "from-json", + timeout: 5, + }), + litellm_credential_name: null, + }; + + const deployments = await prepareModelAddRequest({ ...formValues }, "token", null); + + expect(deployments).toHaveLength(1); + const [deployment] = deployments!; + expect(deployment.litellmParamsObj.litellm_credential_name).toBe("from-json"); + expect(deployment.litellmParamsObj.timeout).toBe(5); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index bb2f78fa84e..b4ae51d5423 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -91,6 +91,9 @@ export const prepareModelAddRequest = async (formValues: Record, ac if (value === "") { continue; } + if (key === "litellm_credential_name" && value == null) { + continue; + } // Skip the custom_pricing and pricing_model fields as they're only used for UI control if (key === "custom_pricing" || key === "pricing_model" || key === "cache_control") { continue; @@ -124,7 +127,7 @@ export const prepareModelAddRequest = async (formValues: Record, ac if (value && value != undefined) { try { litellmExtraParams = JSON.parse(value); - if ("litellm_credential_name" in litellmExtraParams) { + if ("litellm_credential_name" in litellmExtraParams && formValues.litellm_credential_name) { delete litellmExtraParams.litellm_credential_name; } } catch (error) { From b473339ac02c9618299a03483c5f1dc4a118aed4 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 31 Aug 2026 18:00:29 -0700 Subject: [PATCH 20/34] =?UTF-8?q?Revert=20"fix(ui):=20keep=20litellm=5Fcre?= =?UTF-8?q?dential=5Fname=20from=20LiteLLM=20Params=20JSON=20when=20n?= =?UTF-8?q?=E2=80=A6"=20(#39046)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 33cc9c1c48d7d8551828d0137a97300833294582. --- .../panels/AddModelPanel.integration.test.tsx | 1 + .../handle_add_model_submit.test.tsx | 24 ------------------- .../add_model/handle_add_model_submit.tsx | 5 +--- 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index efba26734ff..19e1e3aa8bd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -87,6 +87,7 @@ const alwaysMounted = { api_key: undefined, api_base: undefined, custom_llm_provider: "openai", + litellm_credential_name: null, model: "gpt-4o", }; diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx index 2aa8f93b72b..7ef09d34924 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx @@ -73,28 +73,4 @@ describe("prepareModelAddRequest", () => { expect(deployment.litellmParamsObj.litellm_credential_name).toBe("selected-credential"); expect(deployment.litellmParamsObj.timeout).toBe(5); }); - - it("keeps litellm_credential_name from LiteLLM Params JSON when no credential is selected", async () => { - const formValues = { - model_mappings: [ - { - public_name: "Public Model", - litellm_model: "litellm/public", - }, - ], - model_name: "custom-model-name", - litellm_extra_params: JSON.stringify({ - litellm_credential_name: "from-json", - timeout: 5, - }), - litellm_credential_name: null, - }; - - const deployments = await prepareModelAddRequest({ ...formValues }, "token", null); - - expect(deployments).toHaveLength(1); - const [deployment] = deployments!; - expect(deployment.litellmParamsObj.litellm_credential_name).toBe("from-json"); - expect(deployment.litellmParamsObj.timeout).toBe(5); - }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index b4ae51d5423..bb2f78fa84e 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -91,9 +91,6 @@ export const prepareModelAddRequest = async (formValues: Record, ac if (value === "") { continue; } - if (key === "litellm_credential_name" && value == null) { - continue; - } // Skip the custom_pricing and pricing_model fields as they're only used for UI control if (key === "custom_pricing" || key === "pricing_model" || key === "cache_control") { continue; @@ -127,7 +124,7 @@ export const prepareModelAddRequest = async (formValues: Record, ac if (value && value != undefined) { try { litellmExtraParams = JSON.parse(value); - if ("litellm_credential_name" in litellmExtraParams && formValues.litellm_credential_name) { + if ("litellm_credential_name" in litellmExtraParams) { delete litellmExtraParams.litellm_credential_name; } } catch (error) { From 3fadcd71553dcf02c76e465a17af34cca715e7ef Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 31 Aug 2026 18:19:40 -0700 Subject: [PATCH 21/34] fix(auth): quiet malformed virtual key rejections to stdout (#38838) * fix(auth): quiet malformed virtual key rejections to stdout Reduce noisy invalid-api-key error logs by classifying malformed virtual keys and routing their rejections to stdout as WARNING instead of stderr as ERROR. Suppressible via LITELLM_LOG=ERROR or log_client_error_tracebacks=true. Changes: - auth_utils: is_invalid_virtual_key_error() classifier and marker functions - auth_exception_handler: log invalid keys as WARNING to child logger before identity seeding and callbacks, escalate non-401 transforms to ERROR - user_api_key_auth: websocket early-raise WebSocketException(1008) to avoid double-logging at HTTP layer - _logging: child logger verbose_proxy_stdout_logger with no handler/level; LevelRoutingStreamHandler routes its WARNING records to stdout; handler setLevel in _turn_on_json() closes JSON config handler level leak - test_auth_exception_handler: new test case verifying malformed-key logs at WARNING with marker retention through transformations Fixes LIT-5362 * fix(auth): classify malformed-key 401 by raise-site marker, not message text Review round 1 (Greptile P2, veria Low): - Move the marker attribute name to litellm/constants.py per the shared sentinel convention - Stamp the marker on the malformed-key 401 where it is raised and classify only by it. Message text is caller-influenceable on other 401s (vector store ids, organization ids are interpolated into their messages), so a phrase match would let a request body demote an authorization failure to the quiet log path - Regression test: a 401 carrying the phrase but not the marker stays at ERROR on stderr --- litellm/_logging.py | 15 ++- litellm/constants.py | 6 + litellm/proxy/auth/auth_exception_handler.py | 103 +++++++++++------- litellm/proxy/auth/auth_utils.py | 38 +++++++ litellm/proxy/auth/user_api_key_auth.py | 14 ++- .../proxy/auth/test_auth_exception_handler.py | 36 +++++- 6 files changed, 162 insertions(+), 50 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index fbb35b72be2..9435562f890 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -264,13 +264,17 @@ def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: class LevelRoutingStreamHandler(logging.StreamHandler): - """Writes records below WARNING to stdout and WARNING and above to stderr. + """Writes records below WARNING and invalid-key warnings to stdout, others to stderr. Collectors that derive severity from the stream report every stderr line as an error. + Invalid-key warnings route to stdout so LITELLM_LOG=ERROR can suppress them. """ def emit(self, record: logging.LogRecord) -> None: - preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr + is_stdout_record: Final = record.levelno < logging.WARNING or ( + record.levelno == logging.WARNING and record.name == verbose_proxy_stdout_logger.name + ) + preferred: Final = sys.stdout if is_stdout_record else sys.stderr if preferred is None or getattr(preferred, "closed", False): self.stream = sys.stderr # rebind-ok: fall back to the pre-fix stream rather than raising per record else: @@ -508,6 +512,9 @@ else: handler.setFormatter(formatter) verbose_proxy_logger = logging.getLogger("LiteLLM Proxy") +# Malformed virtual key rejections log through this child; LevelRoutingStreamHandler +# writes its WARNING records to stdout. It has no handler or level of its own. +verbose_proxy_stdout_logger: Final = verbose_proxy_logger.getChild("stdout") verbose_router_logger = logging.getLogger("LiteLLM Router") verbose_logger = logging.getLogger("LiteLLM") @@ -520,6 +527,7 @@ verbose_logger.addHandler(handler) # handlers (JSON mode, uvicorn log config, a host app's root handler). verbose_router_logger.addFilter(_stdout_truncation_filter) verbose_proxy_logger.addFilter(_stdout_truncation_filter) +verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter) verbose_logger.addFilter(_stdout_truncation_filter) @@ -683,6 +691,7 @@ def _turn_on_json(): - Adds a JSON formatter to all loggers """ handler: Final = LevelRoutingStreamHandler() + handler.setLevel(numeric_level) handler.setFormatter(JsonFormatter()) _initialize_loggers_with_handler(handler) # Set up exception handlers @@ -700,12 +709,14 @@ def _disable_debugging(): verbose_logger.disabled = True verbose_router_logger.disabled = True verbose_proxy_logger.disabled = True + verbose_proxy_stdout_logger.disabled = True def _enable_debugging(): verbose_logger.disabled = False verbose_router_logger.disabled = False verbose_proxy_logger.disabled = False + verbose_proxy_stdout_logger.disabled = False def print_verbose(print_statement): diff --git a/litellm/constants.py b/litellm/constants.py index 1c8a6dc5874..c482ab0e39a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1427,6 +1427,12 @@ DEFAULT_SOFT_BUDGET: Final = float( ) # by default all litellm proxy keys have a soft budget of 50.0 # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY: Final = "LiteLLM Virtual Key user_api_key_hash" +# Prefix of the 401 raised when a submitted virtual key is not shaped like one. +INVALID_VIRTUAL_KEY_ERROR_MESSAGE: Final = "LiteLLM Virtual Key expected" +# Attribute stamped on that 401 at its raise site so log routing recognises it by +# provenance. Message text is caller-influenceable on other 401s, so it must not +# be used to classify. +INVALID_VIRTUAL_KEY_ERROR_MARKER: Final = "_litellm_invalid_virtual_key_error" # Python garbage collection threshold configuration # Format: "gen0,gen1,gen2" e.g., "1000,50,50" diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index a42187b3a44..64878a480a7 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -2,13 +2,14 @@ Handles Authentication Errors """ +import logging from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from fastapi import HTTPException, Request, status import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger from litellm.constants import EMPTY_MAPPING from litellm.integrations.otel.runtime import seed_request_identity from litellm.litellm_core_utils.core_helpers import is_expected_client_error @@ -18,7 +19,11 @@ from litellm.proxy._types import ( ProxyException, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_utils import _get_request_ip_address +from litellm.proxy.auth.auth_utils import ( + _get_request_ip_address, + is_invalid_virtual_key_error, + mark_invalid_virtual_key_error, +) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -36,6 +41,41 @@ else: Span = Any +def _as_proxy_exception(e: Exception) -> ProxyException: + """Convert an authentication failure into the ProxyException the client receives.""" + if isinstance(e, litellm.BudgetExceededError): + return ProxyException( + message=e.message, + type=ProxyErrorTypes.budget_exceeded, + param=None, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), + ) + if isinstance(e, HTTPException): + return ProxyException( + message=getattr(e, "detail", f"Authentication Error({e})"), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), + ) + if isinstance(e, ProxyException): + return e + if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): + return ProxyException( + message=( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + return ProxyException( + message="Authentication Error, " + str(e), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=status.HTTP_401_UNAUTHORIZED, + ) + + def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]: """Auth gate rejections are raised before `add_litellm_data_to_request` records the caller IP, so their failure logs would otherwise carry no IP nor key/user identity.""" @@ -110,16 +150,21 @@ class UserAPIKeyAuthExceptionHandler: request=request, use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True, ) - log_fn: Final = ( - verbose_proxy_logger.error - if is_expected_client_error(e) and not litellm.log_client_error_tracebacks - else verbose_proxy_logger.exception - ) - log_fn( + + # Log authentication failures before identity seeding and callbacks, so the log + # survives a raising callback pipeline. Classify and route malformed virtual-key + # rejections to WARNING on stdout (suppressible via LITELLM_LOG=ERROR). + log_extra: Final = {"requester_ip": requester_ip} + is_invalid_virtual_key: Final = is_invalid_virtual_key_error(e) + is_quiet_log: Final = is_invalid_virtual_key and not litellm.log_client_error_tracebacks + logger: Final = verbose_proxy_stdout_logger if is_quiet_log else verbose_proxy_logger + logger.log( + logging.WARNING if is_quiet_log else logging.ERROR, "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", e, requester_ip, - extra={"requester_ip": requester_ip}, + exc_info=True if litellm.log_client_error_tracebacks or not is_expected_client_error(e) else None, + extra=log_extra, ) # Log this exception to OTEL, Datadog etc. Reuse the identity resolved @@ -167,35 +212,13 @@ class UserAPIKeyAuthExceptionHandler: if transformed_exception is not None: e = transformed_exception - if isinstance(e, litellm.BudgetExceededError): - raise ProxyException( - message=e.message, - type=ProxyErrorTypes.budget_exceeded, - param=None, - code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), + final_exception: Final = mark_invalid_virtual_key_error(_as_proxy_exception(e), is_invalid_virtual_key) + # If a quiet-logged malformed-key transform yields non-401, escalate to ERROR + if is_quiet_log and str(final_exception.code) != str(status.HTTP_401_UNAUTHORIZED): + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", + final_exception, + requester_ip, + extra=log_extra, ) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), - ) - elif isinstance(e, ProxyException): - raise e - if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - raise ProxyException( - message=( - "Service Unavailable, the authentication database is " - "temporarily unreachable. Please retry shortly." - ), - type=ProxyErrorTypes.no_db_connection, - param="None", - code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_401_UNAUTHORIZED, - ) + raise final_exception diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9b1a6ba5aa7..83ee10b8108 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, EMPTY_MAPPING, + INVALID_VIRTUAL_KEY_ERROR_MARKER, MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) @@ -34,6 +35,43 @@ from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS from litellm.types.utils import CustomPricingLiteLLMParams +def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: + """True when an authentication error rejects a malformed virtual key. + + Classifies only by the marker stamped where that 401 is raised. Message + content is never inspected: other 401s interpolate caller-supplied values + (vector store ids, organization ids) into their messages, so a phrase + match would let a request body demote an authorization failure to the + quiet log path. + """ + if not isinstance(exception, (HTTPException, ProxyException)): + return False + + code: Final[object] = getattr(exception, "code", None) + status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None) + if str(status_code) != str(status.HTTP_401_UNAUTHORIZED): + return False + + return getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True + + +def mark_invalid_virtual_key_error(exception: ProxyException, is_invalid_virtual_key: bool) -> ProxyException: + """Return an independently marked malformed-key exception after callback transformations.""" + if not is_invalid_virtual_key or str(exception.code) != str(status.HTTP_401_UNAUTHORIZED): + return exception + marked_exception: Final = ProxyException( + message=exception.message, + type=exception.type, + param=exception.param, + code=exception.code, + headers=exception.headers.copy(), + openai_code=None if exception.openai_code is None else str(exception.openai_code), + provider_specific_fields=exception.provider_specific_fields, + ) + setattr(marked_exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + return marked_exception + + def _get_request_ip_address(request: Request, use_x_forwarded_for: bool | None = False) -> str | None: client_ip = None if use_x_forwarded_for is True and "x-forwarded-for" in request.headers: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e92d090a2fb..5fb6dad0cd7 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -19,12 +19,15 @@ import fastapi import orjson from fastapi import HTTPException, Request, WebSocket, status from fastapi.security.api_key import APIKeyHeader +from starlette.exceptions import WebSocketException import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( GLOBAL_PROXY_SPEND_CACHE_KEY, + INVALID_VIRTUAL_KEY_ERROR_MARKER, + INVALID_VIRTUAL_KEY_ERROR_MESSAGE, LITELLM_PROXY_BUDGET_NAME, LITELLM_PROXY_MASTER_KEY_ALIAS, ) @@ -65,6 +68,7 @@ from litellm.proxy.auth.auth_utils import ( get_model_from_request, get_request_route, get_request_route_template, + is_invalid_virtual_key_error, iter_request_fallback_targets, normalize_request_route, pre_db_read_auth_checks, @@ -539,6 +543,8 @@ async def user_api_key_auth_websocket(websocket: WebSocket): try: return await user_api_key_auth(request=request, api_key=f"Bearer {api_key}") except Exception as e: + if is_invalid_virtual_key_error(e): + raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) verbose_proxy_logger.exception(e) await websocket.close(code=status.WS_1008_POLICY_VIOLATION) raise HTTPException(status_code=403, detail=str(e)) @@ -1867,13 +1873,17 @@ async def _user_api_key_auth_builder( _masked_key: Final = f"{api_key[:4]}****{api_key[-4:]}" if len(api_key) > 8 else "****" if not api_key.startswith("sk-"): _hint = _JWT_AUTH_DISABLED_HINT if not enable_jwt_auth and JWTHandler.is_jwt(token=api_key) else "" - raise HTTPException( + _malformed_key_error = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail=( - f"LiteLLM Virtual Key expected. Received={_masked_key}, " + f"{INVALID_VIRTUAL_KEY_ERROR_MESSAGE}. Received={_masked_key}, " f"expected to start with 'sk-'.{_hint}" ), ) # prevent token hashes from being used + # Stamp provenance here so log routing classifies this 401 by + # where it was raised, never by its message text. + setattr(_malformed_key_error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + raise _malformed_key_error else: verbose_logger.warning( "litellm.proxy.proxy_server.user_api_key_auth(): Warning - Key is not a string. Got type={}".format( diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 90b3b29d919..90be51cfa5b 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -26,6 +26,7 @@ from prisma.errors import ( from litellm._logging import verbose_proxy_logger +from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler @@ -703,23 +704,43 @@ async def test_auth_failure_ip_stamp_does_not_mutate_callers_request_data(): assert request_data == {"model": "gpt-4o"} +def _marked_malformed_key_error() -> HTTPException: + """Build the malformed-key 401 as its raise site does: marker stamped on it.""" + error = HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test") + setattr(error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + return error + + @pytest.mark.asyncio @pytest.mark.parametrize( - "auth_error,expect_traceback", + "auth_error,expect_traceback,expect_level", [ pytest.param( ProxyException( message="Authentication Error", type=ProxyErrorTypes.auth_error, param=None, code=401 ), False, + "ERROR", id="expected_401_no_traceback", ), - pytest.param(ValueError("unexpected internal error"), True, id="unexpected_error_keeps_traceback"), + pytest.param(ValueError("unexpected internal error"), True, "ERROR", id="unexpected_error_keeps_traceback"), + pytest.param( + _marked_malformed_key_error(), + False, + "WARNING", + id="malformed_virtual_key_warning_no_traceback", + ), + pytest.param( + HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test"), + False, + "ERROR", + id="phrase_without_marker_stays_loud", + ), ], ) -async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, caplog): +async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, expect_level, caplog): """Regression for LIT-6043: expected 4xx auth rejections must not format a - traceback via logger.exception; unexpected errors must keep it.""" + traceback via logger.exception; malformed virtual keys log at WARNING.""" handler = UserAPIKeyAuthExceptionHandler() with ( @@ -740,8 +761,8 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( try: try: raise auth_error - except (ProxyException, ValueError) as caught: - with caplog.at_level("ERROR", logger="LiteLLM Proxy"), pytest.raises(ProxyException): + except (ProxyException, ValueError, HTTPException) as caught: + with caplog.at_level(expect_level, logger="LiteLLM Proxy"), pytest.raises((ProxyException, HTTPException)): await handler._handle_authentication_error( caught, MagicMock(), @@ -756,3 +777,6 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( records = [r for r in caplog.records if "user_api_key_auth(): Exception occured" in r.getMessage()] assert len(records) == 1 assert (records[0].exc_info is not None) is expect_traceback + assert records[0].levelname == expect_level + expected_logger_name = "LiteLLM Proxy.stdout" if expect_level == "WARNING" else "LiteLLM Proxy" + assert records[0].name == expected_logger_name From f7accc4e29da707c2e1797c2217b6d1c420356f8 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 31 Aug 2026 19:33:04 -0700 Subject: [PATCH 22/34] test(e2e): drop the two mgmt registry cells no shared-proxy test can cover mgmt.cache_settings.update.happy_path and mgmt.config_override.hashicorp_vault.happy_path were the last two uncovered Management/UI cells, and neither can be covered against the shared proxy the e2e suites run on. Both routes reconfigure the whole process rather than a resource the test owns. /cache/settings persists whatever it receives into a row that outranks the YAML cache_params and is re-applied on a timer, so a partial write downgrades a TLS cluster to a plaintext standalone node and every later Redis call hangs. That is what took out 60 of 72 tests on 2026-07-25 and got the original test removed in PR #34664. /config_overrides/hashicorp_vault has the same shape: a POST sets the HCP_VAULT_* env vars, swaps litellm.secret_manager_client process-wide, and writes a row the config-reload poll re-applies, so every os.environ/ lookup on the pod resolves against the test's Vault until the DELETE lands. Its constructor also never dials Vault, so a POST to a bogus address still returns 200 and a smoke test built on it would pass for the wrong reason. Keeping rows we have decided not to cover only inflates the denominator, so drop them and record the reasoning where someone would go to write the test. Filing the isolated-proxy harness they both need separately; the cells come back with it. Management/UI goes 75/77 to 75/75, headline 402/544 to 402/542. --- tests/e2e/coverage_registry/mgmt.yaml | 2 -- tests/e2e/management/test_config_misc_endpoints_e2e.py | 10 +++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 1e6de0c3d6a..d571fb36546 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -64,14 +64,12 @@ - {id: mgmt.budget.list_v1.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "management_v1/budgets.py:129", rationale: "Budget enumeration the Budgets page can page, sort and filter"} - {id: mgmt.budget.list_v1.admin_only, module: mgmt, tier: P1, surface: api, assertions: [admin_only], source: "management_v1/budgets.py:129", rationale: "A caller without admin view is refused, not served an empty page"} - {id: mgmt.callback.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "callback_management_endpoints.py", rationale: "Callback config (smoke)"} -- {id: mgmt.cache_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cache_settings_endpoints.py", rationale: "Cache config (smoke). Deliberately uncovered: the previous test read the live settings and wrote them back, which proves nothing (identical values in, so a no-op POST still passes) while being able to break the deployment. /cache/settings persists what it receives and that row outranks YAML cache_params, re-applied on a timer, so a write that omits ssl or redis_startup_nodes turns a TLS cluster into a plaintext standalone node and every later Redis call hangs. That took out 60 of 72 tests on 2026-07-25. GET cannot round-trip it either: it resolves the stored row overlaid with REDIS_* env and never reads YAML, so on a fresh deploy it cannot see YAML ssl to echo back. A safe test needs an isolated proxy, or LIT-4816 fixed so a partial write cannot downgrade transport. Do not re-add a read-then-write-back test against a shared proxy."} - {id: mgmt.cost_tracking.estimate.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cost_tracking_settings.py", rationale: "Cost estimate (smoke)"} - {id: mgmt.router_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "router_settings_endpoints.py", rationale: "Router config (smoke)"} - {id: mgmt.jwt_key_mapping.new.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "jwt_key_mapping_endpoints.py", rationale: "JWT->key mapping (smoke)"} - {id: mgmt.compliance.gdpr.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "compliance_endpoints.py", rationale: "GDPR ops (smoke)"} - {id: mgmt.tool_management.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "tool_management_endpoints.py", rationale: "Tool inventory (smoke)"} - {id: mgmt.fallback_management.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "fallback_management_endpoints.py", rationale: "Fallback config (smoke)"} -- {id: mgmt.config_override.hashicorp_vault.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "config_override_endpoints.py", rationale: "Vault integration (smoke)"} - {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"} - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} diff --git a/tests/e2e/management/test_config_misc_endpoints_e2e.py b/tests/e2e/management/test_config_misc_endpoints_e2e.py index 195732c0201..099ffa4b3bd 100644 --- a/tests/e2e/management/test_config_misc_endpoints_e2e.py +++ b/tests/e2e/management/test_config_misc_endpoints_e2e.py @@ -7,9 +7,13 @@ so a read-back reflects the change. Router settings, which mutate global proxy state, are exercised with a benign, self-restoring change so a shared proxy is left as it was found. -Cache settings are deliberately not covered here; see the rationale on -mgmt.cache_settings.update.happy_path in coverage_registry/mgmt.yaml before adding -a test for that route. +Cache settings and the Vault config override are deliberately not covered here. +Both routes reconfigure the whole proxy: /cache/settings persists what it receives +into a row that outranks the YAML cache_params and is re-applied on a timer, and +/config_overrides/hashicorp_vault swaps the process-wide secret manager. Neither can +be exercised safely against the shared proxy the suites run on, so they need an +isolated proxy before a test lands. Do not add a read-then-write-back test for +either one. """ from __future__ import annotations From ccd76dac505035885b34e54d325ea9bfe9a6718d Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 31 Aug 2026 19:38:39 -0700 Subject: [PATCH 23/34] fix(proxy): wire team-level logging callbacks into passthrough endpoints (#38979) * fix(proxy): wire team-level logging callbacks into passthrough endpoints LIT-5152: passthrough routes now wire dynamic team-level callbacks (success_callback, failure_callback, callback_vars) into Logging constructor, mirroring the add_litellm_data_to_request behavior. Three hardening fixes: 1. Catch TypeError/AttributeError in _get_validated_callback_metadata when team logging metadata has wrong shape (e.g., logging list instead of dict), preventing HTTP 500 on passthrough routes with malformed config. 2. Wrap websocket passthrough logging initialization in try/except, since the socket is already accepted at that point; errors after accept() yield abrupt close (1006/1011) rather than clean HTTP error response. 3. Handle malformed deprecated callback_settings gracefully with try/except. 4. Wrap HTTP passthrough callback resolution in try/except to prevent 500 on malformed team metadata (backward-compatibility fix). Changes: - pass_through_endpoints.py: wire dynamic callbacks in HTTP+WS paths, handle malformed metadata gracefully with try/except fallbacks - litellm_pre_call_utils.py: expand exception handling in validators - test file: regression test for happy-path team callback wiring * refactor(proxy): share passthrough team-callback resolution and cover its fail-open path Collapse the duplicated callback wiring on the HTTP and websocket passthrough paths into one helper that returns a frozen wiring value, log resolution failures at error level so a broken logging config stays visible, and add regression tests for malformed team metadata and an operational lookup failure. Reverts the _get_validated_callback_metadata except widening: it changed behavior for normal LLM routes, which is outside this ticket's scope. * fix(proxy): keep passthrough alive when team callback vars hold env references The deprecated team_metadata.callback_settings branch builds TeamCallbackMetadata directly, skipping the AddTeamCallback validation that strips os.environ/ references from the newer logging list. Stamping those vars onto the Logging object made its constructor raise, so a team on the legacy shape got HTTP 500 on every passthrough call. Validate the resolved vars inside the fail-open boundary instead, so the request goes through with dynamic callbacks skipped and the reason logged. * fix(proxy): lint violations in team callback wiring helper * style: format lint --- .../pass_through_endpoints.py | 95 ++++++++++- .../test_pass_through_endpoints.py | 161 ++++++++++++++++++ 2 files changed, 251 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 09d3dedaafa..ddd27f0fc2a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -6,9 +6,10 @@ import posixpath import traceback from base64 import b64encode from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from itertools import groupby -from typing import Any, Final, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, TypedDict, cast from urllib.parse import urlencode, urlparse import httpx @@ -47,6 +48,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_no_callback_env_reference from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -78,7 +80,10 @@ from litellm.proxy.common_utils.http_parsing_utils import ( from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above +) from litellm.proxy.utils import normalize_route_for_root_path from litellm.repositories.team_repository import TeamRepository from litellm.secret_managers.main import get_secret_str @@ -90,7 +95,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import Usage +from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging @@ -99,6 +104,9 @@ from .upstream_usage_headers import ( apply_upstream_reported_usage, ) +if TYPE_CHECKING: + from litellm.proxy.proxy_server import ProxyConfig + router: Final = APIRouter() pass_through_endpoint_logging: Final = PassThroughEndpointLogging() @@ -752,6 +760,67 @@ def _build_passthrough_failure_request_payload( return request_payload +@dataclass(frozen=True, slots=True) +class _TeamCallbackWiring: + success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg + + +def _resolve_team_callback_wiring( + user_api_key_dict: UserAPIKeyAuth, + proxy_config: "ProxyConfig", + route_description: str, +) -> _TeamCallbackWiring: + """Resolve key/team dynamic logging callbacks for a passthrough request. + + Mirrors add_litellm_data_to_request: callback_vars are unpacked top-level + (read by initialize_standard_callback_dynamic_params) and also stamped on + the proxy-owned trusted-vars field (read by get_trusted_callback_params). + + Fails open: a callback resolution or validation error is logged at error + level and the request proceeds without dynamic callbacks, since a broken + logging config must not fail the customer's upstream call (and the + websocket is already accepted by the time this runs on that path). The + env-reference check runs here because the deprecated callback_settings + branch skips AddTeamCallback validation, and Logging.__init__ would + otherwise reject the vars mid-request. + """ + try: + callback_settings_obj: Final = _get_dynamic_logging_metadata( + user_api_key_dict=user_api_key_dict, proxy_config=proxy_config + ) + if callback_settings_obj and callback_settings_obj.callback_vars: + for ( + item + ) in callback_settings_obj.callback_vars.items(): # rebind-ok: dict.items iteration for env-ref validation + validate_no_callback_env_reference(item[0], item[1], source="key/team callback metadata") + except Exception: # noqa: BLE001 - a broken logging config must never fail the passthrough request + verbose_proxy_logger.exception( + "%s: failed to resolve team logging callbacks, continuing without them", + route_description, + ) + return _TeamCallbackWiring() + if callback_settings_obj is None: + return _TeamCallbackWiring() + callback_vars: Final = callback_settings_obj.callback_vars + success_callbacks: Final = callback_settings_obj.success_callback + failure_callbacks: Final = callback_settings_obj.failure_callback + logging_kwargs: Final = ( + None + if not callback_vars + else { # mutable-ok: Logging arg + **callback_vars, + TRUSTED_CALLBACK_VARS_FIELD: callback_vars, + } + ) + return _TeamCallbackWiring( + success_callbacks=None if success_callbacks is None else [*success_callbacks], # mutable-ok: Logging arg + failure_callbacks=None if failure_callbacks is None else [*failure_callbacks], # mutable-ok: Logging arg + logging_kwargs=logging_kwargs, + ) + + async def _log_passthrough_upstream_failure( response: httpx.Response, user_api_key_dict: UserAPIKeyAuth, @@ -845,7 +914,7 @@ async def pass_through_request( from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( PassthroughGuardrailHandler, ) - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj ######################################################### # Initialize variables @@ -930,6 +999,11 @@ async def pass_through_request( # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. passthrough_model: Final = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown" start_time: Final = datetime.now() + team_callbacks: Final = _resolve_team_callback_wiring( + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_description="pass_through_endpoint", + ) logging_obj = Logging( model=passthrough_model, messages=[{"role": "user", "content": safe_dumps(_parsed_body)}], @@ -938,6 +1012,9 @@ async def pass_through_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="1245", + dynamic_success_callbacks=team_callbacks.success_callbacks, + dynamic_failure_callbacks=team_callbacks.failure_callbacks, + kwargs=team_callbacks.logging_kwargs, ) # Store passthrough guardrails config on logging_obj for field targeting @@ -2022,7 +2099,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream """ from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) @@ -2055,6 +2132,11 @@ async def websocket_passthrough_request( upstream_headers[header_name] = header_value # Initialize logging object similar to HTTP passthrough + team_callbacks: Final = _resolve_team_callback_wiring( + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_description="websocket_passthrough", + ) logging_obj: Final = Logging( model="unknown", messages=[{"role": "user", "content": "WebSocket connection"}], @@ -2063,6 +2145,9 @@ async def websocket_passthrough_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="websocket_passthrough", + dynamic_success_callbacks=team_callbacks.success_callbacks, + dynamic_failure_callbacks=team_callbacks.failure_callbacks, + kwargs=team_callbacks.logging_kwargs, ) # Create passthrough logging payload diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a3f56adb86f..f5ae0fe5977 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5462,3 +5462,164 @@ def test_the_marker_check_distinguishes_the_two_route_kinds(): builtin = MagicMock(spec=Request) builtin.scope = {"endpoint": llm_passthrough_endpoints.anthropic_proxy_route} assert request_dispatched_to_pass_through_endpoint(builtin) is False + + +async def _drive_passthrough_request_and_capture_logging(user_api_key_dict: UserAPIKeyAuth) -> tuple[int, object]: + import litellm + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + def transport_handler(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"ok": True}) + + real_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": resolve_pass_through_request_timeout(None)}, + ) + cache_dict = litellm.in_memory_llm_clients_cache.cache_dict + cache_key = next((key for key, cached in cache_dict.items() if cached is real_handler), None) + assert cache_key is not None + cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + mock_request.body = AsyncMock(return_value=b'{"model": "gemini-2.0-flash"}') + + captured_data: dict = {} + + async def capture_pre_call_hook(user_api_key_dict, data, call_type): + captured_data.update(data) + return data + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=capture_pre_call_hook) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + mock_proxy_logging.get_proxy_hook = MagicMock(return_value=None) + + try: + with patch( # test-quality-ok: proxy_logging_obj is a proxy_server module global read inside pass_through_request; there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ): + response = await pass_through_request( + request=mock_request, + target="https://upstream.example.test/v1/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + finally: + cache_dict[cache_key] = real_handler + + return response.status_code, captured_data.get("litellm_logging_obj") + + +@pytest.mark.asyncio +async def test_pass_through_request_wires_team_callbacks(): + """LIT-5152 regression: pass_through_request must resolve team-level logging + callbacks from key/team metadata and wire them into the Logging object, the + same way add_litellm_data_to_request does for normal LLM routes.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={ + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": { + "langfuse_public_key": "pk_test", + "langfuse_secret_key": "sk_test", + "langfuse_host": "https://langfuse.example.test", + }, + } + ] + }, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert logging_obj.dynamic_success_callbacks, "team success callbacks not wired into Logging" + assert logging_obj.dynamic_failure_callbacks, "team failure callbacks not wired into Logging" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_public_key") == "pk_test" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_secret_key") == "sk_test" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_host") == "https://langfuse.example.test" + assert ("langfuse_public_key", "pk_test") in logging_obj._trusted_callback_vars + + +@pytest.mark.asyncio +async def test_pass_through_request_survives_malformed_team_logging_metadata(): + """LIT-5152 fail-open: a malformed team ``logging`` value (here a non-iterable) + raises inside callback resolution; the passthrough request must still succeed, + just without dynamic callbacks.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={"logging": 5}, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert not logging_obj.dynamic_success_callbacks + assert not logging_obj.dynamic_failure_callbacks + + +@pytest.mark.asyncio +async def test_pass_through_request_survives_env_reference_in_deprecated_callback_settings(): + """LIT-5152 fail-open: the deprecated ``callback_settings`` team metadata skips + AddTeamCallback validation, so an ``os.environ/`` callback var would otherwise + blow up inside ``Logging.__init__`` and fail the request; the passthrough must + instead succeed without dynamic callbacks.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={ + "callback_settings": { + "success_callback": ["langfuse"], + "failure_callback": ["langfuse"], + "callback_vars": { + "langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY", + "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY", + "langfuse_host": "https://langfuse.example.test", + }, + } + }, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert not logging_obj.dynamic_success_callbacks + assert not logging_obj.dynamic_failure_callbacks + assert not logging_obj.standard_callback_dynamic_params.get("langfuse_public_key") + + +@pytest.mark.asyncio +async def test_resolve_team_callback_wiring_fails_open_on_operational_error(): + """LIT-5152 fail-open: an operational error while resolving callback metadata + (e.g. team config lookup hitting a dead secret manager) must not raise; the + request proceeds without dynamic callbacks and the error is logged.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + _resolve_team_callback_wiring, + ) + from litellm.proxy.proxy_server import ProxyConfig + + class RaisingTeamConfig(ProxyConfig): + def load_team_config(self, team_id: str) -> dict: + raise RuntimeError("secret manager unavailable") + + wiring = _resolve_team_callback_wiring( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", team_id="test-team"), + proxy_config=RaisingTeamConfig(), + route_description="pass_through_endpoint", + ) + + assert wiring.success_callbacks is None + assert wiring.failure_callbacks is None + assert wiring.logging_kwargs is None From 8d6d7f9ce9e46155e5b4a63ba894c6e6eb4cd3f9 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 31 Aug 2026 19:51:41 -0700 Subject: [PATCH 24/34] feat(complexity_router): opt-in modality-based capability routing for image requests (#39032) --- .../prompt_templates/common_utils.py | 35 ++ .../complexity_router/README.md | 22 ++ .../complexity_router/complexity_router.py | 215 +++++++++++- .../complexity_router/config.py | 12 + litellm/types/utils.py | 4 + litellm/utils.py | 24 +- ...ore_utils_prompt_templates_common_utils.py | 47 +++ .../router_strategy/test_complexity_router.py | 320 ++++++++++++++++++ tests/test_litellm/test_utils.py | 30 ++ .../RoutingDecisionCard.test.tsx | 6 + .../LogDetailsDrawer/RoutingDecisionCard.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +- 12 files changed, 705 insertions(+), 19 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 1c8f10d3307..bda365102af 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -205,6 +205,41 @@ def is_non_content_values_set(message: AllMessageValues) -> bool: return any(message.get(key, None) is not None for key in message if key not in ignore_keys) +_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"}) +_IMAGE_SCAN_MAX_DEPTH: Final = 4 + + +def _content_parts_contain_image(parts: Sequence[object]) -> bool: + """Depth-bounded frontier walk over nested content lists, iterative because the repo bans + recursion; an Anthropic tool_result nests its image parts exactly one level down.""" + frontier = parts # rebind-ok: depth-bounded frontier walk + for _ in range(_IMAGE_SCAN_MAX_DEPTH): + if any(isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_PART_TYPES for part in frontier): + return True + frontier = tuple( # rebind-ok: depth-bounded frontier walk + nested + for part in frontier + if isinstance(part, Mapping) + for content in (part.get("content"),) + if isinstance(content, list) + for nested in content + ) + if not frontier: + return False + return False + + +def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool: + """Whether any message carries an image content part, across the dialects that reach + pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``, + and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks.""" + return any( + isinstance(content, list) and _content_parts_contain_image(content) + for message in messages + for content in (message.get("content"),) + ) + + def _audio_or_image_in_message_content(message: AllMessageValues) -> bool: """ Checks if message content contains an image or audio diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 63ba760ff66..bc8df67cc28 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -154,6 +154,9 @@ model_list: # Fallback model if tier cannot be determined default_model: gpt-4o + + # Replace a routed model that cannot take image input (default: false) + modality_routing: true ``` ## Usage @@ -178,6 +181,25 @@ response = litellm.completion( ## Special Behaviors +### Modality-based capability routing + +The classifier reads text alone, so a request carrying an image can classify cheap and land on a +text-only model, which rejects it with a provider 400 no fallback catches. With +`modality_routing: true`, one gate inspects every decided placement: when the routed model is +explicitly declared `supports_vision: false` (deployment `model_info` first, the model cost map +otherwise; unmapped names stay routable, and a multi-deployment group must accept on every +deployment), the request is re-placed on the nearest HIGHER tier holding a capable model, with +routing plugins still applied to the re-pick, then on `default_model` (never on plugin routers +and never for a plan-floored decision), and otherwise rejected with a clear 400 naming the +router. The walk only ever goes up, so a plan-mode floor cannot be undercut; a router whose only +vision model sits below the decided tier gets the 400 and an actionable message instead. + +A same-tier re-pick keeps the decision's cause and adds `modality:image` to `signals`; a tier +change or default takeover records `cause: modality_escalation` with the displaced placement +(`modality_escalated_from:` or `modality_displaced_default_model`). Escalations are never +pinned by session affinity, and a KEPT session pin bypasses the gate entirely: a session pinned +to a text-only model keeps it even when an image arrives. + ### Heuristic-first chaining `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 329da35eab3..be7653902a7 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -30,6 +30,7 @@ from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata +from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( @@ -738,6 +739,10 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo size shrinks again the moment the client compacts: pinning the escalated tier would hold the session on the big-window model long after the oversized context that forced it is gone. The gate re-fires per request, so leaving these unpinned costs nothing but the classifier call. + + A modality escalation is transient the same way: it describes what this one call carries (an + image), not what the session's traffic looks like, and pinning it would hold every following + text turn on the vision-capable model the image forced. """ return decision is None or ( decision.get("cause") @@ -745,6 +750,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "default_model_fallback", "plan_mode", "housekeeping", + "modality_escalation", ) and not decision.get("context_escalated") ) @@ -2274,6 +2280,175 @@ class ComplexityRouter(CustomLogger): return pinned_model return self.get_model_for_tier(escalated_tier) + def _model_accepts_image_input(self, model_name: str) -> bool: + """Whether a routed model or pool entry can serve an image request. + + Resolved through the deployments that would actually serve the name; a name with no + deployment on the router is served by the SDK directly and is checked against the model + cost map itself. Only an explicit supports_vision false excludes, a deployment-level + model_info override first and the map otherwise, so unmapped custom names stay routable. + + A multi-deployment group must accept on EVERY deployment: the router picks a deployment + inside the group after this gate runs, so a mixed group marked eligible could still hand + the image to its text-only member and fail with the exact 400 the gate exists to prevent. + """ + from litellm.utils import is_vision_explicitly_disabled + + def deployment_accepts(deployment: Mapping[str, Any]) -> bool: + declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision") + if declared is not None: + return declared is True + litellm_model: Final = (deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name + return not is_vision_explicitly_disabled(litellm_model) + + deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name) + if not deployments: + return not is_vision_explicitly_disabled(model_name) + return all(deployment_accepts(deployment) for deployment in deployments) + + def _modality_eligible_models(self) -> frozenset[str]: + """Every configured pool entry, plus default_model, that can serve an image request.""" + names: Final = frozenset(entry for pool in self._tier_pools().values() for entry in pool) | frozenset( + name for name in (self.config.default_model,) if name + ) + return frozenset(name for name in names if self._model_accepts_image_input(name)) + + async def _gate_response_modality( + self, + response: PreRoutingHookResponse, + messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + resolved_messages: Sequence[Mapping[str, object]] | None, + request_kwargs: dict, # mutable-ok: same shape the hook receives + ) -> PreRoutingHookResponse: + """Replace a routed model that cannot accept this request's image input. + + The single modality owner, applied to the decided response at the hook's exits so every + routing path is covered uniformly. A KEPT session pin is exempt by design (its cause); + replacement picks and every other path are just responses. The re-placement walks + UPWARD-ONLY from the decision's tier (so a plan-mode floor can never be undercut), picks + through `_pick_model_for_tier` so routing plugins still apply, then falls to + default_model (never on plugin routers, and never on a plan-floored decision, since + default_model carries no tier guarantee), else raises the clear 400. The rewritten + decision keeps its cause on a same-tier repick and becomes modality_escalation when the + tier moved or default_model took over, with the displaced placement in signals. + """ + decision: Final = response.routing_decision + if ( + not self.config.modality_routing + or not resolved_messages + or response.model is None + or (decision is not None and decision.get("cause") == "session_affinity_pin") + or not request_contains_image_content(resolved_messages) + or self._model_accepts_image_input(response.model) + ): + return response + eligible: Final = self._modality_eligible_models() + names: Final = self.config.tier_names() + pools: Final = self._tier_pools() + decided: Final = decision.get("tier") if decision is not None else None + start: Final = names.index(decided) if isinstance(decided, str) and decided in names else 0 + capable: Final = next( + (name for name in names[start:] if any(entry in eligible for entry in pools.get(name, ()))), None + ) + if capable is not None: + new_tier: ComplexityTier | str | None = capable if self.config.has_custom_tiers else ComplexityTier(capable) + repick_messages: Final = list(resolved_messages) # mutable-ok: the pick's param is list-typed + new_model = await self._pick_model_for_tier( + new_tier, + messages, + repick_messages, # pyright: ignore[reportArgumentType] # hook-resolved message dicts; the pick only reads them + request_kwargs, + allowed_models=tuple(entry for entry in pools.get(capable, ()) if entry in eligible), + ) + elif self._modality_default_model_usable(request_kwargs, resolved_messages, eligible): + new_tier = None + new_model = self._placed_default_model() + else: + import litellm + + raise litellm.BadRequestError( + message=( + f"Auto-router {self.model_name} received a request with image input, but no model " + f"at or above the decided tier accepts images and modality_routing is enabled. " + f"Tiers checked: {', '.join(names[start:])}. Add a vision-capable model to a tier, " + f"or set a vision-capable default_model, or remove the image content." + ), + model=self.model_name, + llm_provider="", + ) + self._restamp_adaptive_choice(request_kwargs, response.model, new_model) + same_tier: Final = capable is not None and decided == capable + base_cause: Final = (decision.get("cause") if decision is not None else None) or "default_fallback" + displaced_default: Final = decided is None and response.model == self.config.default_model + markers: Final = ( + "modality:image", + *((f"modality_escalated_from:{decided}",) if not same_tier and isinstance(decided, str) else ()), + *(("modality_displaced_default_model",) if not same_tier and displaced_default else ()), + ) + old_signals: Final = tuple(decision.get("signals") or ()) if decision is not None else () + new_decision: Final = self._build_routing_decision( + routed_model=new_model, + cause=base_cause if same_tier else "modality_escalation", + tier=new_tier, + score=decision.get("score") if decision is not None else None, + signals=(*old_signals, *markers), + matched_keyword=decision.get("matched_keyword") if decision is not None else None, + escalation_keyword=decision.get("escalation_keyword") if decision is not None else None, + escalated=bool(decision.get("escalated", False)) if decision is not None else False, + classifier_model=decision.get("classifier_model") if decision is not None else None, + classifier_cost=decision.get("classifier_cost") if decision is not None else None, + conversation_continuing=bool(decision.get("conversation_continuing", True)) + if decision is not None + else True, + tier_litellm_params=self._litellm_params_for_model(new_tier, new_model), + context_escalation_original_tier=( + decision.get("context_escalation_original_tier") if decision is not None else None + ), + ) + from litellm.types.router import PreRoutingHookResponse as HookResponse + + return HookResponse( + model=new_model, + messages=response.messages, + litellm_params=self._litellm_params_for_model(new_tier, new_model), + routing_decision=new_decision, + ) + + def _modality_default_model_usable( + self, + request_kwargs: Mapping[str, object], + resolved_messages: Sequence[Mapping[str, object]] | None, + eligible: frozenset[str], + ) -> bool: + """default_model may serve a gated request only when it is configured, plugin-free + (it is never checked against the plugin pipeline), capability-eligible, and the turn + carries no plan-mode sentinel. The sentinel is re-detected here rather than read off + the decision record, because the record only marks turns the floor RAISED; a sentinel + turn already at or above the floor keeps its ordinary cause, and default_model carries + no tier the floor could vouch for on any sentinel turn.""" + return ( + bool(self.config.default_model) + and not self.config.plugins + and self.config.default_model in eligible + and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None + ) + + def _placed_default_model(self) -> str: + """The default_model behind a usable-default verdict; the raise is the type-level + proof, not a reachable path.""" + model: Final = self.config.default_model + if model is None: + raise ValueError(f"Auto-router {self.model_name}: modality gate routed to an unset default_model") + return model + + @staticmethod + def _restamp_adaptive_choice(request_kwargs: Mapping[str, object], old_model: str, new_model: str) -> None: + """The adaptive feedback loop reads its chosen-model marker from request metadata; a + gate rewrite must move the marker with the model or rewards land on the displaced one.""" + metadata: Final = request_kwargs.get("metadata") + if isinstance(metadata, dict) and metadata.get("adaptive_router_chosen_model") == old_model: + metadata["adaptive_router_chosen_model"] = new_model + def _lexical_tier_override(self, user_message: str) -> KeywordOverride | None: """When keyword_tier_rules match literally, the most-severe matched tier wins. @@ -2655,25 +2830,30 @@ class ComplexityRouter(CustomLogger): session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model) has_original_messages: Final = messages is not None and len(messages) > 0 return self._with_session_deployment_affinity( - PreRoutingHookResponse( - model=routed_model, - messages=messages if has_original_messages else None, - litellm_params=session_tier_litellm_params, - routing_decision=self._build_routing_decision( - routed_model=routed_model, - cause=cause, - tier=routed_pin_tier, - matched_keyword=pin_plan_sentinel if plan_floored else None, - escalation_keyword=pin_escalation_keyword, - escalated=escalated, - conversation_continuing=conversation_continuing, - tier_litellm_params=session_tier_litellm_params, - context_escalation_original_tier=pin_context_original_tier, + await self._gate_response_modality( + PreRoutingHookResponse( + model=routed_model, + messages=messages if has_original_messages else None, + litellm_params=session_tier_litellm_params, + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause=cause, + tier=routed_pin_tier, + matched_keyword=pin_plan_sentinel if plan_floored else None, + escalation_keyword=pin_escalation_keyword, + escalated=escalated, + conversation_continuing=conversation_continuing, + tier_litellm_params=session_tier_litellm_params, + context_escalation_original_tier=pin_context_original_tier, + ), ), + messages, + resolved_messages, + request_kwargs, ) ) - response: Final = await self._classify_and_route( + routed_response: Final = await self._classify_and_route( model=model, request_kwargs=request_kwargs, messages=messages, @@ -2682,6 +2862,11 @@ class ComplexityRouter(CustomLogger): conversation_continuing=conversation_continuing, resolved_messages=resolved_messages, ) + response: Final = ( + await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs) + if routed_response is not None + else None + ) # Sentinel presence, not the plan_mode cause, gates the pin write: a plan-mode turn # classified at or above the floor keeps its ordinary cause, yet on an adaptive router # the hard floor constrained its pick, so pinning it would carry a plan-mode-shaped diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 3c5e8aafa18..70aeecb31c6 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -848,6 +848,18 @@ class ComplexityRouterConfig(BaseModel): "drift plus the response tokens." ), ) + modality_routing: bool = Field( + default=False, + description=( + "Route image-bearing requests only to models that can accept image input. The " + "classifier reads text alone, so an image request whose text classifies cheap " + "otherwise lands on a text-only model and fails with a provider 400. When enabled, " + "a routed model explicitly declared supports_vision false (deployment model_info " + "or the model cost map; unmapped names stay routable) is replaced by the nearest " + "HIGHER tier holding a capable model, then default_model, else a clear 400. A kept " + "session-affinity pin still wins even when an image arrives." + ), + ) # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching semantic_keyword_matching: bool = Field( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a1b3523442b..55a32989b1c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2840,6 +2840,10 @@ RoutingDecisionCause = Literal[ # never called. The matched sentinel rides in matched_keyword. Distinct from the keyword causes, # which are operator-authored rules; these sentinels ship with the router. "housekeeping", + # modality_routing replaced the decided placement: the request carries an image and the + # routed model does not accept image input, so the nearest higher capable tier or + # default_model served instead. The displaced placement rides in signals. + "modality_escalation", "session_affinity_pin", "session_affinity_escalation", # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new diff --git a/litellm/utils.py b/litellm/utils.py index d8b19a406e6..f5adca8f272 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2660,10 +2660,19 @@ def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, ``_supports_factory`` so caching, fallback, and normalisation improvements apply here automatically. """ + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider + try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider - ) + declared: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared is not None: + model = model.removeprefix( + f"{declared}/" + ) # rebind-ok: mirrors get_llm_provider's split without its OAuth flow + custom_llm_provider = declared # rebind-ok: same + else: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) val: Final = model_info.get(key) if val is False: @@ -2751,6 +2760,15 @@ def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> ) +def is_vision_explicitly_disabled(model: str, custom_llm_provider: str | None = None) -> bool: + """True only when supports_vision is explicitly declared false for the model. + + The opt-out mirror of :func:`supports_vision`: a missing declaration reads as not + disabled, so unknown or newly added models stay eligible for image routing. + """ + return _is_explicitly_disabled_factory(model, custom_llm_provider, "supports_vision") + + def supports_vision(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports vision and return a boolean value. diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 772fbf98c57..9ab66d55f3f 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1433,3 +1433,50 @@ class TestFlattenTopLevelSchemaCombinators: flatten_top_level_schema_combinators(schema) assert schema == snapshot + + +class TestRequestContainsImageContent: + """One detector for every dialect that reaches pre-routing hooks untranslated.""" + + @pytest.mark.parametrize( + "part", + [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}, + {"type": "input_image", "image_url": "data:image/png;base64,aGk="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}, + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "image", "source": {"type": "base64", "data": "aGk="}}], + }, + ], + ) + def test_detects_every_image_dialect_including_tool_results(self, part): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}, part]}] + assert request_contains_image_content(messages) is True + + @pytest.mark.parametrize( + "messages", + [ + [{"role": "user", "content": "plain string"}], + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "x"}}]}], + [{"role": "user", "content": [{"type": "tool_result", "content": [{"type": "text", "text": "ok"}]}]}], + [{"role": "user", "content": None}], + [], + ], + ) + def test_ignores_text_audio_and_degenerate_shapes(self, messages): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + assert request_contains_image_content(messages) is False + + def test_hostile_nesting_is_depth_bounded(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + nested: dict = {"type": "image", "source": {"type": "base64", "data": "aGk="}} + for _ in range(50): + nested = {"type": "tool_result", "content": [nested]} + assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3f7844cffba..1ec8be88c9b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -10417,3 +10417,323 @@ class TestContextWindowEscalation: assert oversized["model_name"] == "big-model" assert small["model_name"] == "small-model" + + +IMG_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}} +PLAN_BODY = { + "messages": [{"role": "system", "content": [{"type": "text", "text": "Plan mode is active. Do not execute."}]}] +} + + +class TestModalityRouting: + """modality_routing: the response gate replaces a routed model that cannot take images.""" + + IMAGE_MESSAGE = [{"role": "user", "content": [{"type": "text", "text": "What color is this?"}, IMG_PART]}] + BASE_TIERS = {"SIMPLE": "text-cheap", "MEDIUM": "vision-mid", "COMPLEX": "vision-big"} + BASE_VISION = {"text-cheap": False, "vision-mid": True, "vision-big": True, "vision-default": True} + + @staticmethod + def _router(mock_router_instance, config, vision_by_model): + """vision_by_model: model name -> True/False (deployment model_info) or None (undeclared).""" + + def get_model_list(model_name=None): + if model_name not in vision_by_model: + return [] + declared = vision_by_model[model_name] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}"}, + "model_info": {} if declared is None else {"supports_vision": declared}, + } + ] + + mock_router_instance.get_model_list = get_model_list + return ComplexityRouter( + model_name="modality-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "config_extra, vision, send_image, expected_model, expect_marker", + [ + ({}, {"text-cheap": False}, True, "text-cheap", False), + ({"modality_routing": True}, {"text-cheap": False}, False, "text-cheap", False), + ({"modality_routing": True}, {"text-cheap": None}, True, "text-cheap", False), + ], + ids=["flag_off", "no_image", "undeclared_model_stays_routable"], + ) + async def test_gate_leaves_ungated_requests_untouched( + self, mock_router_instance, config_extra, vision, send_image, expected_model, expect_marker + ): + router = self._router(mock_router_instance, {"tiers": dict(self.BASE_TIERS), **config_extra}, vision) + request = self.IMAGE_MESSAGE if send_image else [{"role": "user", "content": "What color is the sky?"}] + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=request) + assert result.model == expected_model + assert result.routing_decision["cause"] == "heuristic_scorer" + assert ("modality:image" in (result.routing_decision.get("signals") or ())) is expect_marker + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "part", + [ + IMG_PART, + {"type": "input_image", "image_url": "data:image/png;base64,aGk="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}, + {"type": "tool_result", "tool_use_id": "tu_1", "content": [dict(IMG_PART, type="image")]}, + ], + ids=["image_url", "input_image", "anthropic_image", "tool_result_nested"], + ) + async def test_every_image_dialect_escalates(self, mock_router_instance, part): + router = self._router( + mock_router_instance, {"tiers": dict(self.BASE_TIERS), "modality_routing": True}, dict(self.BASE_VISION) + ) + message = [{"role": "user", "content": [{"type": "text", "text": "What color is this?"}, part]}] + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=message) + assert result.model == "vision-mid" + assert result.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in result.routing_decision["signals"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "path, expected_model, expected_cause", + [ + ("classifier_escalates", "vision-mid", "modality_escalation"), + ("same_tier_repick_keeps_cause", "vision-cheap", "heuristic_scorer"), + ("keyword_tier_escalates", "vision-mid", "modality_escalation"), + ("no_ask_capable_default_kept", "vision-default", "default_fallback"), + ("no_ask_text_default_displaced", "vision-mid", "modality_escalation"), + ("custom_tiers_walk", "premium-model", "modality_escalation"), + ("pin_kept_bypasses", "text-cheap", "session_affinity_pin"), + ("pin_replacement_gated", "vision-big", "modality_escalation"), + ("adaptive_pick_rewritten", "vision-mid", "modality_escalation"), + ], + ) + async def test_placements_across_decision_paths(self, mock_router_instance, path, expected_model, expected_cause): + config = {"tiers": dict(self.BASE_TIERS), "modality_routing": True} + vision = dict(self.BASE_VISION) + request_kwargs = {} + messages = self.IMAGE_MESSAGE + if path == "same_tier_repick_keeps_cause": + config["tiers"]["SIMPLE"] = ["text-cheap", "vision-cheap"] + vision["vision-cheap"] = True + with patch( # test-quality-ok: the mixed-pool repick is unreachable deterministically without pinning the first random pick + "litellm.router_strategy.complexity_router.complexity_router.random.choice", + side_effect=lambda pool: sorted(pool)[0], + ): + router = self._router(mock_router_instance, config, vision) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=messages) + assert result.model == expected_model + assert result.routing_decision["cause"] == expected_cause + assert result.routing_decision["signals"][-1] == "modality:image" + return + if path == "keyword_tier_escalates": + config["keyword_tier_rules"] = [{"keywords": ["quick lookup"], "tier": "SIMPLE"}] + messages = [ + {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} + ] + elif path == "no_ask_capable_default_kept": + config["default_model"] = "vision-default" + messages = [{"role": "user", "content": [IMG_PART]}] + elif path == "no_ask_text_default_displaced": + config["default_model"] = "text-default" + vision["text-default"] = False + messages = [{"role": "user", "content": [IMG_PART]}] + elif path == "custom_tiers_walk": + config = { + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-4o-mini"}, + "fallback_tier": "cheap", + "tier_definitions": [ + {"name": "cheap", "description": "trivial asks"}, + {"name": "premium", "description": "hard asks"}, + ], + "tiers": {"cheap": "cheap-model", "premium": "premium-model"}, + "keyword_tier_rules": [{"keywords": ["quick lookup"], "tier": "cheap"}], + "modality_routing": True, + } + vision = {"cheap-model": False, "premium-model": True} + messages = [ + {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} + ] + elif path in ("pin_kept_bypasses", "pin_replacement_gated"): + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + config["session_affinity"] = True + request_kwargs = {"metadata": {"session_id": "s1"}} + if path == "pin_replacement_gated": + config["tiers"]["MEDIUM"] = "text-mid" + vision["text-mid"] = False + messages = [ + {"role": "user", "content": [{"type": "text", "text": "LITELLM ESCALATE describe this"}, IMG_PART]} + ] + elif path == "adaptive_pick_rewritten": + config["adaptive"] = True + mock_router_instance.model_list = [] + mock_router_instance.model_name_to_deployment_indices = {} + router = self._router(mock_router_instance, config, vision) + result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) + assert result.model == expected_model + assert result.routing_decision["cause"] == expected_cause + if path == "adaptive_pick_rewritten": + assert request_kwargs["metadata"]["adaptive_router_chosen_model"] == expected_model + + @pytest.mark.asyncio + async def test_plan_floored_decision_never_falls_to_default_model(self, mock_router_instance): + """An upward-only walk cannot undercut the floor; default_model must not either.""" + config = { + "tiers": {"SIMPLE": "vision-cheap", "MEDIUM": "text-mid"}, + "default_model": "vision-default", + "plan_mode_min_tier": "MEDIUM", + "modality_routing": True, + } + vision = {"vision-cheap": True, "text-mid": False, "vision-default": True} + router = self._router(mock_router_instance, config, vision) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", + request_kwargs={"proxy_server_request": {"body": PLAN_BODY}}, + messages=[{"role": "user", "content": [{"type": "text", "text": "plan this"}, IMG_PART]}], + ) + + @pytest.mark.asyncio + async def test_at_floor_plan_turn_never_falls_to_default_model(self, mock_router_instance): + """A sentinel turn whose classified tier already satisfies the floor keeps its ordinary + cause, so the record carries no floor marker; the default arm must still refuse it.""" + config = { + "tiers": {"SIMPLE": "text-a", "MEDIUM": "text-b"}, + "default_model": "vision-default", + "plan_mode_min_tier": "SIMPLE", + "modality_routing": True, + } + vision = {"text-a": False, "text-b": False, "vision-default": True} + router = self._router(mock_router_instance, config, vision) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", + request_kwargs={"proxy_server_request": {"body": PLAN_BODY}}, + messages=[{"role": "user", "content": [{"type": "text", "text": "plan this"}, IMG_PART]}], + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "default_model, default_vision, expect_error", + [(None, None, True), ("text-default", False, True), ("vision-default", True, False)], + ids=["no_default", "text_only_default", "vision_default_serves"], + ) + async def test_no_capable_tier_above_uses_default_or_rejects( + self, mock_router_instance, default_model, default_vision, expect_error + ): + config = {"tiers": {"SIMPLE": "text-cheap", "COMPLEX": "text-big"}, "modality_routing": True} + vision = {"text-cheap": False, "text-big": False} + if default_model is not None: + config["default_model"] = default_model + vision[default_model] = default_vision + router = self._router(mock_router_instance, config, vision) + if expect_error: + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + return + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + assert result.model == "vision-default" + assert result.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in result.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_mixed_deployment_group_is_treated_text_only(self, mock_router_instance): + def get_model_list(model_name=None): + declared = {"mixed-group": [True, False], "vision-big": [True]}.get(model_name) + if declared is None: + return [] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}-{i}"}, + "model_info": {"supports_vision": accepts}, + } + for i, accepts in enumerate(declared) + ] + + mock_router_instance.get_model_list = get_model_list + router = ComplexityRouter( + model_name="modality-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "mixed-group", "COMPLEX": "vision-big"}, + "modality_routing": True, + }, + ) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + assert result.model == "vision-big" + assert result.routing_decision["cause"] == "modality_escalation" + + @pytest.mark.asyncio + async def test_continuation_turn_screenshot_escalates_past_the_held_model(self, mock_router_instance): + """classification_mode user_turn replays the held model on continuation turns; a + continuation carrying a screenshot must still be re-placed when that model is text-only.""" + mock_router_instance.cache = DualCache() + config = { + "tiers": dict(self.BASE_TIERS), + "classification_mode": "user_turn", + "modality_routing": True, + } + router = self._router(mock_router_instance, config, dict(self.BASE_VISION)) + first = await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": "cont-1"}}, + messages=[{"role": "user", "content": "hi there"}], + ) + assert first.model == "text-cheap" + continuation = [ + {"role": "user", "content": "hi there"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_1", "name": "screenshot", "input": {}}]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "image", "source": {"type": "base64", "data": "aGk="}}], + } + ], + }, + ] + second = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "cont-1"}}, messages=continuation + ) + assert second.model == "vision-mid" + assert second.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in second.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_rewrite_carries_the_context_escalation_record(self, mock_router_instance): + """A context-window escalation and a modality re-place are separate facts on one + record; rewriting for the image must not drop the sibling gate's fields.""" + from litellm.types.router import PreRoutingHookResponse + + router = self._router( + mock_router_instance, + {"tiers": dict(self.BASE_TIERS), "modality_routing": True}, + dict(self.BASE_VISION), + ) + decision = router._build_routing_decision( + routed_model="text-cheap", + cause="heuristic_scorer", + tier=ComplexityTier.SIMPLE, + context_escalation_original_tier=ComplexityTier.SIMPLE, + ) + response = PreRoutingHookResponse(model="text-cheap", messages=None, routing_decision=decision) + rewritten = await router._gate_response_modality(response, None, self.IMAGE_MESSAGE, {}) + assert rewritten.model == "vision-mid" + assert rewritten.routing_decision["cause"] == "modality_escalation" + assert rewritten.routing_decision["context_escalated"] is True + assert rewritten.routing_decision["context_escalation_original_tier"] == "SIMPLE" + + def test_modality_escalation_is_never_pinnable(self): + from litellm.router_strategy.complexity_router.complexity_router import _decision_is_pinnable + + assert _decision_is_pinnable({"cause": "modality_escalation"}) is False + assert _decision_is_pinnable({"cause": "heuristic_scorer"}) is True diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6524353aa48..a7fa6ac5568 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -5765,3 +5765,33 @@ class TestHuggingFaceConfigFetch: assert _get_max_position_embeddings("some-org/some-model") == 512 request_timeout = hf_config_route.calls.last.request.extensions["timeout"] assert request_timeout["read"] == HF_CONFIG_FETCH_TIMEOUT_SECONDS + + +class TestIsVisionExplicitlyDisabled: + """github_copilot and chatgpt run an OAuth device flow inside get_llm_provider; the + explicit-disable lookup must adopt the declared prefix instead of resolving it, exactly + as _supports_factory does, or a capability check on a copilot deployment blocks routing + on a device-code prompt.""" + + @pytest.mark.parametrize("model", ["github_copilot/gpt-4o", "chatgpt/gpt-5"]) + def test_never_resolves_an_authenticating_prefix(self, model, monkeypatch): + from litellm.utils import is_vision_explicitly_disabled + + lookups: list = [] + + def _record(*args, **kwargs): + lookups.append((args, kwargs)) + raise RuntimeError("provider resolution must not run for an authenticating provider") + + monkeypatch.setattr(litellm, "get_llm_provider", _record) + + assert is_vision_explicitly_disabled(model) is False + assert lookups == [] + + def test_explicit_false_detected_and_absent_reads_enabled(self): + from litellm.utils import is_vision_explicitly_disabled + + assert ( + is_vision_explicitly_disabled("fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731") is True + ) + assert is_vision_explicitly_disabled("anthropic/claude-sonnet-4-5") is False diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index e084fdf37e6..bf6a20a4af8 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -186,6 +186,12 @@ describe("RoutingDecisionCard", () => { expect(screen.queryByText("housekeeping")).not.toBeInTheDocument(); }); + it("labels a modality escalation instead of showing the raw cause token", () => { + render(); + expect(screen.getByText("Escalated for image input")).toBeInTheDocument(); + expect(screen.queryByText("modality_escalation")).not.toBeInTheDocument(); + }); + it("shows the escalation keyword", () => { render( , diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index d2aa20901f5..aa1d45a859e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -90,6 +90,7 @@ const CONSTANT_CAUSE_LABELS: Record = { session_affinity_pin: "Pinned to session", session_affinity_escalation: "Escalated from session pin", user_turn_continuation: "Continuation turn, classifier skipped", + modality_escalation: "Escalated for image input", quality_tier: "Quality tier mapping", bandit: "Adaptive bandit", default_fallback: "Default model, no route matched", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c20545f6fb2..fb81191cfbf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34463,6 +34463,12 @@ export interface components { * @default 0.5 */ match_threshold: number; + /** + * Modality Routing + * @description Route image-bearing requests only to models that can accept image input. The classifier reads text alone, so an image request whose text classifies cheap otherwise lands on a text-only model and fails with a provider 400. When enabled, a routed model explicitly declared supports_vision false (deployment model_info or the model cost map; unmapped names stay routable) is replaced by the nearest HIGHER tier holding a capable model, then default_model, else a clear 400. A kept session-affinity pin still wins even when an image arrives. + * @default false + */ + modality_routing: boolean; /** * Plan Mode Min Tier * @description When set, requests carrying a coding-agent plan-mode sentinel (Claude Code plan mode, VS Code Copilot Plan mode, Copilot CLI's exit_plan_mode tool) are routed to at least this tier: the classified tier still wins when it is higher, and the floor also overrides a session-affinity pin to a lower tier for exactly the turns carrying the sentinel, without rewriting the pin -- the first turn after plan mode exits routes as if plan mode had never happened. Names a built-in tier, or with tier_definitions set, one of the defined tier names (list order is ascending severity, same as keyword_tier_rules). Unset disables detection entirely. The sentinels ride in client-injected prompt text, so a caller who pastes one can spend up to this tier's models -- never down, and never outside the configured pools. @@ -35620,7 +35626,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */ From 0565d33fa50b849c27edace7a413aec5d2611ed6 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 31 Aug 2026 19:55:16 -0700 Subject: [PATCH 25/34] fix(ui): let the auto-router scoring tier list follow the theme (#39040) * fix(ui): let the auto-router scoring tier list follow the theme * test(ui): assert the tier list carries the muted-foreground token --- .../components/add_model/ClassificationMethodConfig.tsx | 2 +- .../components/add_model/ComplexityRouterConfig.test.tsx | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 2947e29319b..84f61c95eca 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -111,7 +111,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = How Classification Works {scoringExplanation(value)} {scorerRuns && ranges && ( -
    +
    • {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium}
      • diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 33ce1169c46..218c99e32c5 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -80,6 +80,15 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); }); + it("leaves the score threshold list color to the theme instead of an inline style", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + const list = screen.getByText(/Score < 0.15/).closest("ul"); + expect(list).toBeInTheDocument(); + expect(list).toHaveClass("text-muted-foreground"); + expect(list?.style.color).toBe(""); + }); + it("should default to heuristic and hide classifier model/timeout fields", () => { renderWithProviders(); expect(screen.getByText("Advanced: Classification Method")).toBeInTheDocument(); From 502b3a2f794a6e806835e41ffa8e5fc951a7fda9 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 31 Aug 2026 20:02:08 -0700 Subject: [PATCH 26/34] feat(ui): auto-router controls for context-window escalation (#39054) * feat(ui): auto-router controls for context-window escalation Adds an Advanced: Context Window Escalation section to the auto-router form, both create and edit arms, with the toggle for enable_context_window_escalation and a clamped decimal input for context_window_escalation_buffer. An untouched control keeps both keys out of the payload so the router tracks the backend defaults; an explicit opt-out (false) survives the edit round-trip through the managed-keys projection and the hydrator, and preset prefill maps both keys straight through so a preset cannot silently drop them Resolves LIT-6601 * fix(ui): clearing the context-window buffer removes it from the payload Both review bots converged on the same defect: an emptied buffer field early-returned in commitBuffer, the draft was discarded on blur, and the stale number reappeared and stayed in the saved config, contradicting the copy that an empty field tracks the backend default. An empty commit now removes the key, which the managed-keys projection propagates as a real deletion on edit. Also trims the narrative comments the review flagged as restating behavior --- .../add_model/ComplexityRouterConfig.tsx | 13 ++++ .../ContextWindowEscalationConfig.tsx | 60 +++++++++++++++++ .../add_model/add_auto_router_tab.test.tsx | 65 +++++++++++++++++++ .../add_model/add_auto_router_tab.tsx | 2 + .../build_complexity_router_config.test.ts | 10 +++ .../build_complexity_router_config.ts | 12 ++++ ...d_updated_complexity_router_config.test.ts | 2 + .../edit_auto_router_modal.tsx | 14 ++++ .../src/lib/autorouter_presets.test.ts | 16 +++++ .../src/lib/autorouter_presets.ts | 2 + 10 files changed, 196 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 153afa0b586..111a7c9f10a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -29,6 +29,7 @@ import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; import { Restricted, restrictedBy } from "./TierRestrictions"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { @@ -392,6 +393,13 @@ export interface ComplexityRouterConfigValue { tier_distance_penalty?: number; adaptive_eligible?: AdaptiveEligible; return_raw_model_name?: boolean; + /** + * Context-window escalation gate. Undefined means untouched, which keeps both keys out of the + * payload so the router tracks the backend defaults (enabled, 0.95 buffer); an explicit false + * is a real opt-out and must survive the edit round-trip. + */ + enable_context_window_escalation?: boolean; + context_window_escalation_buffer?: number; /** * Heuristic scorer knobs. Undefined means the operator never touched them, which keeps the key out of the * payload so the router tracks the backend defaults rather than freezing today's numbers. @@ -827,6 +835,11 @@ const ComplexityRouterConfig: React.FC = ({ ), }, + { + key: "context-window", + label: Advanced: Context Window Escalation, + children: , + }, { key: "response", label: Advanced: Response Format, diff --git a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx new file mode 100644 index 00000000000..c0a65076d20 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx @@ -0,0 +1,60 @@ +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import React from "react"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const ContextWindowEscalationConfig: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => { + const enabled = value.enable_context_window_escalation ?? true; + // A number input renders Number("0.") as "0", so a decimal cannot be typed without a local draft. + const [bufferDraft, setBufferDraft] = React.useState(null); + const commitBuffer = (raw: string) => { + setBufferDraft(null); + if (raw.trim() === "") { + onChange({ ...value, context_window_escalation_buffer: undefined }); + return; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return; + onChange({ ...value, context_window_escalation_buffer: Math.min(1, Math.max(0.01, parsed)) }); + }; + return ( + <> +
      + onChange({ ...value, enable_context_window_escalation: next })} + aria-label="Escalate oversized prompts to a tier that fits" + /> + Escalate oversized prompts to a tier that fits +
      + + When a prompt provably cannot fit the decided tier's context windows, route it to the lowest tier whose + window holds it instead of letting the provider reject it. Off means requests dispatch on complexity alone. + + {enabled && ( +
      + + setBufferDraft(event.target.value)} + onBlur={(event) => commitBuffer(event.target.value)} + /> + + Fraction of a model's window the counted prompt must fit within, above 0 up to 1. Empty tracks the + backend default of 0.95. + +
      + )} + + ); +}; + +export default ContextWindowEscalationConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 01cb41bcb95..71b454dbb06 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -373,6 +373,71 @@ describe("AddAutoRouterTab", () => { }); }); + it("carries a context-window escalation opt-out through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-window-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); + expect(toggle).toBeChecked(); + await user.click(toggle); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + enable_context_window_escalation: false, + }); + }); + + it("clamps the context-window buffer to 1 and keeps an untouched buffer out of the payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const buffer = await screen.findByLabelText("Window fit buffer"); + fireEvent.change(buffer, { target: { value: "1.5" } }); + fireEvent.blur(buffer, { target: { value: "1.5" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; + expect(config).toMatchObject({ context_window_escalation_buffer: 1 }); + expect(config).not.toHaveProperty("enable_context_window_escalation"); + }); + + it("clearing the buffer removes it from the payload so the router tracks the backend default", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const buffer = await screen.findByLabelText("Window fit buffer"); + fireEvent.change(buffer, { target: { value: "0.8" } }); + fireEvent.blur(buffer, { target: { value: "0.8" } }); + fireEvent.change(buffer, { target: { value: "" } }); + fireEvent.blur(buffer, { target: { value: "" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty( + "context_window_escalation_buffer", + ); + }); + // The scalar floor is the one scorer knob with no group dict behind it, so its wiring into the create // payload is only proven end to end. 0 is the case a truthy check would silently drop. it("carries a reasoning override floor of 0 through to the create payload", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 4e5e5e8d460..ed584a4882b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -367,6 +367,8 @@ const AddAutoRouterTab: React.FC = ({ tokenThresholds: complexityRouterConfig.token_thresholds, dimensionWeights: complexityRouterConfig.dimension_weights, reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score, + enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation, + contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer, }; const submitRecommendedRouter = async (name: string) => { diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 84406a2093e..feddaaa0eac 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -59,6 +59,16 @@ describe("buildComplexityRouterConfig", () => { expect(config).toEqual(expected); }); + it("carries an explicit context-window escalation opt-out and buffer, false included", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + enableContextWindowEscalation: false, + contextWindowEscalationBuffer: 0.9, + }); + expect(config.enable_context_window_escalation).toBe(false); + expect(config.context_window_escalation_buffer).toBe(0.9); + }); + it("trims escalation keywords and drops blank entries", () => { const config = buildComplexityRouterConfig({ ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index ba500d116ce..9a14e956207 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -123,6 +123,8 @@ export interface BuildComplexityRouterConfigParams { dimensionWeights?: DimensionWeights; reasoningOverrideMinScore?: number; tierModelParams?: TierModelParamsByTier; + enableContextWindowEscalation?: boolean; + contextWindowEscalationBuffer?: number; } /** @@ -174,6 +176,8 @@ export interface ComplexityRouterConfigPayload { token_thresholds?: TokenThresholds; dimension_weights?: DimensionWeights; reasoning_override_min_score?: number; + enable_context_window_escalation?: boolean; + context_window_escalation_buffer?: number; tier_model_configs?: Record; } @@ -407,6 +411,8 @@ export const buildComplexityRouterConfig = ({ dimensionWeights, reasoningOverrideMinScore, tierModelParams, + enableContextWindowEscalation, + contextWindowEscalationBuffer, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const serializedTierModelConfigs = customTierSet ? serializeTierModelConfigs( @@ -463,6 +469,12 @@ export const buildComplexityRouterConfig = ({ adaptive_eligible: adaptiveEligible, }), ...(returnRawModelName && { return_raw_model_name: true }), + ...(enableContextWindowEscalation !== undefined && { + enable_context_window_escalation: enableContextWindowEscalation, + }), + ...(contextWindowEscalationBuffer !== undefined && { + context_window_escalation_buffer: contextWindowEscalationBuffer, + }), ...scorerKnobs, }; if (!customTierSet) return payload; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index a7bd4b8eab4..d8c2987ead5 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -487,6 +487,8 @@ describe("managed keys survive an untouched open-and-save", () => { token_thresholds: { simple: 20, complex: 500 }, dimension_weights: { tokenCount: 0.1 }, reasoning_override_min_score: 0.3, + enable_context_window_escalation: false, + context_window_escalation_buffer: 0.9, }; // tier_definitions, fallback_tier and classification_prompt cannot sit beside heuristic_first, which diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 425d5d51f06..4751c2e64b7 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -107,6 +107,8 @@ export interface StoredComplexityRouterConfig { tier_distance_penalty?: number; adaptive_eligible?: AdaptiveEligible; return_raw_model_name?: boolean; + enable_context_window_escalation?: unknown; + context_window_escalation_buffer?: unknown; } /** @@ -178,6 +180,14 @@ export const hydrateComplexityRouterConfig = ( tier_distance_penalty: parsedConfig.tier_distance_penalty, adaptive_eligible: parsedConfig.adaptive_eligible || "all", return_raw_model_name: parsedConfig.return_raw_model_name || false, + enable_context_window_escalation: + typeof parsedConfig.enable_context_window_escalation === "boolean" + ? parsedConfig.enable_context_window_escalation + : undefined, + context_window_escalation_buffer: + typeof parsedConfig.context_window_escalation_buffer === "number" + ? parsedConfig.context_window_escalation_buffer + : undefined, }; }; @@ -208,6 +218,8 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "token_thresholds", "dimension_weights", "reasoning_override_min_score", + "enable_context_window_escalation", + "context_window_escalation_buffer", ]); // Managed only when the caller passes the corresponding state. A caller that does not render @@ -307,6 +319,8 @@ export const buildUpdatedComplexityRouterConfig = ( dimensionWeights: value.dimension_weights, reasoningOverrideMinScore: value.reasoning_override_min_score, tierModelParams: value.tier_model_params, + enableContextWindowEscalation: value.enable_context_window_escalation, + contextWindowEscalationBuffer: value.context_window_escalation_buffer, }; const built = buildComplexityRouterConfig(builderParams); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 2de1ac11db2..f14d6279e32 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -573,6 +573,22 @@ describe("autorouter_presets", () => { expect(prefill.escalationKeywords).toEqual([]); }); + it("carries a preset's context-window escalation opt-out and buffer through the prefill", () => { + const prefill = buildPresetPrefill( + { + tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic", + session_affinity: false, + deployment_affinity: true, + enable_context_window_escalation: false, + context_window_escalation_buffer: 0.9, + }, + groupsOnly(["gpt-5-nano"]), + ); + expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(false); + expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); + }); + it("falls back to the defaults when a preset omits match_threshold and escalation_keywords", () => { const prefill = buildPresetPrefill( { diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index a35b868db5e..f96dd5ddb4c 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -295,6 +295,8 @@ export const buildPresetPrefill = ( tier_distance_penalty: config.tier_distance_penalty, adaptive_eligible: config.adaptive_eligible, return_raw_model_name: config.return_raw_model_name, + enable_context_window_escalation: config.enable_context_window_escalation, + context_window_escalation_buffer: config.context_window_escalation_buffer, }, customTechnicalKeywords: config.custom_technical_keywords ?? [], keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []), From d9c43d5e17746b11414c8aa17755b1e11bec16f6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 31 Aug 2026 20:07:28 -0700 Subject: [PATCH 27/34] fix(key management): read budget window usage from the window spend table Pass window_duration to get_current_spend so /key/info re-checks a stale-low counter against the LiteLLM_BudgetWindowSpend row instead of aggregating LiteLLM_SpendLogs, and reuse _budget_limit_windows for the stored-column coercion. Drop the /v2/key/info batch cap (a new 422 for callers that work today) and the unrelated CI timeout bump and soft_budget test --- .../key_management_endpoints.py | 103 +++--------- .../test_key_management_endpoints.py | 156 +++--------------- 2 files changed, 51 insertions(+), 208 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 03abcb2ff98..02bfa5c0ca8 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3579,95 +3579,58 @@ async def _build_model_max_budget_usage( ) -def _budget_window_to_dict(window: object) -> Mapping[str, object] | None: - """Coerce a budget_limits entry to a dict; None when the entry is unusable.""" - if isinstance(window, dict): - return window - model_dump: Final = getattr(window, "model_dump", None) - if not callable(model_dump): +def _window_max_budget(window: Mapping[str, object]) -> float | None: + """A window's max_budget as a float; None when absent or unparseable.""" + value: Final = window.get("max_budget") + if not isinstance(value, (int, float, str)): return None try: - dumped: Final = model_dump() - except Exception: # noqa: BLE001 # model_dump implementations can raise arbitrary errors + return float(value) + except ValueError: return None - return dumped if isinstance(dumped, dict) else None - - -def _coerce_budget_limits(budget_limits: object) -> Sequence[object] | None: - """Coerce budget_limits to a sequence of windows, parsing JSON strings; None when unusable.""" - if isinstance(budget_limits, str): - try: - parsed: Final = json.loads(budget_limits) - except (TypeError, ValueError): - return None - return parsed if isinstance(parsed, list) else None - return budget_limits if isinstance(budget_limits, list) else None - - -def _parse_window_max_budget(value: object) -> float | None: - """Coerce a window's max_budget to float; None when absent or unparseable.""" - if isinstance(value, (int, float, str)): - try: - return float(value) - except (TypeError, ValueError): - return None - return None async def _budget_window_with_usage(window: Mapping[str, object], api_key_hash: str) -> Mapping[str, object]: """ - Return a copy of a budget window with current-window spend attached. + Copy of a budget window with current-window spend attached. - Per-window spend is not persisted in the DB; it lives in the cross-pod spend - counters (spend:key:{hashed_token}:window:{budget_duration}) that - _virtual_key_multi_budget_check enforces against, so we read the same - counters via get_current_spend. Passing max_budget + window_start makes the - read re-check against the authoritative spend-log aggregate when the counter - is stale-low (e.g. after a Redis flush), same as the enforcement path. + Reads the same cross-pod counter (spend:key:{hashed_token}:window:{budget_duration}) + that _virtual_key_multi_budget_check enforces against, passing the same + window_duration + window_start so a stale-low counter is re-checked against + the LiteLLM_BudgetWindowSpend row instead of a spend-log aggregate. """ from litellm.proxy.proxy_server import get_current_spend duration: Final = window.get("budget_duration") - if not duration: - return dict(window) # mutable-ok: per-window response copy, built once per window + if not isinstance(duration, str) or not duration: + return window spend: Final = await get_current_spend( counter_key=f"spend:key:{api_key_hash}:window:{duration}", fallback_spend=0.0, - max_budget=_parse_window_max_budget(window.get("max_budget")), + max_budget=_window_max_budget(window), window_entity_type="Key", window_entity_id=api_key_hash, + window_duration=duration, window_start=get_budget_window_start(window), ) return {**window, "current_spend": round(spend, 4)} # mutable-ok: per-window response copy, built once per window -async def _budget_limits_entry_with_usage(window: object, api_key_hash: str) -> object: - """Return the window as an enriched dict when dict-coercible; the original entry otherwise.""" - coerced: Final = _budget_window_to_dict(window) - if not coerced: - return window - return await _budget_window_with_usage(window=coerced, api_key_hash=api_key_hash) - - -async def _budget_limits_with_usage(budget_limits: object, api_key_hash: str) -> Sequence[object] | None: +async def _budget_limits_with_usage( + budget_limits: Sequence[object] | str | None, api_key_hash: str +) -> tuple[Mapping[str, object], ...] | None: """ - Return budget_limits as window dicts with current-window spend attached. - - None when budget_limits is not a usable (possibly JSON-encoded) list; the - caller keeps the original value then. Entries that are not dict-coercible - are preserved as-is. + budget_limits as window dicts with current-window spend attached; None when + the key has no windows so the caller keeps the stored value. """ - windows: Final = _coerce_budget_limits(budget_limits) - if windows is None: + windows: Final = _budget_limit_windows(budget_limits) + if not windows: return None - return [ # mutable-ok: entries are awaited, so they cannot be built inside a frozen wrapper - await _budget_limits_entry_with_usage(window=window, api_key_hash=api_key_hash) for window in windows - ] - - -# Caps per-request fan-out: each key with budget windows costs one spend-counter -# read (worst case a SpendLogs aggregation) per window. -MAX_KEY_INFO_KEYS_PER_REQUEST: Final = 100 + return tuple( + await asyncio.gather( + *(_budget_window_with_usage(window=window, api_key_hash=api_key_hash) for window in windows) + ) + ) @router.post( @@ -3712,18 +3675,6 @@ async def info_key_fn_v2( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={"message": "Malformed request. No keys passed in."}, ) - requested_key_count: Final = len(data.keys or ()) + len(data.key_aliases or ()) - if requested_key_count > MAX_KEY_INFO_KEYS_PER_REQUEST: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail={ # mutable-ok: one-shot HTTPException payload matching the sibling detail dict above; never mutated after construction - "message": ( - f"Too many keys requested: {requested_key_count}. " - f"At most {MAX_KEY_INFO_KEYS_PER_REQUEST} keys and key_aliases combined per request." - ) - }, - ) - # Resolve key_aliases to tokens so we never pass token=None (unbounded query) tokens_to_query: Final = list(data.keys) if data.keys else [] if data.key_aliases: diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index a2f248ca55c..425365c8560 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -539,57 +539,6 @@ async def test_key_generation_with_object_permission(monkeypatch): assert key_insert_calls[0]["data"].get("object_permission_id") == "objperm123" -@pytest.mark.asyncio -async def test_generate_key_with_soft_budget_creates_budget_row(monkeypatch): - """soft_budget on /key/generate must create a budget table row and link its budget_id to the key.""" - mock_prisma_client = AsyncMock() - mock_prisma_client.jsonify_object = lambda data: data - mock_prisma_client.db = MagicMock() - mock_budget_create = AsyncMock(return_value=MagicMock(budget_id="budget-soft-123")) - mock_prisma_client.db.litellm_budgettable = MagicMock() - mock_prisma_client.db.litellm_budgettable.create = mock_budget_create - - async def _insert_data_side_effect(*args, **kwargs): - if kwargs.get("table_name") == "user": - return MagicMock(models=[], spend=0) - return MagicMock( - token="hashed_token_soft", - litellm_budget_table=None, - object_permission=None, - ) - - mock_prisma_client.insert_data = AsyncMock(side_effect=_insert_data_side_effect) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - - from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles - from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth - from litellm.proxy.management_endpoints.key_management_endpoints import ( - generate_key_fn, - ) - - await generate_key_fn( - data=GenerateKeyRequest(soft_budget=5.0), - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-1234", - user_id="admin-1", - ), - ) - - mock_budget_create.assert_awaited_once() - created_budget = mock_budget_create.call_args.kwargs["data"] - assert created_budget["soft_budget"] == 5.0 - assert created_budget["created_by"] == "admin-1" - - key_insert_calls = [ - call.kwargs - for call in mock_prisma_client.insert_data.call_args_list - if call.kwargs.get("table_name") == "key" - ] - assert len(key_insert_calls) == 1 - assert key_insert_calls[0]["data"].get("budget_id") == "budget-soft-123" - - @pytest.mark.asyncio async def test_generate_key_debug_log_never_contains_raw_token(monkeypatch, caplog): """Regression for LIT-4356: /key/generate must never emit the raw virtual key @@ -14267,6 +14216,7 @@ async def test_info_key_fn_budget_limits_includes_current_spend(monkeypatch): assert call_kwargs["max_budget"] == 2.0 assert call_kwargs["window_entity_type"] == "Key" assert call_kwargs["window_entity_id"] == test_key_token + assert call_kwargs["window_duration"] == "1h" assert call_kwargs["window_start"] is not None @@ -14397,39 +14347,9 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): f"spend:key:{test_key_token}:window:1h", f"spend:key:{test_key_token}:window:1d", } - - -@pytest.mark.asyncio -async def test_info_key_fn_v2_rejects_oversized_batch(monkeypatch): - """/v2/key/info must reject over-cap batches before doing any DB work.""" - from unittest.mock import AsyncMock - - from litellm.proxy._types import KeyRequest, ProxyException - from litellm.proxy.management_endpoints.key_management_endpoints import ( - MAX_KEY_INFO_KEYS_PER_REQUEST, - info_key_fn_v2, - ) - - mock_prisma_client = AsyncMock() - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", AsyncMock()) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin-batch-cap", - ) - - with pytest.raises(ProxyException) as exc_info: - await info_key_fn_v2( - data=KeyRequest( - keys=[f"hash-{i}" for i in range(MAX_KEY_INFO_KEYS_PER_REQUEST)], - key_aliases=["alias-over-cap"], - ), - user_api_key_dict=user_api_key_dict, - ) - - assert exc_info.value.code == "422" - mock_prisma_client.get_data.assert_not_awaited() + assert { + call.kwargs["window_duration"] for call in mock_get_current_spend.await_args_list + } == {"1h", "1d"} @pytest.mark.asyncio @@ -14452,7 +14372,7 @@ async def test_budget_limits_with_usage_json_string_input(monkeypatch): ) result = await _budget_limits_with_usage(budget_limits=raw, api_key_hash="hash-1") - assert result == [ + assert list(result) == [ { "budget_duration": "1h", "max_budget": 2.0, @@ -14464,8 +14384,8 @@ async def test_budget_limits_with_usage_json_string_input(monkeypatch): @pytest.mark.asyncio -async def test_budget_limits_with_usage_skips_unusable_inputs(monkeypatch): - """Invalid JSON strings, non-list values, and malformed windows are skipped.""" +async def test_budget_limits_with_usage_empty_windows_keep_stored_value(monkeypatch): + """A key with no windows (None, [], or "[]") returns None so /key/info keeps the stored value; no spend lookup runs.""" from unittest.mock import AsyncMock from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -14477,32 +14397,9 @@ async def test_budget_limits_with_usage_skips_unusable_inputs(monkeypatch): "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend ) - # invalid JSON string and non-list values return None: callers keep the original - assert await _budget_limits_with_usage(budget_limits="{not json", api_key_hash="hash-1") is None - assert await _budget_limits_with_usage(budget_limits={"budget_duration": "1h"}, api_key_hash="hash-1") is None - - # windows that are falsy, missing budget_duration, or not dict-like - windows = [ - {}, - {"max_budget": 2.0}, - {"budget_duration": "1h", "max_budget": "not-a-number"}, - 42, - ] - result = await _budget_limits_with_usage(budget_limits=windows, api_key_hash="hash-1") - - # only the well-formed window (with unparseable max_budget coerced to None) - # triggers a spend lookup - mock_get_current_spend.assert_awaited_once() - call_kwargs = mock_get_current_spend.await_args.kwargs - assert call_kwargs["counter_key"] == "spend:key:hash-1:window:1h" - assert call_kwargs["max_budget"] is None - assert result is not None - assert result[0] == {} - assert result[1] == {"max_budget": 2.0} - assert result[2] == {"budget_duration": "1h", "max_budget": "not-a-number", "current_spend": 0.0} - assert result[3] == 42 - # input is not mutated - assert windows[2] == {"budget_duration": "1h", "max_budget": "not-a-number"} + for stored in (None, [], "[]"): + assert await _budget_limits_with_usage(budget_limits=stored, api_key_hash="hash-1") is None + mock_get_current_spend.assert_not_awaited() @pytest.mark.asyncio @@ -14523,17 +14420,19 @@ async def test_budget_limits_with_usage_window_without_max_budget(monkeypatch): budget_limits=[{"budget_duration": "2d"}], api_key_hash="hash-no-max" ) - assert result == [{"budget_duration": "2d", "current_spend": 0.75}] + assert list(result) == [{"budget_duration": "2d", "current_spend": 0.75}] call_kwargs = mock_get_current_spend.await_args.kwargs assert call_kwargs["counter_key"] == "spend:key:hash-no-max:window:2d" + assert call_kwargs["window_duration"] == "2d" assert call_kwargs["max_budget"] is None @pytest.mark.asyncio async def test_budget_limits_with_usage_pydantic_windows(monkeypatch): - """Window objects with model_dump() are converted to dicts; failing windows pass through.""" - from unittest.mock import AsyncMock, MagicMock + """BudgetLimitEntry windows (the shape UserAPIKeyAuth carries) are dumped to dicts and annotated.""" + from unittest.mock import AsyncMock + from litellm.models.team import BudgetLimitEntry from litellm.proxy.management_endpoints.key_management_endpoints import ( _budget_limits_with_usage, ) @@ -14543,25 +14442,18 @@ async def test_budget_limits_with_usage_pydantic_windows(monkeypatch): "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend ) - good_window = MagicMock() - good_window.model_dump.return_value = { - "budget_duration": "7d", - "max_budget": 10.0, - "reset_at": None, - } - bad_window = MagicMock() - bad_window.model_dump.side_effect = ValueError("boom") - result = await _budget_limits_with_usage( - budget_limits=[good_window, bad_window], api_key_hash="hash-2" + budget_limits=[BudgetLimitEntry(budget_duration="7d", max_budget=10.0)], + api_key_hash="hash-2", ) - # good window converted to dict and annotated; failing window left as-is - assert result is not None - assert isinstance(result[0], dict) - assert result[0]["current_spend"] == 1.0 - assert result[1] is bad_window - mock_get_current_spend.assert_awaited_once() + assert list(result) == [ + {"budget_duration": "7d", "max_budget": 10.0, "reset_at": None, "current_spend": 1.0} + ] + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == "spend:key:hash-2:window:7d" + assert call_kwargs["window_duration"] == "7d" + assert call_kwargs["max_budget"] == 10.0 @pytest.mark.asyncio From 46d073b26f9b0037bb824384edf851155c9aecd6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 31 Aug 2026 20:48:42 -0700 Subject: [PATCH 28/34] refactor(key): report window spend under budget_limits_usage instead of inlining current_spend budget_limits now comes back exactly as stored on /key/info and /v2/key/info. The per-window usage moves to a sibling budget_limits_usage field keyed by budget_duration (current_spend, budget_limit, reset_at), mirroring model_max_budget_usage, so the stored shape that /key/update accepts never carries a computed field. --- .../key_management_endpoints.py | 52 +++++--- .../test_key_management_endpoints.py | 117 +++++++++--------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 7 +- 3 files changed, 97 insertions(+), 79 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 02bfa5c0ca8..c43b6ddf06a 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -20,6 +20,7 @@ import secrets import traceback from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeVar, cast import fastapi @@ -3590,9 +3591,12 @@ def _window_max_budget(window: Mapping[str, object]) -> float | None: return None -async def _budget_window_with_usage(window: Mapping[str, object], api_key_hash: str) -> Mapping[str, object]: +async def _budget_window_usage( + window: Mapping[str, object], api_key_hash: str +) -> tuple[str, Mapping[str, object]] | None: """ - Copy of a budget window with current-window spend attached. + (budget_duration, usage entry) for one budget window; None when the window + has no budget_duration to key it by. Reads the same cross-pod counter (spend:key:{hashed_token}:window:{budget_duration}) that _virtual_key_multi_budget_check enforces against, passing the same @@ -3603,34 +3607,41 @@ async def _budget_window_with_usage(window: Mapping[str, object], api_key_hash: duration: Final = window.get("budget_duration") if not isinstance(duration, str) or not duration: - return window + return None + max_budget: Final = _window_max_budget(window) spend: Final = await get_current_spend( counter_key=f"spend:key:{api_key_hash}:window:{duration}", fallback_spend=0.0, - max_budget=_window_max_budget(window), + max_budget=max_budget, window_entity_type="Key", window_entity_id=api_key_hash, window_duration=duration, window_start=get_budget_window_start(window), ) - return {**window, "current_spend": round(spend, 4)} # mutable-ok: per-window response copy, built once per window + return duration, MappingProxyType( + { + "current_spend": round(spend, 4), + "budget_limit": max_budget, + "reset_at": window.get("reset_at"), + } + ) -async def _budget_limits_with_usage( +async def _build_budget_limits_usage( budget_limits: Sequence[object] | str | None, api_key_hash: str -) -> tuple[Mapping[str, object], ...] | None: +) -> Mapping[str, Mapping[str, object]] | None: """ - budget_limits as window dicts with current-window spend attached; None when - the key has no windows so the caller keeps the stored value. + Current-window spend per budget window, keyed by budget_duration, reported + next to the stored budget_limits (which is returned untouched). None when + the key has no windows, so the field only appears on keys that have them. """ windows: Final = _budget_limit_windows(budget_limits) if not windows: return None - return tuple( - await asyncio.gather( - *(_budget_window_with_usage(window=window, api_key_hash=api_key_hash) for window in windows) - ) + usages: Final = await asyncio.gather( + *(_budget_window_usage(window=window, api_key_hash=api_key_hash) for window in windows) ) + return MappingProxyType({duration: usage for duration, usage in (u for u in usages if u is not None)}) @router.post( @@ -3717,12 +3728,12 @@ async def info_key_fn_v2( user_api_key_cache=model_max_budget_limiter.dual_cache, ) if k_token_hash: - budget_limits_usage = await _budget_limits_with_usage( + budget_limits_usage = await _build_budget_limits_usage( budget_limits=k_dict.get("budget_limits"), api_key_hash=k_token_hash, ) if budget_limits_usage is not None: - k_dict["budget_limits"] = budget_limits_usage + k_dict["budget_limits_usage"] = budget_limits_usage filtered_key_info.append(k_dict) return {"key": data.keys, "info": filtered_key_info} @@ -3759,9 +3770,10 @@ async def info_key_fn( - model_max_budget: dict - Per-model budgets, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}} - model_max_budget_usage: dict | None - Current-window spend per model, present only when the key has per-model budgets - - budget_limits: list | None - Concurrent budget windows. Each entry includes - current_spend: spend accumulated in the window so far (read from the same cross-pod - spend counter the budget enforcement uses) + - budget_limits: list | None - Concurrent budget windows, exactly as stored + - budget_limits_usage: dict | None - Current-window spend per budget window, keyed by + budget_duration, present only when the key has budget windows (read from the same + cross-pod spend counter the budget enforcement uses) - models: list - Model_name's the key is allowed to call - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} @@ -3841,12 +3853,12 @@ async def info_key_fn( model_max_budget=model_max_budget, user_api_key_cache=model_max_budget_limiter.dual_cache, ) - budget_limits_usage: Final = await _budget_limits_with_usage( + budget_limits_usage: Final = await _build_budget_limits_usage( budget_limits=key_info.get("budget_limits"), api_key_hash=key_token_hash, ) if budget_limits_usage is not None: - key_info["budget_limits"] = budget_limits_usage + key_info["budget_limits_usage"] = budget_limits_usage # Attach object_permission if object_permission_id is set key_info = await attach_object_permission_to_dict(key_info, prisma_client) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 425365c8560..f6e57607717 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -14143,11 +14143,11 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch): @pytest.mark.asyncio -async def test_info_key_fn_budget_limits_includes_current_spend(monkeypatch): +async def test_info_key_fn_reports_budget_limits_usage(monkeypatch): """ - /key/info should attach current_spend to each budget_limits window, read from - the same spend counter (spend:key:{token}:window:{duration}) that budget - enforcement uses. + /key/info reports current-window spend per budget window under budget_limits_usage, + keyed by budget_duration and read from the same counter enforcement uses, while + budget_limits itself comes back exactly as stored. """ from unittest.mock import AsyncMock, MagicMock @@ -14204,11 +14204,14 @@ async def test_info_key_fn_budget_limits_includes_current_spend(monkeypatch): user_api_key_dict=user_api_key_dict, ) - windows = result["info"]["budget_limits"] - assert len(windows) == 1 - assert windows[0]["current_spend"] == 0.73 - assert windows[0]["max_budget"] == 2.0 - assert windows[0]["budget_duration"] == "1h" + assert result["info"]["budget_limits"] == budget_limits + assert result["info"]["budget_limits_usage"] == { + "1h": { + "current_spend": 0.73, + "budget_limit": 2.0, + "reset_at": "2026-08-15T18:00:00+00:00", + } + } mock_get_current_spend.assert_awaited_once() call_kwargs = mock_get_current_spend.await_args.kwargs @@ -14222,7 +14225,7 @@ async def test_info_key_fn_budget_limits_includes_current_spend(monkeypatch): @pytest.mark.asyncio async def test_info_key_fn_no_budget_limits_skips_spend_lookup(monkeypatch): - """Keys without budget_limits should not trigger window spend lookups.""" + """Keys without budget windows get no budget_limits_usage field and trigger no spend lookup.""" from unittest.mock import AsyncMock, MagicMock from litellm.proxy._types import LiteLLM_VerificationToken @@ -14272,12 +14275,13 @@ async def test_info_key_fn_no_budget_limits_skips_spend_lookup(monkeypatch): ) assert result["info"]["budget_limits"] is None + assert "budget_limits_usage" not in result["info"] mock_get_current_spend.assert_not_awaited() @pytest.mark.asyncio -async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): - """/v2/key/info should attach current_spend to each budget_limits window.""" +async def test_info_key_fn_v2_reports_budget_limits_usage(monkeypatch): + """/v2/key/info reports budget_limits_usage per window and leaves budget_limits as stored.""" from unittest.mock import AsyncMock, MagicMock from litellm.proxy._types import KeyRequest, LiteLLM_VerificationToken @@ -14286,6 +14290,18 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): ) test_key_token = "hashed_token_v2_window_test" + budget_limits = [ + { + "reset_at": "2026-08-15T18:00:00+00:00", + "max_budget": 2.0, + "budget_duration": "1h", + }, + { + "reset_at": "2026-08-16T00:00:00+00:00", + "max_budget": 20.0, + "budget_duration": "1d", + }, + ] mock_prisma_client = AsyncMock() monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) @@ -14304,18 +14320,7 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): mock_key.team_id = None mock_key.model_dump.return_value = { "token": test_key_token, - "budget_limits": [ - { - "reset_at": "2026-08-15T18:00:00+00:00", - "max_budget": 2.0, - "budget_duration": "1h", - }, - { - "reset_at": "2026-08-16T00:00:00+00:00", - "max_budget": 20.0, - "budget_duration": "1d", - }, - ], + "budget_limits": [dict(w) for w in budget_limits], "user_id": "user-v2-w", "team_id": None, "litellm_budget_table": None, @@ -14335,10 +14340,19 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): ) assert len(result["info"]) == 1 - windows = result["info"][0]["budget_limits"] - assert len(windows) == 2 - assert windows[0]["current_spend"] == 1.25 - assert windows[1]["current_spend"] == 1.25 + assert result["info"][0]["budget_limits"] == budget_limits + assert result["info"][0]["budget_limits_usage"] == { + "1h": { + "current_spend": 1.25, + "budget_limit": 2.0, + "reset_at": "2026-08-15T18:00:00+00:00", + }, + "1d": { + "current_spend": 1.25, + "budget_limit": 20.0, + "reset_at": "2026-08-16T00:00:00+00:00", + }, + } assert mock_get_current_spend.await_count == 2 counter_keys = { call.kwargs["counter_key"] for call in mock_get_current_spend.await_args_list @@ -14353,13 +14367,13 @@ async def test_info_key_fn_v2_budget_limits_includes_current_spend(monkeypatch): @pytest.mark.asyncio -async def test_budget_limits_with_usage_json_string_input(monkeypatch): - """budget_limits stored as a JSON string should be parsed and annotated.""" +async def test_build_budget_limits_usage_json_string_input(monkeypatch): + """budget_limits stored as a JSON string is parsed and reported per window.""" import json as json_module from unittest.mock import AsyncMock from litellm.proxy.management_endpoints.key_management_endpoints import ( - _budget_limits_with_usage, + _build_budget_limits_usage, ) mock_get_current_spend = AsyncMock(return_value=0.5) @@ -14370,26 +14384,19 @@ async def test_budget_limits_with_usage_json_string_input(monkeypatch): raw = json_module.dumps( [{"budget_duration": "1h", "max_budget": 2.0, "reset_at": None}] ) - result = await _budget_limits_with_usage(budget_limits=raw, api_key_hash="hash-1") + result = await _build_budget_limits_usage(budget_limits=raw, api_key_hash="hash-1") - assert list(result) == [ - { - "budget_duration": "1h", - "max_budget": 2.0, - "reset_at": None, - "current_spend": 0.5, - } - ] + assert result == {"1h": {"current_spend": 0.5, "budget_limit": 2.0, "reset_at": None}} mock_get_current_spend.assert_awaited_once() @pytest.mark.asyncio -async def test_budget_limits_with_usage_empty_windows_keep_stored_value(monkeypatch): - """A key with no windows (None, [], or "[]") returns None so /key/info keeps the stored value; no spend lookup runs.""" +async def test_build_budget_limits_usage_empty_windows_returns_none(monkeypatch): + """A key with no windows (None, [], or "[]") returns None so the field is left off; no spend lookup runs.""" from unittest.mock import AsyncMock from litellm.proxy.management_endpoints.key_management_endpoints import ( - _budget_limits_with_usage, + _build_budget_limits_usage, ) mock_get_current_spend = AsyncMock(return_value=0.0) @@ -14398,17 +14405,17 @@ async def test_budget_limits_with_usage_empty_windows_keep_stored_value(monkeypa ) for stored in (None, [], "[]"): - assert await _budget_limits_with_usage(budget_limits=stored, api_key_hash="hash-1") is None + assert await _build_budget_limits_usage(budget_limits=stored, api_key_hash="hash-1") is None mock_get_current_spend.assert_not_awaited() @pytest.mark.asyncio -async def test_budget_limits_with_usage_window_without_max_budget(monkeypatch): - """A window with only budget_duration still gets current_spend, read without a budget ceiling.""" +async def test_build_budget_limits_usage_window_without_max_budget(monkeypatch): + """A window with only budget_duration still reports current_spend, read without a budget ceiling.""" from unittest.mock import AsyncMock from litellm.proxy.management_endpoints.key_management_endpoints import ( - _budget_limits_with_usage, + _build_budget_limits_usage, ) mock_get_current_spend = AsyncMock(return_value=0.75) @@ -14416,11 +14423,11 @@ async def test_budget_limits_with_usage_window_without_max_budget(monkeypatch): "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend ) - result = await _budget_limits_with_usage( + result = await _build_budget_limits_usage( budget_limits=[{"budget_duration": "2d"}], api_key_hash="hash-no-max" ) - assert list(result) == [{"budget_duration": "2d", "current_spend": 0.75}] + assert result == {"2d": {"current_spend": 0.75, "budget_limit": None, "reset_at": None}} call_kwargs = mock_get_current_spend.await_args.kwargs assert call_kwargs["counter_key"] == "spend:key:hash-no-max:window:2d" assert call_kwargs["window_duration"] == "2d" @@ -14428,13 +14435,13 @@ async def test_budget_limits_with_usage_window_without_max_budget(monkeypatch): @pytest.mark.asyncio -async def test_budget_limits_with_usage_pydantic_windows(monkeypatch): - """BudgetLimitEntry windows (the shape UserAPIKeyAuth carries) are dumped to dicts and annotated.""" +async def test_build_budget_limits_usage_pydantic_windows(monkeypatch): + """BudgetLimitEntry windows (the shape UserAPIKeyAuth carries) are dumped to dicts and reported.""" from unittest.mock import AsyncMock from litellm.models.team import BudgetLimitEntry from litellm.proxy.management_endpoints.key_management_endpoints import ( - _budget_limits_with_usage, + _build_budget_limits_usage, ) mock_get_current_spend = AsyncMock(return_value=1.0) @@ -14442,14 +14449,12 @@ async def test_budget_limits_with_usage_pydantic_windows(monkeypatch): "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend ) - result = await _budget_limits_with_usage( + result = await _build_budget_limits_usage( budget_limits=[BudgetLimitEntry(budget_duration="7d", max_budget=10.0)], api_key_hash="hash-2", ) - assert list(result) == [ - {"budget_duration": "7d", "max_budget": 10.0, "reset_at": None, "current_spend": 1.0} - ] + assert result == {"7d": {"current_spend": 1.0, "budget_limit": 10.0, "reset_at": None}} call_kwargs = mock_get_current_spend.await_args.kwargs assert call_kwargs["counter_key"] == "spend:key:hash-2:window:7d" assert call_kwargs["window_duration"] == "7d" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9b5adec516d..7b6f428c5a4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7710,9 +7710,10 @@ export interface paths { * - model_max_budget: dict - Per-model budgets, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}} * - model_max_budget_usage: dict | None - Current-window spend per model, present only when * the key has per-model budgets - * - budget_limits: list | None - Concurrent budget windows. Each entry includes - * current_spend: spend accumulated in the window so far (read from the same cross-pod - * spend counter the budget enforcement uses) + * - budget_limits: list | None - Concurrent budget windows, exactly as stored + * - budget_limits_usage: dict | None - Current-window spend per budget window, keyed by + * budget_duration, present only when the key has budget windows (read from the same + * cross-pod spend counter the budget enforcement uses) * - models: list - Model_name's the key is allowed to call * - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits * - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} From d1320404feff822f822b812951a0782c30d6cbb5 Mon Sep 17 00:00:00 2001 From: Kolade Fajimi <107228310+koladefaj@users.noreply.github.com> Date: Tue, 1 Sep 2026 04:51:31 +0100 Subject: [PATCH 29/34] fix(redis): coerce env var string types and fix param discovery through decorator wrappers (#30644) * fix(redis): coerce env var string types and fix param discovery through decorator wrappers inspect.getfullargspec doesn't work on redis.Redis/redis.RedisCluster because their __init__ is wrapped by @deprecated_args, which replaces the explicit signature with *args/**kwargs internally. getfullargspec returns an empty arg list, so _get_redis_kwargs and _get_redis_cluster_kwargs silently dropped every real constructor parameter not in their hand-picked include_args set -- cluster_error_retry_attempts and connection_error_retry_attempts among them, so an operator's configured retry bound never reached the Redis Cluster client and it fell back to redis-py's own default instead. Rebased onto litellm_internal_staging, which had independently added _init_arg_names (MRO-walking, inspect.unwrap-based) for the same class of bug in _get_redis_url_kwargs. Reused that pattern (as _unwrapped_init_args, without the MRO walk: redis.Redis/RedisCluster declare every real parameter directly on their own __init__, and MRO-walking breaks the tests here that mock the class with autospec=True, since inspect.getmro needs a real __mro__) rather than introducing a second, differently-shaped fix for the same problem. _get_redis_cluster_kwargs now also honors its own client argument instead of ignoring it, so the async cluster client's own extra constructor kwargs (cluster_error_retry_attempts, connection_error_retry_attempts, decode_responses, ...) are no longer filtered out by introspecting the sync class regardless of which client is actually built. Also fixes environment variables and Helm --set values always arriving as strings: redis-py 8.x changed health_check_interval's arithmetic to require a real number, so a stringified value raised TypeError on every Redis operation instead of connecting. _coerce_redis_kwargs_types coerces to each parameter's declared type at the end of _get_redis_client_logic, with an explicit type table for max_connections/socket_timeout/socket_connect_timeout since redis-py 8.x changed the timeout defaults from None to int 5, which would otherwise make a fractional value fail int() and get dropped. Co-authored-by: mangabits <1457532+mangabits@users.noreply.github.com> * ci: verify redis-py client version compatibility across a version matrix * test(redis): assert an async-only cluster kwarg every matrix version declares connection_error_retry_attempts is on the async cluster constructor in redis-py 5.x only; 6.0 removed it in favor of retry. The 6.4.0, 7.4.1 and 8.0.1 legs were failing on that missing parameter name rather than on the behavior under test, while the allow-list itself was doing the right thing on all four versions. decode_responses is async-cluster-only on every version the matrix covers, so it stands in for the same property: the sync cluster class takes it through **kwargs and never names it in its signature. Reverting _get_redis_cluster_kwargs to ignore its client argument still fails both tests on 5.3.1 and 8.0.1. test_async_cluster_passes_async_only_kwargs now builds the real async cluster client and reads connection_kwargs off it, so it no longer needs a patched class factory; the constructor does no I/O. The retry-attempts test keeps its patch, since redis-py >= 6 stores no cluster_error_retry_attempts attribute on the built client and the constructor call is the only place the forwarded value shows up. The _get_redis_cluster_kwargs docstring cited the same two parameters as its examples of async-only kwargs, which is what made the test look reasonable; cluster_error_retry_attempts is on both classes and connection_error_retry_attempts is gone from 6.0 on, so it now names decode_responses instead. * test(redis): drop internal patches from the kwarg coercion tests The test-quality gate flagged the new patch() calls on litellm internals these tests added. Three of them faked litellm._redis.inspect.signature with a MagicMock to hand _coerce_redis_kwargs_types a synthetic parameter; that function already takes a client argument, so they pass stub functions instead, matching the _redis_signature_8x idiom the file uses elsewhere. The fourth patched _redis_kwargs_from_environment to {} to prove _get_redis_client_logic raises without a host or url, which clearing the real env keys through _get_redis_env_kwarg_mapping does without pinning the test to that call. Both files now sit one TQ008 below the merge base rather than six above it. * fix(redis): keep the sync client construction inside the basedpyright budget _get_redis_client_logic now returns dict[str, object] rather than an untyped dict, which is the honest type for operator-supplied config, but it turns the 33 reportUnknownArgumentType errors at redis.Redis(**redis_kwargs) into 33 reportArgumentType errors plus one reportCallIssue, both over their budget. No static type fits: redis-py's constructor declares 40-odd differently typed parameters and the values arrive from config and env, so the allow-list and coercion above are derived from that same signature and redis-py validates each value itself at runtime. The two suppressions name their exact rule and carry that reason. The file ends up 42 basedpyright errors below the merge base, with reportArgumentType and reportCallIssue back at the base counts of 3 and 0. * fix(redis): coerce cluster-only and None-default bool kwargs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mangabits <1457532+mangabits@users.noreply.github.com> Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-redis-compat.yml | 77 +++++++ litellm/_redis.py | 125 ++++++++++- .../caching/test_redis_connection_pool.py | 201 ++++++++++++++++-- tests/test_litellm/test_redis.py | 67 ++++++ 4 files changed, 443 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/test-redis-compat.yml diff --git a/.github/workflows/test-redis-compat.yml b/.github/workflows/test-redis-compat.yml new file mode 100644 index 00000000000..f29755a74b1 --- /dev/null +++ b/.github/workflows/test-redis-compat.yml @@ -0,0 +1,77 @@ +name: "Unit Tests: Redis Client Version Compatibility" + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + paths: + - "litellm/_redis.py" + - "litellm/_redis_credential_provider.py" + - "tests/test_litellm/test_redis.py" + - "tests/test_litellm/caching/test_redis_connection_pool.py" + - ".github/workflows/test-redis-compat.yml" + - "pyproject.toml" + - "uv.lock" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + redis-compat: + name: "redis-py ${{ matrix.redis-version }}" + runs-on: ubuntu-latest + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the + # newer legs prove the inspect.signature introspection in litellm/_redis.py + # keeps extracting kwargs on the redis-py releases people actually run now. + # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra) + # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in + # for the 6.x line. + redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"] + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Pin redis-py to the matrix version + env: + REDIS_VERSION: ${{ matrix.redis-version }} + run: | + uv pip install "redis==${REDIS_VERSION:?}" + uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)" + + - name: Run redis unit tests + run: | + uv run --no-sync pytest \ + tests/test_litellm/test_redis.py \ + tests/test_litellm/caching/test_redis_connection_pool.py \ + --tb=short -vv \ + --reruns 2 \ + --reruns-delay 1 \ + --durations=20 diff --git a/litellm/_redis.py b/litellm/_redis.py index 9381357931e..3e68d50cf16 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -13,6 +13,7 @@ import json # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation import os from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final from urllib.parse import urlsplit, urlunsplit @@ -38,9 +39,25 @@ from ._logging import verbose_logger AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" -def _get_redis_kwargs(): - arg_spec: Final = inspect.getfullargspec(redis.Redis) +def _unwrapped_init_args(cls: type) -> frozenset[str]: + """Every parameter on a single class's own ``__init__``, decorator-unwrapped. + Unlike ``_init_arg_names`` below, this does not walk the MRO: ``redis.Redis`` + and ``redis.RedisCluster`` (sync and async) each declare every real + constructor parameter directly on their own ``__init__``, so MRO-walking is + unnecessary — and it actively breaks the several tests here that mock the + class with ``patch(..., autospec=True)``, since ``inspect.getmro`` needs a + real ``__mro__`` that an autospec'd stand-in for a class does not provide. + + Still unwraps first: redis-py >= 7.4 decorates these ``__init__``s with + ``@deprecated_args`` too, which the same class of bug as ``_init_arg_names`` + would otherwise silently empty this allowlist through (see its docstring). + """ + spec: Final = inspect.getfullargspec(inspect.unwrap(cls.__init__)) + return frozenset(spec.args + spec.kwonlyargs) + + +def _get_redis_kwargs(): # Only allow primitive arguments exclude_args: Final = { "self", @@ -60,7 +77,7 @@ def _get_redis_kwargs(): "azure_client_secret", } - available_args: Final = {x for x in arg_spec.args if x not in exclude_args} | include_args + available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args return available_args @@ -120,15 +137,23 @@ def _get_redis_url_kwargs(client: type | None = None) -> tuple[str, ...]: return tuple(x for x in _init_arg_names(connection_cls) if x not in exclude_args) + include_args -def _get_redis_cluster_kwargs(client=None): +def _get_redis_cluster_kwargs(client: type | None = None): + """Config kwargs the target cluster client's constructor actually accepts. + + Defaults to the sync ``redis.RedisCluster``, but the async cluster client + (``redis.asyncio.cluster.RedisCluster``) declares connection settings such as + ``decode_responses`` on its own constructor, where the sync class takes them + through ``**kwargs`` and so never names them in its signature. Introspecting + only the sync class regardless of which client is actually built silently + drops those for every async cluster caller. + """ if client is None: - client = redis.Redis.from_url - arg_spec: Final = inspect.getfullargspec(redis.RedisCluster) + client = redis.RedisCluster # Only allow primitive arguments exclude_args: Final = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"} - available_args = {x for x in arg_spec.args if x not in exclude_args} + available_args = {x for x in _unwrapped_init_args(client) if x not in exclude_args} available_args |= { "password", "username", @@ -161,6 +186,79 @@ def _get_redis_env_kwarg_mapping(): return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment} +def _str_to_bool(value: str) -> bool: + return value.lower() in ("true", "1", "yes") + + +def _coerce_redis_kwargs_types( + redis_kwargs: Mapping[str, object], + client: type | tuple[type, ...] = redis.Redis, +) -> dict[str, object]: # mutable-ok: a caller mutates the returned kwargs before constructing its client + """Coerces string values to the numeric/boolean type ``client``'s constructor + declares for that parameter. ``client`` may be a tuple of client classes; a + parameter's type is taken from the first signature that declares it, which + lets cluster callers coerce cluster-only kwargs such as + ``cluster_error_retry_attempts`` alongside the shared connection kwargs. + + Environment variables are always strings, and Helm ``--set`` stringifies values + too, so a config value like ``health_check_interval`` or ``socket_timeout`` + can arrive as ``"30"``/``"5.5"`` rather than a real number. redis-py's own + connection-health-check arithmetic (``loop.time() + self.health_check_interval``) + then raises ``TypeError`` on every Redis operation instead of connecting. + + ``max_connections``, ``socket_timeout``, and ``socket_connect_timeout`` use an + explicit target type rather than the parameter's own signature default: redis-py + 8.x changed the timeout defaults from ``None`` to int ``5``, so inferring the + type from the default would make a fractional ``"5.5"`` fail ``int()`` and get + silently dropped on 8.x while working on older versions. ``socket_keepalive`` + is explicit too: its signature default is ``None``, which carries no type to + infer from, and leaving it a string makes ``"false"`` truthy. + """ + signatures: Final = tuple(inspect.signature(c) for c in (client if isinstance(client, tuple) else (client,))) + explicit_param_types: Final = MappingProxyType( + { + "max_connections": int, + "socket_timeout": float, + "socket_connect_timeout": float, + "socket_keepalive": bool, + } + ) + result: Final = dict(redis_kwargs) # mutable-ok: per-key try/except coercion below needs to drop individual keys + for key, value in redis_kwargs.items(): + if not isinstance(value, str): + continue + param = next((sig.parameters[key] for sig in signatures if key in sig.parameters), None) + if param is None: + continue + explicit_type = explicit_param_types.get(key) + if explicit_type is bool: + result[key] = _str_to_bool(value) + continue + if explicit_type is not None: + try: + result[key] = explicit_type(value) + except (ValueError, TypeError): + del result[key] + continue + default: object = param.default # pyright: ignore[reportAny] # inspect.Parameter.default is stubbed as Any + if default is inspect.Parameter.empty: + continue + # bool must be checked before int, since bool subclasses int + if isinstance(default, bool): + result[key] = _str_to_bool(value) + elif isinstance(default, int): + try: + result[key] = int(value) + except (ValueError, TypeError): + del result[key] + elif isinstance(default, float): + try: + result[key] = float(value) + except (ValueError, TypeError): + del result[key] + return result + + def _redis_kwargs_from_environment(): mapping: Final = _get_redis_env_kwarg_mapping() @@ -505,7 +603,12 @@ def _get_redis_client_logic(**env_overrides): raise ValueError("Either 'host' or 'url' must be specified for redis.") # litellm.print_verbose(f"redis_kwargs: {redis_kwargs}") - return redis_kwargs + coercion_client: Final = ( + (redis.Redis, redis.RedisCluster, async_redis.RedisCluster) + if redis_kwargs.get("startup_nodes") + else redis.Redis + ) + return _coerce_redis_kwargs_types(redis_kwargs, client=coercion_client) def init_redis_cluster(redis_kwargs) -> redis.RedisCluster: @@ -657,7 +760,9 @@ def get_redis_client(**env_overrides): if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs: return _init_redis_sentinel(redis_kwargs) - return redis.Redis(**redis_kwargs) + return redis.Redis( # pyright: ignore[reportCallIssue] # object-valued kwargs match no overload statically + **redis_kwargs, # pyright: ignore[reportArgumentType] # allow-listed and coerced against this signature + ) def get_redis_async_client( @@ -669,7 +774,7 @@ def get_redis_async_client( if "startup_nodes" in redis_kwargs: from redis.cluster import ClusterNode - args = _get_redis_cluster_kwargs() + args = _get_redis_cluster_kwargs(async_redis.RedisCluster) cluster_kwargs: Final = {} for arg in redis_kwargs: if arg in args: diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index c824d3e7a0e..54dbe5361d7 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,15 +1,14 @@ -""" -Regression tests for Redis connection pool leak fixes (RC1-RC5). - -Tests are pure unit tests — no Redis server required. -""" - from unittest.mock import AsyncMock, MagicMock, patch import pytest -import redis.asyncio as async_redis -from litellm._redis import get_redis_async_client, get_redis_connection_pool +from litellm._redis import ( + _coerce_redis_kwargs_types, + _get_redis_client_logic, + _get_redis_env_kwarg_mapping, + get_redis_async_client, + get_redis_connection_pool, +) def test_url_config_uses_passed_pool(): @@ -60,16 +59,14 @@ def test_max_connections_url_config_string_value(monkeypatch): assert pool.max_connections == 25 -def test_max_connections_url_config_invalid_value(): - """Invalid max_connections should be silently ignored, falling back - to the pool default (50 for BlockingConnectionPool).""" - with patch("litellm._redis._get_redis_client_logic") as mock_logic: - mock_logic.return_value = { - "url": "redis://localhost:6379/0", - "max_connections": "not_a_number", - } +def test_max_connections_url_config_invalid_value(monkeypatch): + """Invalid max_connections from an env var should be silently dropped, + falling back to the pool default (50 for BlockingConnectionPool).""" + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.setenv("REDIS_MAX_CONNECTIONS", "not_a_number") - pool = get_redis_connection_pool() + pool = get_redis_connection_pool() # BlockingConnectionPool default is 50 assert pool.max_connections == 50 @@ -128,3 +125,173 @@ async def test_disconnect_idempotent(): await cache.disconnect() await cache.disconnect() # should not raise + + +def test_coerce_redis_kwargs_types_int(): + """String values for int-typed Redis params are coerced to int.""" + result = _coerce_redis_kwargs_types({"health_check_interval": "30", "port": "6380", "db": "1"}) + assert result["health_check_interval"] == 30 + assert isinstance(result["health_check_interval"], int) + assert result["port"] == 6380 + assert result["db"] == 1 + + +def test_coerce_redis_kwargs_types_bool(): + """String values for bool-typed Redis params are coerced to bool.""" + result = _coerce_redis_kwargs_types({"ssl": "true", "decode_responses": "false"}) + assert result["ssl"] is True + assert result["decode_responses"] is False + + +def test_coerce_redis_kwargs_types_none_default_numeric(): + """String values for known None-default numeric params are coerced.""" + result = _coerce_redis_kwargs_types({"max_connections": "20", "socket_timeout": "5.5"}) + assert result["max_connections"] == 20 + assert isinstance(result["max_connections"], int) + assert result["socket_timeout"] == 5.5 + assert isinstance(result["socket_timeout"], float) + + +def _redis_signature_pre_8x( + socket_timeout=None, + socket_connect_timeout=None, + max_connections=None, + health_check_interval=0, +): + """Stand-in for the redis-py <= 7.x Redis signature, where the timeout defaults are None.""" + + +def _redis_signature_8x( + socket_timeout=5, + socket_connect_timeout=5, + max_connections=None, + health_check_interval=0, +): + """Stand-in for the redis-py 8.x Redis signature, where the timeout defaults became int 5.""" + + +@pytest.mark.parametrize( + "client", + [_redis_signature_pre_8x, _redis_signature_8x], + ids=["redis-py<=7.x", "redis-py-8.x"], +) +def test_coerce_fractional_socket_timeout_survives_signature_default_change(client): + """redis-py 8.x changed socket_timeout's default from None to int 5. Deriving the + target type from the signature default made int("5.5") raise, so the key was dropped + and REDIS_SOCKET_TIMEOUT=5.5 silently disappeared on 8.x.""" + result = _coerce_redis_kwargs_types( + {"socket_timeout": "5.5", "socket_connect_timeout": "2.5", "max_connections": "20"}, + client=client, + ) + + assert result["socket_timeout"] == pytest.approx(5.5) + assert isinstance(result["socket_timeout"], float) + assert result["socket_connect_timeout"] == pytest.approx(2.5) + assert isinstance(result["socket_connect_timeout"], float) + assert result["max_connections"] == 20 + assert isinstance(result["max_connections"], int) + + +def test_coerce_invalid_socket_timeout_is_still_dropped(): + """Garbage must not survive the explicit-type path; Redis falls back to its own default.""" + result = _coerce_redis_kwargs_types({"socket_timeout": "not_a_number"}, client=_redis_signature_8x) + + assert "socket_timeout" not in result + + +def test_coerce_redis_kwargs_types_invalid_drops_key(): + """A string that cannot be coerced to the expected numeric type is dropped.""" + result = _coerce_redis_kwargs_types({"health_check_interval": "not_a_number"}) + assert "health_check_interval" not in result + + +def test_coerce_redis_kwargs_types_non_string_unchanged(): + """Non-string values pass through without modification.""" + result = _coerce_redis_kwargs_types({"health_check_interval": 30, "ssl": True}) + assert result["health_check_interval"] == 30 + assert result["ssl"] is True + + +def test_health_check_interval_from_env_is_int(monkeypatch): + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_HEALTH_CHECK_INTERVAL", "30") + + pool = get_redis_connection_pool() + + assert pool is not None + interval = pool.connection_kwargs.get("health_check_interval") + assert interval == 30 + assert isinstance(interval, int), f"Expected int, got {type(interval)}: {interval!r}" + + +def _signature_without_defaults(testkey): + """Stand-in for a client whose parameter declares no default at all.""" + + +def _signature_with_float_default(myparam=1.0): + """Stand-in for a client whose parameter declares a float default.""" + + +def test_coerce_redis_kwargs_types_empty_default_param_unchanged(): + """String params whose signature entry has no default (inspect.Parameter.empty) are left as-is.""" + result = _coerce_redis_kwargs_types({"testkey": "some_value"}, client=_signature_without_defaults) + + assert result["testkey"] == "some_value" + assert isinstance(result["testkey"], str) + + +def test_coerce_redis_kwargs_types_float_valid(): + """String values for params whose signature default is a float are coerced to float.""" + result = _coerce_redis_kwargs_types({"myparam": "3.14"}, client=_signature_with_float_default) + + assert result["myparam"] == pytest.approx(3.14) + assert isinstance(result["myparam"], float) + + +def test_coerce_redis_kwargs_types_float_invalid_drops_key(): + """An unconvertible string for a float-default param is dropped from the result.""" + result = _coerce_redis_kwargs_types({"myparam": "not_a_float"}, client=_signature_with_float_default) + + assert "myparam" not in result + + +@pytest.mark.parametrize( + ("raw", "expected"), + [("false", False), ("true", True), ("0", False), ("1", True)], +) +def test_coerce_socket_keepalive_string(raw, expected): + """socket_keepalive's signature default is None, so it needs an explicit bool + coercion: a leftover "false" string is truthy and enables keepalive.""" + result = _coerce_redis_kwargs_types({"socket_keepalive": raw}) + + assert result["socket_keepalive"] is expected + + +def test_get_redis_client_logic_coerces_cluster_only_kwargs(monkeypatch): + """Cluster-only kwargs (absent from redis.Redis's signature) must still be + coerced when routing to a cluster, or Helm-stringified values reach + RedisCluster as strings.""" + for envvar in (*_get_redis_env_kwarg_mapping(), "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(envvar, raising=False) + + result = _get_redis_client_logic( + startup_nodes='[{"host": "localhost", "port": 7000}]', + cluster_error_retry_attempts="5", + require_full_coverage="false", + health_check_interval="30", + ) + + assert result["cluster_error_retry_attempts"] == 5 + assert isinstance(result["cluster_error_retry_attempts"], int) + assert result["require_full_coverage"] is False + assert result["health_check_interval"] == 30 + assert isinstance(result["health_check_interval"], int) + + +def test_get_redis_client_logic_raises_without_host_or_url(monkeypatch): + """_get_redis_client_logic raises ValueError when neither host nor url is provided.""" + for envvar in (*_get_redis_env_kwarg_mapping(), "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(envvar, raising=False) + + with pytest.raises(ValueError, match="Either 'host' or 'url' must be specified for redis"): + _get_redis_client_logic() diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 826beb74a27..a96e8541e06 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -1,3 +1,4 @@ +import inspect import json from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -600,6 +601,72 @@ def test_reconnect_kwargs_in_cluster_kwargs(): assert "socket_keepalive" in kwargs +def test_retry_attempts_in_cluster_kwargs(): + """cluster_error_retry_attempts must survive the cluster kwarg allow-list so + operators can bound worst-case retry latency on a Redis Cluster: it was being + silently dropped because the allow-list was built from redis.RedisCluster's + decorated __init__ without unwrapping it, so getfullargspec saw an empty + (self, *args, **kwargs) wrapper signature.""" + kwargs = _get_redis_cluster_kwargs() + assert "cluster_error_retry_attempts" in kwargs + + +def test_async_only_kwargs_in_cluster_kwargs_when_async_client_requested(): + """decode_responses is on the async cluster client's constructor and not the sync + one, on every redis-py the matrix covers. Introspecting the sync class regardless + of which client is actually built silently drops it for every async cluster caller.""" + sync_kwargs = _get_redis_cluster_kwargs() + async_kwargs = _get_redis_cluster_kwargs(async_redis.RedisCluster) + + assert "decode_responses" not in sync_kwargs + assert "decode_responses" in async_kwargs + + +@patch( # test-quality-ok: redis-py >= 6 keeps no cluster_error_retry_attempts attribute on the built client, so the constructor call is the only place the value is observable + "litellm.caching.redis_cluster_node_isolation.get_litellm_async_redis_cluster_class" +) +def test_async_cluster_forwards_retry_attempts(mock_get_cluster_class): + """Regression: cluster_error_retry_attempts must reach the constructed async + cluster client. Silently dropping it removes an operator's only lever for + bounding a stuck node's worst-case retry latency, and the client falls back + to redis-py's own default (3 retries) instead.""" + mock_cluster_cls = mock_get_cluster_class.return_value + get_redis_async_client( + startup_nodes=[{"host": "cluster-node", "port": 6379}], + cluster_error_retry_attempts=2, + ) + + call_kwargs = mock_cluster_cls.call_args[1] + assert call_kwargs["cluster_error_retry_attempts"] == 2 + + +def test_async_cluster_passes_async_only_kwargs(): + """Regression: decode_responses is an async-cluster-only constructor arg. When + the allow-list came from the sync class it was filtered out and values came + back as bytes instead of str.""" + client = get_redis_async_client( + startup_nodes=[{"host": "cluster-node", "port": 6379}], + decode_responses=True, + ) + + assert client.connection_kwargs["decode_responses"] is True + + +@pytest.mark.parametrize("cluster_client", [redis.RedisCluster, async_redis.RedisCluster], ids=["sync", "async"]) +def test_cluster_kwargs_exclude_variadic_parameters(cluster_client): + """*args / **kwargs are signature placeholders, not connection settings, and + must never land in the allow-list regardless of which cluster client is + introspected.""" + variadic = { + name + for name, param in inspect.signature(cluster_client).parameters.items() + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD) + } + + leaked = variadic & set(_get_redis_cluster_kwargs(cluster_client)) + assert not leaked, f"variadic params leaked into the allow-list: {leaked}" + + @patch("litellm.caching.redis_cluster_node_isolation.get_litellm_async_redis_cluster_class") def test_async_cluster_sets_reconnect_defaults(mock_get_cluster_class): """ From 760b864e43045032bd76348650cf15f6e207b2a9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 31 Aug 2026 21:11:15 -0700 Subject: [PATCH 30/34] refactor(key): trim budget_limits_usage entries to current_spend max_budget and reset_at already live on the matching budget_limits entry, so repeating them (as budget_limit and reset_at) only invited confusion about which copy is authoritative. --- .../key_management_endpoints.py | 17 ++++-------- .../test_key_management_endpoints.py | 26 +++++-------------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 ++--- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c43b6ddf06a..99e201930c1 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3608,23 +3608,16 @@ async def _budget_window_usage( duration: Final = window.get("budget_duration") if not isinstance(duration, str) or not duration: return None - max_budget: Final = _window_max_budget(window) spend: Final = await get_current_spend( counter_key=f"spend:key:{api_key_hash}:window:{duration}", fallback_spend=0.0, - max_budget=max_budget, + max_budget=_window_max_budget(window), window_entity_type="Key", window_entity_id=api_key_hash, window_duration=duration, window_start=get_budget_window_start(window), ) - return duration, MappingProxyType( - { - "current_spend": round(spend, 4), - "budget_limit": max_budget, - "reset_at": window.get("reset_at"), - } - ) + return duration, MappingProxyType({"current_spend": round(spend, 4)}) async def _build_budget_limits_usage( @@ -3771,9 +3764,9 @@ async def info_key_fn( - model_max_budget_usage: dict | None - Current-window spend per model, present only when the key has per-model budgets - budget_limits: list | None - Concurrent budget windows, exactly as stored - - budget_limits_usage: dict | None - Current-window spend per budget window, keyed by - budget_duration, present only when the key has budget windows (read from the same - cross-pod spend counter the budget enforcement uses) + - budget_limits_usage: dict | None - Current-window spend per budget window, e.g. + {"1h": {"current_spend": 0.0009}}, present only when the key has budget windows + (read from the same cross-pod spend counter the budget enforcement uses) - models: list - Model_name's the key is allowed to call - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index f6e57607717..b5a4204f0af 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -14205,13 +14205,7 @@ async def test_info_key_fn_reports_budget_limits_usage(monkeypatch): ) assert result["info"]["budget_limits"] == budget_limits - assert result["info"]["budget_limits_usage"] == { - "1h": { - "current_spend": 0.73, - "budget_limit": 2.0, - "reset_at": "2026-08-15T18:00:00+00:00", - } - } + assert result["info"]["budget_limits_usage"] == {"1h": {"current_spend": 0.73}} mock_get_current_spend.assert_awaited_once() call_kwargs = mock_get_current_spend.await_args.kwargs @@ -14342,16 +14336,8 @@ async def test_info_key_fn_v2_reports_budget_limits_usage(monkeypatch): assert len(result["info"]) == 1 assert result["info"][0]["budget_limits"] == budget_limits assert result["info"][0]["budget_limits_usage"] == { - "1h": { - "current_spend": 1.25, - "budget_limit": 2.0, - "reset_at": "2026-08-15T18:00:00+00:00", - }, - "1d": { - "current_spend": 1.25, - "budget_limit": 20.0, - "reset_at": "2026-08-16T00:00:00+00:00", - }, + "1h": {"current_spend": 1.25}, + "1d": {"current_spend": 1.25}, } assert mock_get_current_spend.await_count == 2 counter_keys = { @@ -14386,7 +14372,7 @@ async def test_build_budget_limits_usage_json_string_input(monkeypatch): ) result = await _build_budget_limits_usage(budget_limits=raw, api_key_hash="hash-1") - assert result == {"1h": {"current_spend": 0.5, "budget_limit": 2.0, "reset_at": None}} + assert result == {"1h": {"current_spend": 0.5}} mock_get_current_spend.assert_awaited_once() @@ -14427,7 +14413,7 @@ async def test_build_budget_limits_usage_window_without_max_budget(monkeypatch): budget_limits=[{"budget_duration": "2d"}], api_key_hash="hash-no-max" ) - assert result == {"2d": {"current_spend": 0.75, "budget_limit": None, "reset_at": None}} + assert result == {"2d": {"current_spend": 0.75}} call_kwargs = mock_get_current_spend.await_args.kwargs assert call_kwargs["counter_key"] == "spend:key:hash-no-max:window:2d" assert call_kwargs["window_duration"] == "2d" @@ -14454,7 +14440,7 @@ async def test_build_budget_limits_usage_pydantic_windows(monkeypatch): api_key_hash="hash-2", ) - assert result == {"7d": {"current_spend": 1.0, "budget_limit": 10.0, "reset_at": None}} + assert result == {"7d": {"current_spend": 1.0}} call_kwargs = mock_get_current_spend.await_args.kwargs assert call_kwargs["counter_key"] == "spend:key:hash-2:window:7d" assert call_kwargs["window_duration"] == "7d" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7b6f428c5a4..77e42c09525 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7711,9 +7711,9 @@ export interface paths { * - model_max_budget_usage: dict | None - Current-window spend per model, present only when * the key has per-model budgets * - budget_limits: list | None - Concurrent budget windows, exactly as stored - * - budget_limits_usage: dict | None - Current-window spend per budget window, keyed by - * budget_duration, present only when the key has budget windows (read from the same - * cross-pod spend counter the budget enforcement uses) + * - budget_limits_usage: dict | None - Current-window spend per budget window, e.g. + * {"1h": {"current_spend": 0.0009}}, present only when the key has budget windows + * (read from the same cross-pod spend counter the budget enforcement uses) * - models: list - Model_name's the key is allowed to call * - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits * - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} From c1b5cacf1fe5eee475a01f87870558b915b5e2ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:12:53 -0700 Subject: [PATCH 31/34] refactor(speech): freeze httpx response header dicts (LIT002) --- .../speech/speech_to_completion_bridge/transformation.py | 4 +++- litellm/llms/vertex_ai/text_to_speech/transformation.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index e2d3fadf852..2ed140c0208 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -174,7 +174,9 @@ class SpeechToCompletionBridgeTransformationHandler: if self._is_gemini_tts_model(model) else (decoded_audio, "audio/mpeg") ) - response: Final = httpx.Response(status_code=200, content=content, headers={"Content-Type": content_type}) + response: Final = httpx.Response( + status_code=200, content=content, headers=MappingProxyType({"Content-Type": content_type}) + ) binary_response: Final = HttpxBinaryResponseContent(response) binary_response.set_response_cost(_completion_response_cost(model_response)) return binary_response diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index a5ff7eca021..332f892ae6b 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -7,6 +7,7 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s import base64 from collections.abc import Coroutine +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -464,7 +465,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): media_type: Final = speech_media_type_from_audio_bytes(binary_data) response: Final = httpx.Response( status_code=200, - headers={} if media_type is None else {"content-type": media_type}, + headers=None if media_type is None else MappingProxyType({"content-type": media_type}), content=binary_data, ) From 65a46a5f32a824e5d42f6d92d4183ad7febf8fe4 Mon Sep 17 00:00:00 2001 From: George Pickett Date: Mon, 31 Aug 2026 21:28:45 -0700 Subject: [PATCH 32/34] fix(websearch): reject invalid explicit search tool selections (#38113) * fix(websearch): reject invalid explicit search tool selections * refactor(websearch): simplify explicit search tool validation --- .../websearch_interception/handler.py | 50 +++-- .../test_websearch_interception_handler.py | 186 +++++++++++++++++- 2 files changed, 217 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 81310b9ddc3..aefd4fa3b47 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -416,15 +416,25 @@ class WebSearchInterceptionLogger(CustomLogger): if not tools: return None - if call_type in (CallTypes.responses, CallTypes.aresponses): - return self._convert_responses_tools(kwargs=kwargs, tools=tools) - - # Check if any tool is a web search tool (native or already LiteLLM standard) - has_websearch: Final = any(is_web_search_tool(t) for t in tools) - + is_responses_call: Final = call_type in (CallTypes.responses, CallTypes.aresponses) + has_websearch: Final = ( + any(is_web_search_tool_responses(tool) for tool in tools) + if is_responses_call + else any(is_web_search_tool(tool) for tool in tools) + ) if not has_websearch: return None + if self.search_tool_name: + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + llm_router = None + self._select_search_tool_from_router(llm_router=llm_router) + + if is_responses_call: + return self._convert_responses_tools(kwargs=kwargs, tools=tools) + verbose_logger.debug("WebSearchInterception: Converting native web_search tools to LiteLLM standard") # If the client sent an Anthropic-native web_search_* tool, mark the @@ -1631,9 +1641,7 @@ class WebSearchInterceptionLogger(CustomLogger): return None def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None": - if llm_router is None or not hasattr(llm_router, "search_tools"): - return None - search_tools: Final = list(getattr(llm_router, "search_tools") or []) + search_tools: Final = list(getattr(llm_router, "search_tools", []) or []) return self._select_search_tool_from_list(search_tools=search_tools, source="router") def _select_search_tool_from_list( @@ -1643,20 +1651,26 @@ class WebSearchInterceptionLogger(CustomLogger): ) -> "_SearchToolConfig | None": if self.search_tool_name: matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name] - if matching_tools: - search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider") - verbose_logger.debug( - "WebSearchInterception: Found search tool '%s' from %s with provider '%s'", - self.search_tool_name, - source, - search_provider, + if not matching_tools: + raise ValueError(f"Configured search tool '{self.search_tool_name}' was not found") + + selected_tool: Final = matching_tools[0] + litellm_params: Final = selected_tool.get("litellm_params") + selected_search_provider: Final = ( + litellm_params.get("search_provider") if isinstance(litellm_params, Mapping) else None + ) + if not isinstance(selected_search_provider, str) or not selected_search_provider.strip(): + raise ValueError( + f"Configured search tool '{self.search_tool_name}' does not define a valid search provider" ) - return matching_tools[0] + verbose_logger.debug( - "WebSearchInterception: Search tool '%s' not found in %s, falling back to first available or perplexity", + "WebSearchInterception: Found search tool '%s' from %s with provider '%s'", self.search_tool_name, source, + selected_search_provider, ) + return selected_tool if search_tools: first_tool: Final = search_tools[0] diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index f39f41a6d12..ec4bc1f49eb 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -14,7 +14,7 @@ from litellm.integrations.websearch_interception.handler import ( ) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, ProxyException, UserAPIKeyAuth -from litellm.types.utils import LlmProviders +from litellm.types.utils import CallTypes, LlmProviders def test_initialize_from_proxy_config(): @@ -230,6 +230,124 @@ async def test_execute_search_passes_selected_search_tool_litellm_params(monkeyp assert forwarded_kwargs["max_retries"] == 2 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_tools", "error"), + [ + pytest.param(None, "was not found", id="router-not-configured"), + pytest.param( + [{"search_tool_name": "other-search", "litellm_params": {"search_provider": "tavily"}}], + "was not found", + id="requested-tool-not-configured", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": "not-a-mapping"}], + "does not define a valid search provider", + id="invalid-parameters", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": {}}], + "does not define a valid search provider", + id="missing-provider", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": {"search_provider": " "}}], + "does not define a valid search provider", + id="whitespace-provider", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": {"search_provider": 123}}], + "does not define a valid search provider", + id="invalid-provider", + ), + ], +) +async def test_execute_search_rejects_invalid_explicit_search_tool(monkeypatch, search_tools, error): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(search_tool_name="parallel-search") + router = None if search_tools is None else MagicMock(search_tools=search_tools) + mock_asearch = AsyncMock() + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + with pytest.raises(ValueError, match=f"Configured search tool 'parallel-search' {error}"): + await logger._execute_search("what is litellm") + + mock_asearch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_execute_search_honors_explicit_parallel_search_tool(monkeypatch): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(search_tool_name="parallel-search") + router = MagicMock( + search_tools=[ + { + "search_tool_name": "other-search", + "litellm_params": {"search_provider": "tavily", "api_key": "other-key"}, + }, + { + "search_tool_name": "parallel-search", + "litellm_params": {"search_provider": "parallel_ai", "api_key": "parallel-key"}, + }, + ], + ) + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search("what is litellm") + + mock_asearch.assert_awaited_once_with( + query="what is litellm", + search_provider="parallel_ai", + api_key="parallel-key", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_tools", "expected_search_kwargs"), + [ + pytest.param(None, {"search_provider": "perplexity"}, id="router-not-configured"), + pytest.param( + [ + { + "search_tool_name": "first-search", + "litellm_params": {"search_provider": "tavily", "api_key": "first-key"}, + }, + { + "search_tool_name": "parallel-search", + "litellm_params": {"search_provider": "parallel_ai", "api_key": "parallel-key"}, + }, + ], + {"search_provider": "tavily", "api_key": "first-key"}, + id="first-configured-tool", + ), + ], +) +async def test_execute_search_preserves_implicit_provider_selection(monkeypatch, search_tools, expected_search_kwargs): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger() + router = None if search_tools is None else MagicMock(search_tools=search_tools) + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search("what is litellm") + + mock_asearch.assert_awaited_once_with(query="what is litellm", **expected_search_kwargs) + + @pytest.mark.asyncio async def test_execute_search_attributes_spend_to_the_calling_key(monkeypatch): """An intercepted search is billed and logged against the key that made the LLM request. @@ -397,6 +515,72 @@ async def test_execute_search_enforces_team_search_tool_permission(monkeypatch): mock_asearch.assert_not_awaited() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "web_search_tool"), + [ + pytest.param( + CallTypes.acompletion, + {"type": "web_search_20250305", "name": "web_search"}, + id="chat-completion", + ), + pytest.param(CallTypes.responses, {"type": "web_search"}, id="responses"), + pytest.param(CallTypes.aresponses, {"type": "web_search"}, id="async-responses"), + pytest.param( + CallTypes.anthropic_messages, + {"type": "web_search_20250305", "name": "web_search"}, + id="anthropic-messages", + ), + ], +) +async def test_deployment_hook_dispatcher_propagates_missing_explicit_search_tool( + monkeypatch, call_type, web_search_tool +): + import litellm + from litellm.proxy import proxy_server + from litellm.utils import async_pre_call_deployment_hook + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="parallel-search") + mock_asearch = AsyncMock() + kwargs = { + "model": "bedrock/claude-sonnet-4", + "tools": [web_search_tool], + "custom_llm_provider": "bedrock", + } + + monkeypatch.setattr( + proxy_server, + "llm_router", + MagicMock(search_tools=[{"search_tool_name": "other-search", "litellm_params": {"search_provider": "tavily"}}]), + ) + monkeypatch.setattr(litellm, "callbacks", [logger]) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + with pytest.raises(ValueError, match="Configured search tool 'parallel-search' was not found"): + await async_pre_call_deployment_hook(kwargs=kwargs, call_type=call_type.value) + + assert kwargs["tools"] == [web_search_tool] + mock_asearch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_deployment_hook_skips_explicit_tool_validation_for_non_search_responses(monkeypatch): + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="parallel-search") + monkeypatch.setattr(proxy_server, "llm_router", MagicMock(search_tools=[])) + + result = await logger.async_pre_call_deployment_hook( + kwargs={ + "tools": [{"type": "function", "name": "calculator"}], + "custom_llm_provider": "bedrock", + }, + call_type=CallTypes.aresponses, + ) + + assert result is None + + @pytest.mark.asyncio async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs(): """Test that async_pre_call_deployment_hook finds custom_llm_provider at top-level kwargs. From bfea8a8c19ac83e2f7457f46a4c494ead35642e9 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 31 Aug 2026 21:31:08 -0700 Subject: [PATCH 33/34] feat(shadow_eval): compare several auto-routers on one job's sampled traffic (#39028) --- .../migration.sql | 3 + .../litellm_proxy_extras/schema.prisma | 4 +- litellm/integrations/shadow_eval_logger.py | 151 ++++-- .../auto_router_endpoints.py | 53 ++- litellm/proxy/schema.prisma | 4 +- .../auto_router_endpoints.py | 75 ++- schema.prisma | 4 +- .../integrations/test_shadow_eval_logger.py | 180 +++++++- .../test_auto_router_endpoints.py | 129 +++++- .../_components/ShadowEvalSection.test.tsx | 126 ++++- .../_components/ShadowEvalSection.tsx | 348 +------------- .../_components/ShadowEvalStartForm.tsx | 432 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 35 +- 13 files changed, 1124 insertions(+), 420 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql new file mode 100644 index 00000000000..90b21205310 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "router_names" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; + +ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "router_name" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 01a607b68a9..7604ceadf7a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob { group_id String // legs of one job share this; the API's job id target_type String @default("key") // key | team | user target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows - router_name String // the auto-router under evaluation, in either direction + router_name String // first (often only) auto-router under evaluation; router_names is the full set + router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String @@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt { job_id String request_id String // the judged real request outcome String // real | shadow | tie | error + router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router tier String? // router's tier for the prompt, when classified real_model String? shadow_model String? diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 18bda0a9d55..27da785331a 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -1,8 +1,11 @@ """Shadow Eval Logger: samples a shadowed key's successful LLM requests (chat completions, Anthropic Messages, and Responses API surfaces, each normalized to chat shape), duplicates -each against the job's other arm in a detached task (the auto-router for a forward job, the -fixed baseline model for a reverse one), blind-judges real vs shadow, and appends one -``LiteLLM_ShadowEvalAttempt`` row (verdict or error) as the feature's only hot-path write. +each through every shadow arm in one detached task (each candidate auto-router for a +forward job, the fixed baseline model for a reverse one), blind-judges real vs each arm, +and appends one ``LiteLLM_ShadowEvalAttempt`` row per arm (verdict or error) as the +feature's only hot-path write. A multi-router job's arms therefore score the identical +sampled requests against the identical real responses, which is what makes their win +rates comparable head-to-head. Counts, status, and spend derive from those rows at read time, so nothing can disagree across pods or stop races; the hook reads active jobs through a short-TTL cache.""" @@ -498,12 +501,16 @@ def _decision_classifier_cost(metadata: Mapping[str, object]) -> float: return float(raw) if isinstance(raw, (int, float)) else 0.0 -def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool: - """Whether the router under evaluation served this request, which is what decides - the direction it belongs to. A forward job skips its own router's traffic, since - duplicating it would compare the router to itself: guaranteed ties, judge spend for - zero information. A reverse job samples exactly that traffic and nothing else.""" - return _routing_decision(request_metadata).get("router_model_name") == router_name +def _direction_admits(request_metadata: Mapping[str, object], job: "ActiveShadowEvalJob") -> bool: + """Whether this request belongs to the job's direction. A forward job skips traffic + any of its candidate routers served: duplicating a router's own request compares it + to itself (guaranteed ties), and judging a sibling against another candidate's live + response would score candidates against each other instead of against the incumbent. + A reverse job samples exactly its one router's traffic and nothing else.""" + routed_by: Final = _routing_decision(request_metadata).get("router_model_name") + if job.direction == "reverse": + return routed_by == job.router_name + return routed_by not in job.arm_router_names @dataclass(frozen=True, slots=True) @@ -546,6 +553,7 @@ class ActiveShadowEvalJob(BaseModel): id: str router_name: str + router_names: tuple[str, ...] = () direction: ShadowEvalDirection = "forward" baseline_model: str | None = None shadow_percentage: float @@ -567,12 +575,25 @@ class ActiveShadowEvalJob(BaseModel): raise ValueError("baseline_model is set for exactly the reverse jobs") return self + @model_validator(mode="after") + def _reverse_evaluates_one_router(self) -> "ActiveShadowEvalJob": + """A reverse row naming several routers is unsamplable (there is no one traffic + slice they share) and fails closed.""" + if self.direction == "reverse" and len(self.arm_router_names) > 1: + raise ValueError("a reverse job evaluates exactly one router") + return self + @property - def shadow_target(self) -> str: - """The model the duplicated arm calls: the router itself for a forward job, the - fixed baseline for a reverse one. Total because the validator above pins + def arm_router_names(self) -> tuple[str, ...]: + """The job's full router set; rows from before router_names existed hold it in + router_name alone. The one place that reading lives on the sampling side.""" + return self.router_names or (self.router_name,) + + def arm_target(self, arm_router: str) -> str: + """The model one duplicated arm calls: the candidate router itself for a forward + job, the fixed baseline for a reverse one. Total because the validator above pins baseline_model to reverse jobs and only those.""" - return self.baseline_model or self.router_name + return self.baseline_model or arm_router def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None: @@ -696,7 +717,7 @@ class ShadowEvalLogger(CustomLogger): now >= job.ends_at or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns or (job.max_budget is not None and job.spend >= job.max_budget) - or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse") + or not _direction_admits(request_metadata, job) ): continue if not _sample_hits(request_id, job.id, job.shadow_percentage): @@ -773,7 +794,10 @@ class ShadowEvalLogger(CustomLogger): if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS: self._record_funnel(job.id, "shed") continue - self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1 + # One start writes one attempt row per arm, and max_turns is a row + # ceiling, so admission must pre-count every arm or a multi-router + # job overshoots the valve N-fold within a cache generation. + self._job_starts[job.id] = self._job_starts.get(job.id, 0) + len(job.arm_router_names) self._inflight_shadow_tasks += 1 asyncio.create_task( self._run_shadow_eval( @@ -812,32 +836,74 @@ class ShadowEvalLogger(CustomLogger): shadow_params: Mapping[str, object], parent_metadata: Mapping[str, object], ) -> None: - """Budget gate -> shadow call -> blind judge -> one attempt row, and every exit - in exactly one coverage bucket: the gates that decline to spend on an admitted - sample (no DB to record into, an over-budget key, an unverifiable or exhausted - eval budget) count it withheld, so eligible traffic still reconciles as - not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits - above the dispatch so no provider spend happens without a place to record the - outcome, and the budget read lives here rather than in the success hook.""" + """Budget gates once per sampled request, then every router arm in turn: shadow + call -> blind judge -> one attempt row stamped with the arm. The gates that + decline to spend on an admitted sample (no DB to record into, an over-budget key, + an unverifiable or exhausted eval budget) count the REQUEST withheld before any + arm runs, so funnel counters stay per-request and a leg's eligible traffic still + reconciles as not_sampled + unjudgeable + shed + withheld + sampled requests, + where each sampled request writes one attempt row per arm. A budget crossed + mid-loop lets the remaining arms overshoot by one round, the same class of + overshoot as the samples already in flight when the cap is crossed. The prisma + gate sits above the dispatch so no provider spend happens without a place to + record the outcome, and the budget read lives here rather than in the success + hook.""" prisma: Final = self._prisma_provider() + if prisma is None: + self._record_funnel(job.id, "withheld") + return + if await _key_or_team_is_over_budget(parent_metadata): + self._record_funnel(job.id, "withheld") + return + if job.max_budget is not None: + try: + spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget) + except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it + verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e) + self._record_funnel(job.id, "withheld") + return + if spend >= job.max_budget: + self._record_funnel(job.id, "withheld") + return + for arm_router in job.arm_router_names: + await self._run_shadow_arm( + prisma=prisma, + job=job, + arm_router=arm_router, + request_id=request_id, + messages=messages, + real_text=real_text, + real_model=real_model, + real_cost=real_cost, + real_classifier_cost=real_classifier_cost, + real_cache_hit=real_cache_hit, + control_tier=control_tier, + shadow_params=shadow_params, + parent_metadata=parent_metadata, + ) + + async def _run_shadow_arm( + self, + prisma: "PrismaClient", + job: ActiveShadowEvalJob, + arm_router: str, + request_id: str, + messages: Sequence[Mapping[str, object]], + real_text: str, + real_model: str, + real_cost: float, + real_classifier_cost: float, + real_cache_hit: bool, + control_tier: str | None, + shadow_params: Mapping[str, object], + parent_metadata: Mapping[str, object], + ) -> None: + """One arm's pipeline: shadow call -> blind judge -> one attempt row, every exit + recording this arm's outcome, so one arm's fault never silences a sibling arm.""" try: - if prisma is None: - self._record_funnel(job.id, "withheld") - return - if await _key_or_team_is_over_budget(parent_metadata): - self._record_funnel(job.id, "withheld") - return - if job.max_budget is not None: - try: - spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget) - except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it - verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e) - self._record_funnel(job.id, "withheld") - return - if spend >= job.max_budget: - self._record_funnel(job.id, "withheld") - return - shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata) + shadow: Final = await self._call_router_shadow( + job.arm_target(arm_router), messages, shadow_params, parent_metadata + ) except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e) await self._record_attempt( @@ -845,6 +911,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=f"pipeline error: {e}", real_cost=real_cost, @@ -858,6 +925,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=shadow.error, shadow_cost=shadow.cost, @@ -882,6 +950,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=verdict.error, shadow=shadow, @@ -898,6 +967,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome=verdict.preference, shadow=shadow, real_model=real_model, @@ -916,6 +986,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=f"pipeline error: {e}", shadow=shadow, @@ -933,6 +1004,7 @@ class ShadowEvalLogger(CustomLogger): request_id: str, control_tier: str | None, *, + router_name: str, outcome: str, real_cost: float, real_classifier_cost: float, @@ -955,6 +1027,7 @@ class ShadowEvalLogger(CustomLogger): data={ # mutable-ok: Prisma payload "job_id": job.id, "request_id": request_id, + "router_name": router_name, "outcome": outcome, "tier": control_tier if job.direction == "reverse" else (shadow.tier if shadow else None), "real_model": real_model or None, diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 44b0cdcca2e..21e652114bc 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -833,7 +833,7 @@ def _judge_collisions_for_team( return tuple( (role, model) for role, model in ( - *_router_arm_models(llm_router, data.router_name), + *(arm for name in data.router_names for arm in _router_arm_models(llm_router, name)), *((("baseline", data.baseline_model),) if data.baseline_model is not None else ()), ) if judge & judge_target(llm_router, model, team_id).models @@ -904,7 +904,7 @@ class _AttemptAggRow(BaseModel): _ATTEMPT_AGG_ROWS: Final = TypeAdapter(list[_AttemptAggRow]) -_ATTEMPT_AGG_SELECT: Final = """ +_ATTEMPT_AGG_COLUMNS: Final = """ COUNT(*)::int AS turn_count, COUNT(*) FILTER (WHERE outcome = 'real')::int AS real_wins, COUNT(*) FILTER (WHERE outcome = 'shadow')::int AS shadow_wins, @@ -913,15 +913,34 @@ _ATTEMPT_AGG_SELECT: Final = """ COALESCE(SUM(real_cost + real_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS real_spend, COALESCE(SUM(shadow_cost + shadow_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS shadow_spend, COUNT(*) FILTER (WHERE real_cache_hit)::int AS cache_hit_turns +""" + +_ATTEMPT_AGG_SELECT: Final = ( + _ATTEMPT_AGG_COLUMNS + + """ FROM "LiteLLM_ShadowEvalAttempt" WHERE job_id = ANY($1::text[]) AND outcome != 'error' GROUP BY 1 """ +) _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT +# Attempt rows from before arm stamping carry no router_name; they belong to the job's +# own router, which the join reads off the leg. +_ATTEMPT_AGG_BY_ROUTER_SQL: Final = ( + "SELECT COALESCE(a.router_name, j.router_name) AS grp," + + _ATTEMPT_AGG_COLUMNS + + """ +FROM "LiteLLM_ShadowEvalAttempt" a +JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id +WHERE a.job_id = ANY($1::text[]) AND a.outcome != 'error' +GROUP BY 1 +""" +) + # These guards derive spend from attempt rows, the cross-pod authority; the sampler also # reads the live counter, so admission can stop before a row-based guard would fire (safe # direction, and mid-deploy rows from old pods price as judge-only until the deploy ends). @@ -1060,6 +1079,7 @@ class _LegRow(BaseModel): target_type: ShadowEvalTargetType target_id: str router_name: str + router_names: tuple[str, ...] = () direction: ShadowEvalDirection baseline_model: str | None = None judge_model: str @@ -1071,6 +1091,12 @@ class _LegRow(BaseModel): stopped_at: datetime | None = None stopped_by: str | None = None + @property + def arm_router_names(self) -> tuple[str, ...]: + """The job's full router set; rows from before router_names existed hold it in + router_name alone. The one place that reading lives on the endpoint side.""" + return self.router_names or (self.router_name,) + @field_validator("created_at", "ends_at", "stopped_at") @classmethod def _as_aware_utc(cls, value: datetime | None) -> datetime | None: @@ -1123,7 +1149,7 @@ def _group_response( ) for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id)) ), - router_name=first.router_name, + router_names=first.arm_router_names, direction=first.direction, baseline_model=first.baseline_model, judge_model=first.judge_model, @@ -1252,6 +1278,9 @@ async def _shadow_eval_results( for slice in _slices(by_leg) } ) + by_router: Final = _ATTEMPT_AGG_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_ROUTER_SQL, leg_ids) or () + ) total_turns: Final = sum(r.turn_count for r in by_tier) funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids) counted: Final = _FunnelTotalsRow.model_validate(funnel_rows[0]) if funnel_rows else None @@ -1261,6 +1290,7 @@ async def _shadow_eval_results( result: Final = ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), + by_router=_slices(by_router), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), sampled_real_spend=sum(r.real_spend for r in by_tier), @@ -1314,8 +1344,15 @@ async def start_shadow_eval( _require_admin_writer(user_api_key_dict, "start a shadow eval") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name): - raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router") + unconfigured: Final = tuple( + name + for name in data.router_names + if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, name) + ) + if unconfigured: + raise HTTPException( + status_code=400, detail=f"Not a configured auto-router: {', '.join(repr(n) for n in unconfigured)}" + ) token_rows: Final = ( await _verification_tokens(prisma_client).find_many( where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter @@ -1416,7 +1453,9 @@ async def start_shadow_eval( ends_at: Final = now + timedelta(days=data.duration_days) shared_config: Final = { # mutable-ok: Prisma payload "group_id": group_id, - "router_name": data.router_name, + # a pre-router_names pod samples router_name alone, so it must be a real arm + "router_name": data.router_names[0], + "router_names": list(data.router_names), # mutable-ok: Prisma payload "direction": data.direction, "baseline_model": data.baseline_model, "judge_model": data.judge_model, @@ -1477,7 +1516,7 @@ async def start_shadow_eval( ) for target_type, target_id in sorted(requested_targets) ), - router_name=data.router_name, + router_names=data.router_names, direction=data.direction, baseline_model=data.baseline_model, judge_model=data.judge_model, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 01a607b68a9..7604ceadf7a 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob { group_id String // legs of one job share this; the API's job id target_type String @default("key") // key | team | user target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows - router_name String // the auto-router under evaluation, in either direction + router_name String // first (often only) auto-router under evaluation; router_names is the full set + router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String @@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt { job_id String request_id String // the judged real request outcome String // real | shadow | tie | error + router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router tier String? // router's tier for the prompt, when classified real_model String? shadow_model String? diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index dfc45ccf9bf..88869a1edfb 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -251,8 +251,12 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" # Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that # fails before billing) never consumes spend budget, so it must terminate on count instead. +# A multi-router job writes one attempt row per router arm, so the valve is reached +# proportionally sooner; it is a safety valve, not a sample budget. SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000 +SHADOW_EVAL_MAX_ROUTERS: Final[int] = 4 + class StartShadowEvalRequest(BaseModel): """Start duplicating one or more targets' traffic for blind comparison against an auto-router. @@ -288,7 +292,24 @@ class StartShadowEvalRequest(BaseModel): "to across all their teams: JWT requests carrying their subject claim and virtual keys they own" ), ) - router_name: str = Field(description="The auto-router under evaluation, in either direction") + router_name: str | None = Field( + default=None, + description=( + "The auto-router under evaluation, in either direction: the single-router spelling of " + "router_names. Provide exactly one of the two fields" + ), + ) + router_names: tuple[str, ...] = Field( + default=(), + max_length=SHADOW_EVAL_MAX_ROUTERS, + description=( + "The auto-routers under evaluation, at most " + f"{SHADOW_EVAL_MAX_ROUTERS}. Every sampled request runs through every router listed and each " + "arm is judged independently against the same real response, so routers compare head-to-head " + "on identical traffic. More than one router requires direction 'forward'. After validation " + "this field always carries the full deduplicated set, whichever spelling the caller used" + ), + ) direction: ShadowEvalDirection = Field( default="forward", description=( @@ -332,7 +353,8 @@ class StartShadowEvalRequest(BaseModel): "Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " "the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval " "spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight " - "samples can overshoot the cap by one sampling cache window" + "samples can overshoot the cap by one sampling cache window. Every router arm draws from the " + "same per-target budget, so a multi-router job reaches it proportionally sooner" ), ) @@ -373,6 +395,23 @@ class StartShadowEvalRequest(BaseModel): raise ValueError("baseline_model is only meaningful when direction is 'reverse'") return self + @model_validator(mode="after") + def _resolve_router_set(self) -> "StartShadowEvalRequest": + """Whichever spelling the caller used, router_names leaves validation as the full + deduplicated set, so every downstream reader consumes one field.""" + if (self.router_name is None) == (not self.router_names): + raise ValueError("provide exactly one of router_name or router_names") + single: Final = () if self.router_name is None else (self.router_name,) + routers: Final = tuple(dict.fromkeys(self.router_names or single)) + if not all(name.strip() for name in routers): + raise ValueError("router names must be non-empty strings") + if len(routers) > 1 and self.direction == "reverse": + raise ValueError("a reverse job evaluates one router against baseline_model; pass a single router") + # A returned model_copy is ignored on the __init__ construction path, so the + # normalization must land as a self attribute store to hold for every caller. + self.router_names = routers + return self + class ShadowEvalSlice(BaseModel): """Judge outcomes for one slice of a job's verdicts: a router tier, one of the @@ -428,15 +467,28 @@ class ShadowEvalResult(BaseModel): "and in reverse the models the router itself picked" ) ) + by_router: tuple[ShadowEvalSlice, ...] = Field( + default=(), + description=( + "One slice per router arm, grouped on the router name. Every arm of a multi-router job is " + "judged against the same real responses over the same sampled requests, so these slices " + "compare routers head-to-head: like-for-like win rates and spends on identical traffic. " + "Verdicts from before arm stamping existed count toward the job's own router" + ), + ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float sampled_real_spend: float = Field( default=0.0, - description="USD the real arm billed across all judged turns, cache-served turns excluded", + description=( + "USD the real arm billed across all judged turns, cache-served turns excluded. A judged turn " + "is one (request, router arm) verdict, so a multi-router job counts the real response once per " + "arm it was judged against; per-router comparisons read by_router" + ), ) sampled_shadow_spend: float = Field( default=0.0, - description="USD the shadow arm billed across the same turns, judge excluded, like for like", + description="USD the shadow arms billed across the same turns, judge excluded, like for like", ) not_sampled_count: int | None = Field( default=None, @@ -540,7 +592,13 @@ class ShadowEvalJobResponse(BaseModel): min_length=1, description="The targets whose traffic this job evaluates, and only theirs, each with its own budget", ) - router_name: str + router_names: tuple[str, ...] = Field( + min_length=1, + description=( + "Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of " + "traffic and judge every arm against the same real responses" + ), + ) direction: ShadowEvalDirection = "forward" baseline_model: str | None = None judge_model: str @@ -562,6 +620,13 @@ class ShadowEvalJobResponse(BaseModel): last_error: str | None = Field(default=None, description="Most recent attempt error; detail endpoint only") results: ShadowEvalResult | None = Field(default=None, description="Stratified verdicts; detail endpoint only") + @computed_field + @property + def router_name(self) -> str: + """The first router, kept for callers that predate router_names; derived so the + two fields can never disagree.""" + return self.router_names[0] + @computed_field @property def status(self) -> ShadowEvalStatus: diff --git a/schema.prisma b/schema.prisma index 01a607b68a9..7604ceadf7a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob { group_id String // legs of one job share this; the API's job id target_type String @default("key") // key | team | user target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows - router_name String // the auto-router under evaluation, in either direction + router_name String // first (often only) auto-router under evaluation; router_names is the full set + router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String @@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt { job_id String request_id String // the judged real request outcome String // real | shadow | tie | error + router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router tier String? // router's tier for the prompt, when classified real_model String? shadow_model String? diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 1af3dd3f613..5628d69de26 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -65,6 +65,7 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash target_type=target_type, target_id=target_id, router_name=job.router_name, + router_names=job.router_names, direction=job.direction, baseline_model=job.baseline_model, shadow_percentage=job.shadow_percentage, @@ -81,6 +82,7 @@ def _router( shadow_text="shadow answer", judge_json='{"preference": "A", "confidence": 0.9, "reasoning": "x"}', classifier_cost=None, + sibling_router_texts=None, ): """One mock router serving the shadow call first, the judge call second, told apart by the internal-origin stamp rather than the model, since a reverse job's shadow arm names @@ -100,6 +102,15 @@ def _router( decision["classifier_cost"] = classifier_cost kwargs["metadata"]["routing_decision"] = decision return {"choices": [{"message": {"content": shadow_text}}], "usage": {"completion_tokens": 5}} + if sibling_router_texts and kwargs["model"] in sibling_router_texts: + kwargs["metadata"]["routing_decision"] = { + "tier_label": "MEDIUM", + "routed_model": f"{kwargs['model']}-pick", + } + return { + "choices": [{"message": {"content": sibling_router_texts[kwargs["model"]]}}], + "usage": {"completion_tokens": 5}, + } return ModelResponse( model=kwargs["model"], choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": shadow_text}}], @@ -1210,29 +1221,36 @@ class TestJobValidation: {"direction": "reverse"}, {"baseline_model": "baseline-model"}, {"direction": "sideways", "baseline_model": "baseline-model"}, + {"direction": "reverse", "baseline_model": "baseline-model", "router_names": ("a", "b")}, ], - ids=["reverse-without-baseline", "forward-with-baseline", "unknown-direction"], + ids=["reverse-without-baseline", "forward-with-baseline", "unknown-direction", "reverse-with-router-set"], ) def test_unsamplable_shapes_are_rejected(self, overrides): with pytest.raises(ValidationError): _job(**overrides) - def test_shadow_target_follows_direction(self): - assert _job().shadow_target == "my-router" - assert _reverse_job().shadow_target == "baseline-model" + def test_arm_target_follows_direction(self): + assert _job().arm_target("my-router") == "my-router" + assert _reverse_job().arm_target("my-router") == "baseline-model" + + def test_rows_from_before_router_names_carry_their_set_in_router_name(self): + assert _job().arm_router_names == ("my-router",) + assert _job(router_names=("my-router", "alt-router")).arm_router_names == ("my-router", "alt-router") @pytest.mark.asyncio class TestDirection: @pytest.mark.parametrize( - "job,routed_by,sampled", + "job,routed_by,attempt_rows", [ - (_job(), None, True), - (_job(), "my-router", False), - (_job(), "other-router", True), - (_reverse_job(), "my-router", True), - (_reverse_job(), None, False), - (_reverse_job(), "other-router", False), + (_job(), None, 1), + (_job(), "my-router", 0), + (_job(), "other-router", 1), + (_reverse_job(), "my-router", 1), + (_reverse_job(), None, 0), + (_reverse_job(), "other-router", 0), + (_job(router_names=("my-router", "alt-router")), "alt-router", 0), + (_job(router_names=("my-router", "alt-router")), "other-router", 2), ], ids=[ "forward-samples-unrouted", @@ -1241,20 +1259,24 @@ class TestDirection: "reverse-samples-its-own-router", "reverse-skips-unrouted", "reverse-skips-another-router", + "forward-skips-any-candidates-own-traffic", + "forward-multi-samples-once-per-arm", ], ) - async def test_direction_decides_which_traffic_is_sampled(self, job, routed_by, sampled): + async def test_direction_decides_which_traffic_is_sampled(self, job, routed_by, attempt_rows): """The two directions partition the key's traffic: whatever one samples, the other - skips, so a key running both never judges the same turn twice for the same reason.""" + skips, so a key running both never judges the same turn twice for the same reason. + A multi-router job extends the forward skip to every candidate: a request one + candidate served must not be judged as the incumbent against another candidate.""" prisma = _prisma() - logger = _logger(router=_router(), prisma=prisma, jobs=(job,)) + logger = _logger(router=_router(sibling_router_texts={"alt-router": "alt answer"}), prisma=prisma, jobs=(job,)) await logger.async_log_success_event( _success_kwargs(request_metadata=_routed_by(routed_by) if routed_by else {}), RESPONSE, None, None ) await _drain(logger) - assert prisma.db.litellm_shadowevalattempt.create.await_count == int(sampled) + assert prisma.db.litellm_shadowevalattempt.create.await_count == attempt_rows async def test_reverse_duplicates_against_the_baseline_model(self): prisma = _prisma() @@ -1316,6 +1338,134 @@ class TestDirection: assert logger._job_starts == {"forward-job": 1, "reverse-job": 1} +@pytest.mark.asyncio +class TestMultiRouterArms: + async def test_every_arm_judges_the_same_request_and_stamps_its_own_row(self): + """One sampled request, one row per candidate router, both judged against the same + real response: the paired comparison that makes multi-router win rates comparable.""" + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(router_names=("my-router", "alt-router")), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.001, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + assert [row["router_name"] for row in rows] == ["my-router", "alt-router"] + assert {row["request_id"] for row in rows} == {"req-1"} + assert [row["shadow_model"] for row in rows] == ["cheap-model", "alt-router-pick"] + assert all(row["outcome"] in ("real", "shadow", "tie") for row in rows) + assert all(row["real_cost"] == 0.001 for row in rows) + + async def test_a_single_router_job_stamps_its_router_on_the_row(self): + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["router_name"] == "my-router" + + async def test_one_arms_failure_never_silences_the_sibling(self): + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + healthy = router.acompletion.side_effect + + async def first_arm_explodes(**kwargs): + if kwargs["model"] == "my-router": + raise RuntimeError("provider exploded") + return await healthy(**kwargs) + + router.acompletion.side_effect = first_arm_explodes + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(router_names=("my-router", "alt-router")), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + assert [row["router_name"] for row in rows] == ["my-router", "alt-router"] + assert rows[0]["outcome"] == "error" + assert "provider exploded" in rows[0]["error"] + assert rows[1]["outcome"] in ("real", "shadow", "tie") + + async def test_the_turn_valve_counts_every_arm_a_start_will_write(self): + """max_turns is a row ceiling and one sampled request writes one row per arm, so + admission pre-counts the arms: a two-arm job with two turns of budget admits one + request, not two.""" + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + logger = _logger( + router=router, prisma=prisma, jobs=(_job(router_names=("my-router", "alt-router"), max_turns=2),) + ) + + await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None) + await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None) + await _drain(logger) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + assert {row["request_id"] for row in rows} == {"req-1"} + assert len(rows) == 2 + + async def test_a_withheld_request_runs_no_arm_and_counts_once(self): + """The budget gates run once per sampled request, before any arm: funnel counters + stay per-request, so coverage math is arm-count independent.""" + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(router_names=("my-router", "alt-router"), max_budget=1.0, spend=2.0), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + assert logger._test_funnel == [("job-1", "withheld")] + + @pytest.mark.asyncio class TestActiveJobsFailClosed: async def test_a_row_the_sampler_cannot_read_is_dropped_not_guessed(self): diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index a74aa553449..c525af84511 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -881,6 +881,7 @@ def _leg_record(**overrides: object) -> MagicMock: "target_type": "key", "target_id": "key-hash", "router_name": "my-router", + "router_names": (), "direction": "forward", "baseline_model": None, "judge_model": "anthropic/claude-sonnet-5", @@ -931,6 +932,7 @@ def _shadow_prisma( legs=(), agg_rows=None, by_leg_rows=None, + by_router_rows=None, known_keys=("key-hash", "key-hash-2"), key_teams=None, known_teams=None, @@ -1030,6 +1032,7 @@ def _shadow_prisma( "target_type", "target_id", "router_name", + "router_names", "direction", "baseline_model", "judge_model", @@ -1065,6 +1068,8 @@ def _shadow_prisma( return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] if "SELECT job_id AS grp" in sql: return by_leg_rows if by_leg_rows is not None else [] + if "COALESCE(a.router_name" in sql: + return by_router_rows if by_router_rows is not None else [] if 'FROM "LiteLLM_ShadowEvalFunnel"' in sql: return prisma.funnel_rows return agg_rows if agg_rows is not None else [] @@ -1121,7 +1126,15 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("key", "key-hash-2")] - assert len({frozenset((k, v) for k, v in row.items() if k not in ("target_id", "id")) for row in rows}) == 1 + assert ( + len( + { + frozenset((k, tuple(v) if isinstance(v, list) else v) for k, v in row.items() if k not in ("target_id", "id")) + for row in rows + } + ) + == 1 + ) assert len({row["id"] for row in rows}) == len(rows) assert len({row["group_id"] for row in rows}) == 1 assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows) @@ -1138,6 +1151,63 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert all(target.max_turns == SHADOW_EVAL_TURN_VALVE for target in response.targets) +@pytest.mark.asyncio +async def test_start_shadow_eval_multi_router_writes_the_set_on_every_leg(monkeypatch: pytest.MonkeyPatch): + """A multi-router job stores the full set in router_names and the first router in + router_name, so a rolling-deploy pod that predates router_names still runs a valid + single-arm eval and its unstamped attempt rows attribute to that first router.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval( + _start_request(router_name=None, router_names=("my-router", "classifier-router")), ADMIN + ) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert all(row["router_name"] == "my-router" for row in rows) + assert all(row["router_names"] == ["my-router", "classifier-router"] for row in rows) + assert response.router_names == ("my-router", "classifier-router") + assert response.router_name == "my-router" + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_an_unconfigured_router_in_the_set(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException, match="not-a-router") as exc: + await start_shadow_eval(_start_request(router_name=None, router_names=("my-router", "not-a-router")), ADMIN) + + assert exc.value.status_code == 400 + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_judge_collision_is_found_on_every_router_of_the_set(monkeypatch: pytest.MonkeyPatch): + """The judge-as-candidate guard walks every candidate router: a judge that serves an + arm of the SECOND router still poisons the whole job's win rates.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException, match="also an arm") as exc: + await start_shadow_eval(_start_request(router_name=None, router_names=("my-router", "sonnet-router")), ADMIN) + + assert exc.value.status_code == 400 + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + @pytest.mark.asyncio async def test_start_shadow_eval_rejects_an_uncredentialed_sdk_judge(monkeypatch: pytest.MonkeyPatch) -> None: import litellm @@ -1798,6 +1868,63 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"} +@pytest.mark.asyncio +async def test_get_shadow_eval_job_slices_results_per_router(monkeypatch: pytest.MonkeyPatch): + """A multi-router job's detail carries one slice per arm, aggregated by the arm + stamped on each attempt row, with unstamped legacy rows attributed to the job's own + router by the read (the COALESCE against the leg's router_name).""" + import litellm.proxy.proxy_server as proxy_server + + def agg(grp: str, wins: int) -> dict[str, object]: + return { + "grp": grp, + "turn_count": 4, + "real_wins": 4 - wins, + "shadow_wins": wins, + "ties": 0, + "avg_confidence": 0.8, + "real_spend": 0.08, + "shadow_spend": 0.02, + "cache_hit_turns": 0, + } + + prisma = _shadow_prisma( + legs=[_leg_record(router_names=("my-router", "alt-router"))], + agg_rows=[agg("SIMPLE", 3)], + by_router_rows=[agg("my-router", 1), agg("alt-router", 3)], + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + assert response.router_names == ("my-router", "alt-router") + assert response.router_name == "my-router" + assert [(s.group, s.shadow_win_rate_pct) for s in response.results.by_router] == [ + ("my-router", 25.0), + ("alt-router", 75.0), + ] + router_sql = next( + call.args[0] for call in prisma.db.query_raw.await_args_list if "COALESCE(a.router_name" in call.args[0] + ) + assert "COALESCE(a.router_name, j.router_name)" in router_sql + assert 'JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id' in router_sql + assert "a.job_id = ANY($1::text[])" in router_sql + + +@pytest.mark.asyncio +async def test_job_responses_resolve_router_names_with_legacy_fallback(monkeypatch: pytest.MonkeyPatch): + """Rows from before router_names existed carry their whole set in router_name.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(router_names=())]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + assert response.router_names == ("my-router",) + assert response.router_name == "my-router" + + @pytest.mark.asyncio async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index 8b397f20552..64e03985f57 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -102,6 +102,7 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ job_id: "job-1", status: "running", router_name: "claude-auto", + router_names: ["claude-auto"], direction: "forward", baseline_model: null, judge_model: "anthropic/claude-sonnet-5", @@ -436,7 +437,7 @@ describe("ShadowEvalSection", () => { await user.click(within(keyList).getByText("prod-alpha")); await user.click(keyInput); await user.click(within(keyList).getByText("staging-beta")); - await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); await user.click(await screen.findByText("gpt-auto")); expect(screen.getByText("Start shadow eval")).toBeDisabled(); @@ -449,7 +450,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha", "hash-beta"], team_ids: [], user_ids: [], - router_name: "gpt-auto", + router_names: ["gpt-auto"], direction: "forward", shadow_percentage: 10, duration_days: 7, @@ -469,7 +470,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByPlaceholderText("Search teams by alias")); const teamList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(teamList).getByText("engineering")); - await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); await user.click(await screen.findByText("gpt-auto")); await user.click(screen.getByPlaceholderText("Select a judge model")); await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); @@ -479,7 +480,7 @@ describe("ShadowEvalSection", () => { api_key_ids: [], team_ids: ["team-eng"], user_ids: [], - router_name: "gpt-auto", + router_names: ["gpt-auto"], direction: "forward", shadow_percentage: 10, duration_days: 7, @@ -501,7 +502,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByPlaceholderText("Search keys by alias")); const keyList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(keyList).getByText("prod-alpha")); - await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); await user.click(await screen.findByText("gpt-auto")); await user.click(screen.getByPlaceholderText("Select a judge model")); await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); @@ -517,7 +518,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha"], team_ids: [], user_ids: [], - router_name: "gpt-auto", + router_names: ["gpt-auto"], direction: "reverse", baseline_model: "prod-claude", shadow_percentage: 10, @@ -528,6 +529,119 @@ describe("ShadowEvalSection", () => { expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); + it("submits every picked auto-router so one job compares them on the same traffic", async () => { + const user = userEvent.setup(); + const { start } = mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Search keys by alias")); + const keyList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(keyList).getByText("prod-alpha")); + const routerInput = screen.getByPlaceholderText("Select up to 4 auto-routers"); + await user.click(routerInput); + await user.click(await screen.findByText("gpt-auto")); + await user.click(routerInput); + await user.click(await screen.findByText("claude-auto")); + expect( + screen.getByText("Every router sees the same sampled requests, judged against the same live responses"), + ).toBeInTheDocument(); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByText("Start shadow eval")); + + const expectedBody = { + api_key_ids: ["hash-alpha"], + team_ids: [], + user_ids: [], + router_names: ["gpt-auto", "claude-auto"], + direction: "forward", + shadow_percentage: 10, + duration_days: 7, + max_budget: 10, + judge_model: "anthropic/claude-sonnet-5", + }; + expect(start.mutate).toHaveBeenCalledWith(expectedBody); + }); + + it("blocks starting a reverse job with more than one router and says why", async () => { + const user = userEvent.setup(); + mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Search keys by alias")); + const keyList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(keyList).getByText("prod-alpha")); + const routerInput = screen.getByPlaceholderText("Select up to 4 auto-routers"); + await user.click(routerInput); + await user.click(await screen.findByText("gpt-auto")); + await user.click(routerInput); + await user.click(await screen.findByText("claude-auto")); + await user.click(screen.getByText("Adoption check: key's traffic vs the router")); + await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByPlaceholderText("Select a baseline model")); + await user.click(screen.getByRole("option", { name: /prod-claude/ })); + + expect(screen.getByText("A regression check compares one router to its baseline")).toBeInTheDocument(); + expect(screen.getByText("Start shadow eval")).toBeDisabled(); + }); + + it("renders a per-router comparison table only when the job ran several routers", () => { + const routerSlice = (group: string, wins: number) => ({ + group, + turn_count: 20, + real_win_rate_pct: 100 - wins - 10, + shadow_win_rate_pct: wins, + tie_rate_pct: 10, + avg_judge_confidence: 0.8, + real_spend: 0.4, + shadow_spend: 0.2, + cache_hit_turns: 0, + }); + const base = job(); + const multi = job({ + router_names: ["claude-auto", "gpt-auto"], + results: { ...base.results!, by_router: [routerSlice("claude-auto", 40), routerSlice("gpt-auto", 70)] }, + }); + mockHooks({ jobs: [multi], detailsById: { "job-1": multi } }); + render(); + + expect(screen.getByText("Router")).toBeInTheDocument(); + const rows = screen.getAllByRole("row").map((row) => row.textContent ?? ""); + expect(rows.some((text) => text.includes("claude-auto") && text.includes("40.0%"))).toBe(true); + expect(rows.some((text) => text.includes("gpt-auto") && text.includes("70.0%"))).toBe(true); + expect( + screen.getByText( + (_, element) => + element?.textContent === "Shadowing 10% of prod-alpha traffic via claude-auto, gpt-auto" && + element.tagName === "P", + ), + ).toBeInTheDocument(); + }); + + it("renders a job from an older proxy that predates router_names", () => { + const legacy = { ...job(), router_names: undefined } as unknown as ShadowEvalJob; + mockHooks({ jobs: [legacy], detailsById: { "job-1": legacy } }); + render(); + + expect( + screen.getByText( + (_, element) => + element?.textContent === "Shadowing 10% of prod-alpha traffic via claude-auto" && element.tagName === "P", + ), + ).toBeInTheDocument(); + }); + + it("keeps the per-router table hidden for a single-router job", () => { + const base = job(); + const single = job({ results: { ...base.results!, by_router: [] } }); + mockHooks({ jobs: [single], detailsById: { "job-1": single } }); + render(); + + expect(screen.queryByText("Router")).not.toBeInTheDocument(); + }); + it("flips the arm labels and headline for a reverse job's results", () => { const j = job({ direction: "reverse", baseline_model: "openai/gpt-4o" }); mockHooks({ jobs: [j], detailsById: { "job-1": j } }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index ec145bc617a..c66d74074c2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -2,32 +2,21 @@ import React, { useMemo, useState } from "react"; -import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; -import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; -import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; -import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; -import TeamMultiSelect from "@/components/common_components/team_multi_select"; -import { userOptionLabel } from "@/components/common_components/UserDropdown"; -import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { CircleHelp } from "lucide-react"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card } from "@/components/ui/card"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { ApiError } from "@/lib/http/client"; import { usd } from "./costOptimizationUtils"; +import { StartForm } from "./ShadowEvalStartForm"; import { useShadowEvalJob, useShadowEvalJobs, - useStartShadowEval, useStopShadowEval, type ShadowEvalJob, type ShadowEvalJobTarget, @@ -96,17 +85,19 @@ const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string = return target.stopped_at != null ? "stopped" : "running"; }; +const jobRouters = (job: ShadowEvalJob): string => (job.router_names ?? [job.router_name]).join(", "); + const jobHeadline = (job: ShadowEvalJob): React.ReactNode => job.direction === "reverse" ? ( <> - Comparing {job.router_name} to{" "} + Comparing {jobRouters(job)} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} {shadowedTargetsLabel(job)} traffic ) : ( <> Shadowing {job.shadow_percentage}% of {shadowedTargetsLabel(job)}{" "} - traffic via {job.router_name} + traffic via {jobRouters(job)} ); @@ -352,6 +343,11 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ + {(results.by_router ?? []).length > 1 && ( +
      + +
      + )} {results.by_current_model.length > 0 && ( { - const { data: costMap } = useModelCostMap(); - return useMemo(() => { - if (!costMap) return []; - const chatModels = Object.entries(costMap as Record) - .filter(([, value]) => value?.mode === "chat" && value?.litellm_provider) - .map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`)); - return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b)); - }, [costMap]); -}; - -const useJudgeModelOptions = (): SearchSelectOption[] => { - const chatModels = useChatModelNames(); - return useMemo(() => { - const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({ - label: model, - value: model, - sublabel: "Recommended", - })); - const pinnedNames = new Set(RECOMMENDED_JUDGE_MODELS); - const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model })); - return [...pinned, ...rest]; - }, [chatModels]); -}; - -const useBaselineModelOptions = (): SearchSelectOption[] => { - const configuredGroups = usePlainModelGroups(); - const chatModels = useChatModelNames(); - return useMemo(() => { - const configured = [...configuredGroups] - .toSorted((a, b) => a.localeCompare(b)) - .map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" })); - const rest = chatModels - .filter((model) => !configuredGroups.has(model)) - .map((model) => ({ label: model, value: model })); - return [...configured, ...rest]; - }, [configuredGroups, chatModels]); -}; - -const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [ - { value: "forward", label: "Adoption check: key's traffic vs the router" }, - { value: "reverse", label: "Regression check: router's picks vs a baseline" }, -] as const; - -const START_FORM_DESCRIPTION: Record = { - forward: - "Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.", - reverse: - "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.", -}; - -const DURATION_OPTIONS = [ - { value: "1", label: "1 day" }, - { value: "3", label: "3 days" }, - { value: "7", label: "7 days" }, - { value: "14", label: "14 days" }, - { value: "30", label: "30 days" }, -] as const; - -const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({ - label, - htmlFor, - className, - children, -}) => ( -
      - - {children} -
      -); - -const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => void }> = ({ value, onChange }) => { - const [search, setSearch] = useState(""); - const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, { - selectedKeyAlias: search || null, - }); - const options = useMemo( - () => - (data?.pages ?? []) - .flatMap((page) => page.keys) - .map((key) => ({ - label: key.key_alias || key.key_name || key.token, - value: key.token, - sublabel: key.token, - })), - [data], - ); - return ( - void fetchNextPage()} - hasNextPage={hasNextPage} - isFetchingNextPage={isFetchingNextPage} - isLoading={isPending} - placeholder="Search keys by alias" - emptyText="No matching keys" - errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined} - /> - ); -}; - -const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => { - const [search, setSearch] = useState(""); - const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers( - 50, - search || undefined, - ); - const options = useMemo( - () => - Array.from( - new Map( - (data?.pages ?? []) - .flatMap((page) => page.users) - .map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const), - ).values(), - ), - [data], - ); - return ( - void fetchNextPage()} - hasNextPage={hasNextPage} - isFetchingNextPage={isFetchingNextPage} - isLoading={isPending} - placeholder="Search users by email" - emptyText="No matching users" - errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined} - /> - ); -}; - -const StartForm: React.FC = () => { - const { accessToken } = useAuthorized(); - const [apiKeyIds, setApiKeyIds] = useState([]); - const [teamIds, setTeamIds] = useState([]); - const [userIds, setUserIds] = useState([]); - const [routerName, setRouterName] = useState(""); - const [direction, setDirection] = useState("forward"); - const [baselineModel, setBaselineModel] = useState(""); - const [percentage, setPercentage] = useState("10"); - const [durationDays, setDurationDays] = useState("7"); - const [judgeModel, setJudgeModel] = useState(""); - const [maxBudget, setMaxBudget] = useState("10"); - const { data: autoRouters } = useAutoRouters(); - const judgeModelOptions = useJudgeModelOptions(); - const baselineModelOptions = useBaselineModelOptions(); - const start = useStartShadowEval(); - - const routerOptions = useMemo(() => { - const names = new Set( - (autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)), - ); - return [...names].toSorted().map((name) => ({ label: name, value: name })); - }, [autoRouters]); - - const parsedPct = Number.parseFloat(percentage); - const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; - const parsedMaxBudget = Number.parseFloat(maxBudget); - const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; - const baselinePicked = direction === "forward" || baselineModel !== ""; - const targetsPicked = apiKeyIds.length + teamIds.length + userIds.length > 0; - const filled = targetsPicked && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; - const boundsValid = percentageValid && maxBudgetValid; - const valid = Boolean(accessToken) && filled && boundsValid; - const handleStart = () => { - const startBody = { - api_key_ids: apiKeyIds, - team_ids: teamIds, - user_ids: userIds, - router_name: routerName, - direction, - ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), - shadow_percentage: parsedPct, - duration_days: Number.parseInt(durationDays, 10), - max_budget: parsedMaxBudget, - judge_model: judgeModel, - }; - start.mutate(startBody); - }; - - return ( - - - Start a shadow eval -

      {START_FORM_DESCRIPTION[direction]}

      -       - -
      - - - - - - - - - - - - - - - - -
      - setPercentage(e.target.value)} - /> - % of traffic -
      -
      - {percentage.trim() !== "" && !percentageValid && ( -

      Enter a value from 0.1 to 100

      - )} -
      -       - - - - -
      - $ - setMaxBudget(e.target.value)} - /> - max shadow + judge spend, per target -
      - {maxBudget.trim() !== "" && !maxBudgetValid && ( -

      Enter a value from 0.01 to 10000

      - )} -       - {direction === "reverse" && ( - - - - )} - - - -
      - -       -       - ); -}; - const previousSummary = (job: ShadowEvalJob): string => { const results = job.results; if (results) return pct(routerMatchedOrBeatPct(job.direction, results)); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx new file mode 100644 index 00000000000..f96910a4ad6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx @@ -0,0 +1,432 @@ +"use client"; + +import React, { useMemo, useState } from "react"; + +import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; +import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; +import TeamMultiSelect from "@/components/common_components/team_multi_select"; +import { userOptionLabel } from "@/components/common_components/UserDropdown"; +import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; + +import { useStartShadowEval, type ShadowEvalJob } from "./useShadowEval"; + +type ShadowEvalDirection = ShadowEvalJob["direction"]; + +const MAX_ROUTERS = 4; + +const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const; + +interface CostMapEntry { + litellm_provider?: string; + mode?: string; +} + +const useChatModelNames = (): string[] => { + const { data: costMap } = useModelCostMap(); + return useMemo(() => { + if (!costMap) return []; + const chatModels = Object.entries(costMap as Record) + .filter(([, value]) => value?.mode === "chat" && value?.litellm_provider) + .map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`)); + return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b)); + }, [costMap]); +}; + +const useJudgeModelOptions = (): SearchSelectOption[] => { + const chatModels = useChatModelNames(); + return useMemo(() => { + const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({ + label: model, + value: model, + sublabel: "Recommended", + })); + const pinnedNames = new Set(RECOMMENDED_JUDGE_MODELS); + const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model })); + return [...pinned, ...rest]; + }, [chatModels]); +}; + +const useBaselineModelOptions = (): SearchSelectOption[] => { + const configuredGroups = usePlainModelGroups(); + const chatModels = useChatModelNames(); + return useMemo(() => { + const configured = [...configuredGroups] + .toSorted((a, b) => a.localeCompare(b)) + .map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" })); + const rest = chatModels + .filter((model) => !configuredGroups.has(model)) + .map((model) => ({ label: model, value: model })); + return [...configured, ...rest]; + }, [configuredGroups, chatModels]); +}; + +const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [ + { value: "forward", label: "Adoption check: key's traffic vs the router" }, + { value: "reverse", label: "Regression check: router's picks vs a baseline" }, +] as const; + +const START_FORM_DESCRIPTION: Record = { + forward: + "Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.", + reverse: + "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.", +}; + +const DURATION_OPTIONS = [ + { value: "1", label: "1 day" }, + { value: "3", label: "3 days" }, + { value: "7", label: "7 days" }, + { value: "14", label: "14 days" }, + { value: "30", label: "30 days" }, +] as const; + +const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({ + label, + htmlFor, + className, + children, +}) => ( +
      + + {children} +
      +); + +const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => void }> = ({ value, onChange }) => { + const [search, setSearch] = useState(""); + const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, { + selectedKeyAlias: search || null, + }); + const options = useMemo( + () => + (data?.pages ?? []) + .flatMap((page) => page.keys) + .map((key) => ({ + label: key.key_alias || key.key_name || key.token, + value: key.token, + sublabel: key.token, + })), + [data], + ); + return ( + void fetchNextPage()} + hasNextPage={hasNextPage} + isFetchingNextPage={isFetchingNextPage} + isLoading={isPending} + placeholder="Search keys by alias" + emptyText="No matching keys" + errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined} + /> + ); +}; + +const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => { + const [search, setSearch] = useState(""); + const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers( + 50, + search || undefined, + ); + const options = useMemo( + () => + Array.from( + new Map( + (data?.pages ?? []) + .flatMap((page) => page.users) + .map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const), + ).values(), + ), + [data], + ); + return ( + void fetchNextPage()} + hasNextPage={hasNextPage} + isFetchingNextPage={isFetchingNextPage} + isLoading={isPending} + placeholder="Search users by email" + emptyText="No matching users" + errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined} + /> + ); +}; + +const RouterField: React.FC<{ + options: SearchSelectOption[]; + routerNames: string[]; + onChange: (names: string[]) => void; + direction: ShadowEvalDirection; +}> = ({ options, routerNames, onChange, direction }) => ( + + + {routerNames.length > MAX_ROUTERS && ( +

      Pick at most {MAX_ROUTERS} auto-routers

      + )} + {direction === "reverse" && routerNames.length > 1 && ( +

      A regression check compares one router to its baseline

      + )} + {direction === "forward" && routerNames.length > 1 && ( +

      + Every router sees the same sampled requests, judged against the same live responses +

      + )} +       +); + +interface StartFormValidityInputs { + accessToken: string | null | undefined; + apiKeyIds: string[]; + teamIds: string[]; + userIds: string[]; + routerNames: string[]; + direction: ShadowEvalDirection; + baselineModel: string; + judgeModel: string; + percentage: string; + maxBudget: string; +} + +const startFormValidity = (inputs: StartFormValidityInputs) => { + const parsedPct = Number.parseFloat(inputs.percentage); + const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; + const parsedMaxBudget = Number.parseFloat(inputs.maxBudget); + const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; + const baselinePicked = inputs.direction === "forward" || inputs.baselineModel !== ""; + const targetsPicked = inputs.apiKeyIds.length + inputs.teamIds.length + inputs.userIds.length > 0; + const routerCountValid = inputs.routerNames.length >= 1 && inputs.routerNames.length <= MAX_ROUTERS; + const routersMatchDirection = inputs.direction === "forward" || inputs.routerNames.length === 1; + const routersValid = routerCountValid && routersMatchDirection; + const modelsPicked = routersValid && inputs.judgeModel !== "" && baselinePicked; + const filled = targetsPicked && modelsPicked; + const boundsValid = percentageValid && maxBudgetValid; + const valid = Boolean(inputs.accessToken) && filled && boundsValid; + return { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid }; +}; + +interface StartBodyInputs { + apiKeyIds: string[]; + teamIds: string[]; + userIds: string[]; + routerNames: string[]; + direction: ShadowEvalDirection; + baselineModel: string; + shadowPercentage: number; + durationDays: number; + maxBudget: number; + judgeModel: string; +} + +const buildStartBody = (inputs: StartBodyInputs) => ({ + api_key_ids: inputs.apiKeyIds, + team_ids: inputs.teamIds, + user_ids: inputs.userIds, + router_names: inputs.routerNames, + direction: inputs.direction, + ...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel } : {}), + shadow_percentage: inputs.shadowPercentage, + duration_days: inputs.durationDays, + max_budget: inputs.maxBudget, + judge_model: inputs.judgeModel, +}); + +export const StartForm: React.FC = () => { + const { accessToken } = useAuthorized(); + const [apiKeyIds, setApiKeyIds] = useState([]); + const [teamIds, setTeamIds] = useState([]); + const [userIds, setUserIds] = useState([]); + const [routerNames, setRouterNames] = useState([]); + const [direction, setDirection] = useState("forward"); + const [baselineModel, setBaselineModel] = useState(""); + const [percentage, setPercentage] = useState("10"); + const [durationDays, setDurationDays] = useState("7"); + const [judgeModel, setJudgeModel] = useState(""); + const [maxBudget, setMaxBudget] = useState("10"); + const { data: autoRouters } = useAutoRouters(); + const judgeModelOptions = useJudgeModelOptions(); + const baselineModelOptions = useBaselineModelOptions(); + const start = useStartShadowEval(); + + const routerOptions = useMemo(() => { + const names = new Set( + (autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)), + ); + return [...names].toSorted().map((name) => ({ label: name, value: name })); + }, [autoRouters]); + + const validityInputs: StartFormValidityInputs = { + accessToken, + apiKeyIds, + teamIds, + userIds, + routerNames, + direction, + baselineModel, + judgeModel, + percentage, + maxBudget, + }; + const { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid } = startFormValidity(validityInputs); + const handleStart = () => { + const bodyInputs: StartBodyInputs = { + apiKeyIds, + teamIds, + userIds, + routerNames, + direction, + baselineModel, + shadowPercentage: parsedPct, + durationDays: Number.parseInt(durationDays, 10), + maxBudget: parsedMaxBudget, + judgeModel, + }; + start.mutate(buildStartBody(bodyInputs)); + }; + + return ( + + + Start a shadow eval +

      {START_FORM_DESCRIPTION[direction]}

      +       + +
      + + + + + + + + + + + + + + +
      + setPercentage(e.target.value)} + /> + % of traffic +
      +
      + {percentage.trim() !== "" && !percentageValid && ( +

      Enter a value from 0.1 to 100

      + )} +
      +       + + + + +
      + $ + setMaxBudget(e.target.value)} + /> + max shadow + judge spend, per target +
      + {maxBudget.trim() !== "" && !maxBudgetValid && ( +

      Enter a value from 0.01 to 10000

      + )} +       + {direction === "reverse" && ( + + + + )} + + + +
      + +       +       + ); +}; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4ad3ca06501..e944062e15e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35331,8 +35331,17 @@ export interface components { last_error?: string | null; /** @description Stratified verdicts; detail endpoint only */ results?: components["schemas"]["ShadowEvalResult"] | null; - /** Router Name */ - router_name: string; + /** + * Router Name + * @description The first router, kept for callers that predate router_names; derived so the + * two fields can never disagree. + */ + readonly router_name: string; + /** + * Router Names + * @description Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of traffic and judge every arm against the same real responses + */ + router_names: string[]; /** Shadow Percentage */ shadow_percentage: number; /** @@ -35420,6 +35429,12 @@ export interface components { * @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked */ by_current_model: components["schemas"]["ShadowEvalSlice"][]; + /** + * By Router + * @description One slice per router arm, grouped on the router name. Every arm of a multi-router job is judged against the same real responses over the same sampled requests, so these slices compare routers head-to-head: like-for-like win rates and spends on identical traffic. Verdicts from before arm stamping existed count toward the job's own router + * @default [] + */ + by_router: components["schemas"]["ShadowEvalSlice"][]; /** By Tier */ by_tier: components["schemas"]["ShadowEvalSlice"][]; /** @@ -35433,13 +35448,13 @@ export interface components { overall_tie_rate_pct: number; /** * Sampled Real Spend - * @description USD the real arm billed across all judged turns, cache-served turns excluded + * @description USD the real arm billed across all judged turns, cache-served turns excluded. A judged turn is one (request, router arm) verdict, so a multi-router job counts the real response once per arm it was judged against; per-router comparisons read by_router * @default 0 */ sampled_real_spend: number; /** * Sampled Shadow Spend - * @description USD the shadow arm billed across the same turns, judge excluded, like for like + * @description USD the shadow arms billed across the same turns, judge excluded, like for like * @default 0 */ sampled_shadow_spend: number; @@ -35733,15 +35748,21 @@ export interface components { judge_model: string; /** * Max Budget - * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window + * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window. Every router arm draws from the same per-target budget, so a multi-router job reaches it proportionally sooner * @default 10 */ max_budget: number; /** * Router Name - * @description The auto-router under evaluation, in either direction + * @description The auto-router under evaluation, in either direction: the single-router spelling of router_names. Provide exactly one of the two fields */ - router_name: string; + router_name?: string | null; + /** + * Router Names + * @description The auto-routers under evaluation, at most 4. Every sampled request runs through every router listed and each arm is judged independently against the same real response, so routers compare head-to-head on identical traffic. More than one router requires direction 'forward'. After validation this field always carries the full deduplicated set, whichever spelling the caller used + * @default [] + */ + router_names: string[]; /** * Shadow Percentage * @description Percentage of each target's requests to duplicate through the router From 4a24be886d4d76d06f355cb5164af36fb3f36b4d Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 31 Aug 2026 22:12:59 -0700 Subject: [PATCH 34/34] feat(ui): one classification frequency picker for complexity auto-routers (#39042) Classification timing and session affinity are the same operator question, so Advanced: Classification Method now carries a single "How often to classify" radio: every request, every new user message, or once per session. The session choice writes session_affinity and stays disabled on custom tier sets, where the backend rejects it. Advanced: Affinity keeps the deployment switch alone. The serializer always writes classification_mode, matching session_affinity on the line below it, so an explicitly stored every_request survives an untouched save instead of being dropped back to the backend default. --- .../src/autorouter_presets.json | 4 + .../add_model/ClassificationMethodConfig.tsx | 51 ++++++++++ .../add_model/ComplexityRouterConfig.test.tsx | 96 +++++++++++++++++-- .../add_model/ComplexityRouterConfig.tsx | 41 +++++--- .../add_model/add_auto_router_tab.test.tsx | 46 ++++++++- .../add_model/add_auto_router_tab.tsx | 1 + .../build_complexity_router_config.test.ts | 19 ++++ .../build_complexity_router_config.ts | 6 ++ ...d_updated_complexity_router_config.test.ts | 28 ++++++ .../edit_auto_router_modal.test.ts | 2 + .../edit_auto_router_modal.test.tsx | 85 ++++++++++++---- .../edit_auto_router_modal.tsx | 7 ++ .../src/lib/autorouter_presets.test.ts | 32 +++++++ .../src/lib/autorouter_presets.ts | 2 + 14 files changed, 380 insertions(+), 40 deletions(-) diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/ui/litellm-dashboard/src/autorouter_presets.json index 4cbb548a855..da35d6171dc 100644 --- a/ui/litellm-dashboard/src/autorouter_presets.json +++ b/ui/litellm-dashboard/src/autorouter_presets.json @@ -14,6 +14,7 @@ }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } @@ -30,6 +31,7 @@ }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } @@ -56,6 +58,7 @@ }, "classifier_context_window_size": 0, "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } @@ -72,6 +75,7 @@ }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 84f61c95eca..96c93306611 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -15,9 +15,12 @@ import { RestrictedSection, restrictedBy } from "./TierRestrictions"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { + ClassificationFrequency, ClassifierFallback, ClassifierType, ComplexityRouterConfigValue, + classificationFrequency, + withClassificationFrequency, DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS, MIN_QUOTED_CONTEXT_TURN_CHARS, DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, @@ -211,6 +214,7 @@ const ClassificationMethodConfig: React.FC = ({ const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); const hasDefaultModel = Boolean(defaultModel); const classifierType = effectiveClassifierType(value); + const sessionFrequencyRestriction = restrictedBy(value, "sessionAffinity"); const classifierModelMissing = showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); @@ -305,6 +309,10 @@ const ClassificationMethodConfig: React.FC = ({ onChange({ ...value, classifier_fallback: fallback }); }; + const handleClassificationFrequencyChange = (frequency: ClassificationFrequency) => { + onChange(withClassificationFrequency(value, frequency)); + }; + const handleClassifierContextWindowSizeChange = (windowSize: number) => { onChange({ ...value, @@ -367,6 +375,49 @@ const ClassificationMethodConfig: React.FC = ({ )} +
      + How often to classify + + handleClassificationFrequencyChange(frequency as ClassificationFrequency) + } + > +
      + + + +
      +       +

      + Holding the tier keeps an agent on one model for a whole tool loop and cuts scoring cost. A turn the router + cannot match to a held decision, such as one with no session id or an expired one, is scored again +

      +
      + {usesLlmClassifier(classifierType) && (
      diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 218c99e32c5..bbee03135c7 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -598,6 +598,82 @@ describe("ComplexityRouterConfig classifier fallback", () => { }); }); +describe("ComplexityRouterConfig classification frequency", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + + it("defaults to every request, matching both backend field defaults", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Every request/ })).toBeChecked(); + expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); + expect(screen.getByRole("radio", { name: /Once per session/ })).not.toBeChecked(); + }); + + it("writes both wire fields when the frequency moves to every new user message", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + fireEvent.click(screen.getByRole("radio", { name: /Every new user message/ })); + expect(onChange).toHaveBeenCalledWith({ + ...llmValue, + classification_mode: "user_turn", + session_affinity: false, + }); + }); + + it("writes session affinity, not a classification mode, when the frequency moves to once per session", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + fireEvent.click(screen.getByRole("radio", { name: /Once per session/ })); + expect(onChange).toHaveBeenCalledWith({ + ...llmValue, + classification_mode: "every_request", + session_affinity: true, + }); + }); + + it("shows a hand-authored config that sets both fields as once per session, matching the backend", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Once per session/ })).toBeChecked(); + expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); + }); + + it("records a switch back to every request", () => { + const onChange = vi.fn(); + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeChecked(); + fireEvent.click(screen.getByRole("radio", { name: /Every request/ })); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classification_mode: "every_request" })); + }); + + it("offers the frequency on a heuristic router, where holding the tier still pins the model", () => { + // The backend pin is gated on the two fields alone, so a heuristic router that switches models + // mid tool loop is fixed by this control too. + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeInTheDocument(); + }); +}); + describe("ComplexityRouterConfig classifier rubric", () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -761,12 +837,12 @@ describe("ComplexityRouterConfig tier labels", () => { }); describe("ComplexityRouterConfig affinity panel", () => { - it("holds both affinity switches with their backend defaults", () => { + it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Affinity")); expect(screen.getByRole("switch", { name: "Pin a session to one deployment per model group" })).toBeChecked(); - expect(screen.getByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + expect(screen.queryByRole("switch", { name: "Pin a session to its first model" })).not.toBeInTheDocument(); }); it("writes deployment_affinity through onChange without touching other keys", () => { @@ -1291,10 +1367,18 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.getByLabelText("Fallback tier")).toBeInTheDocument(); }); - it("disables session pinning and says why, rather than letting a stripped value look saved", () => { - renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); - expect(screen.getByLabelText("Pin a session to its first model")).toHaveAttribute("data-disabled"); + it("disables the once-per-session frequency and says why, rather than letting a stripped value look saved", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + const sessionOption = screen.getByRole("radio", { name: /Once per session/ }); + expect(sessionOption).toHaveAttribute("aria-disabled", "true"); + expect(sessionOption).not.toBeChecked(); expect( screen.getByText("Session pinning escalates along the built-in tier ladder", { exact: false }), ).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 111a7c9f10a..5fde87fd638 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -56,6 +56,16 @@ export const MIN_QUOTED_CONTEXT_TURN_CHARS = 120; export const DEFAULT_SESSION_AFFINITY = false; export const DEFAULT_DEPLOYMENT_AFFINITY = true; +export type ClassificationMode = "every_request" | "user_turn"; + +export const DEFAULT_CLASSIFICATION_MODE: ClassificationMode = "every_request"; + +/** + * One operator-facing choice over the two wire fields that share the router's tier-pin machinery: + * session affinity pins every turn, user_turn pins every turn except a new human ask. + */ +export type ClassificationFrequency = ClassificationMode | "session"; + export type ComplexityTiers = { SIMPLE: string[]; MEDIUM: string[]; @@ -384,6 +394,7 @@ export interface ComplexityRouterConfigValue { classification_prompt?: string; /** Highest tier the scorer may decide alone under heuristic_first. Required by that type, rejected by the others. */ heuristic_first_max_tier?: string; + classification_mode?: ClassificationMode; session_affinity?: boolean; deployment_affinity?: boolean; /** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */ @@ -420,6 +431,21 @@ export interface ComplexityRouterConfigValue { tier_model_params?: TierModelParamsByTier; } +/** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */ +export const classificationFrequency = (value: ComplexityRouterConfigValue): ClassificationFrequency => { + if (!value.custom_tier_set && (value.session_affinity ?? DEFAULT_SESSION_AFFINITY)) return "session"; + return value.classification_mode === "user_turn" ? "user_turn" : "every_request"; +}; + +export const withClassificationFrequency = ( + value: ComplexityRouterConfigValue, + frequency: ClassificationFrequency, +): ComplexityRouterConfigValue => ({ + ...value, + classification_mode: frequency === "user_turn" ? "user_turn" : "every_request", + session_affinity: frequency === "session", +}); + interface ComplexityRouterConfigProps { modelInfo: ModelGroup[]; value: ComplexityRouterConfigValue; @@ -498,23 +524,10 @@ const AffinityControls: React.FC<{ /> Pin a session to one deployment per model group
      - + Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to load-balance every turn. -
      - onChange({ ...value, session_affinity: sessionAffinity })} - aria-label="Pin a session to its first model" - /> - Pin a session to its first model -
      - - {restrictedBy(value, "sessionAffinity")?.reason ?? - "Keeps a session on its first turn's model instead of re-classifying each turn. Also pins the deployment."} - ); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 71b454dbb06..d8a955b719c 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -362,8 +362,8 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); expandDetailedConfiguration(); - await user.click(screen.getByText("Advanced: Affinity")); - expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + await user.click(screen.getByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Once per session/ })).not.toBeChecked(); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -468,8 +468,8 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); expandDetailedConfiguration(); - await user.click(screen.getByText("Advanced: Affinity")); - await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Once per session/ })); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -479,6 +479,44 @@ describe("AddAutoRouterTab", () => { }); }); + it("carries every new user message through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "user-turn-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every new user message/ })); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + classification_mode: "user_turn", + }); + }); + + it("writes every_request into the create payload when the default frequency stays selected", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "default-timing-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Every request/ })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect( + vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config.classification_mode, + ).toBe("every_request"); + }); + it("defaults a new router to deployment affinity on, matching the backend field default", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index ed584a4882b..c60ce5e4959 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -342,6 +342,7 @@ const AddAutoRouterTab: React.FC = ({ planModeMinTier: complexityRouterConfig.plan_mode_min_tier, classificationPrompt: complexityRouterConfig.classification_prompt, heuristicFirstMaxTier: complexityRouterConfig.heuristic_first_max_tier, + classificationMode: complexityRouterConfig.classification_mode, tierLabels: complexityRouterConfig.tier_labels, classifierType: complexityRouterConfig.classifier_type, classifierLlmConfig: complexityRouterConfig.classifier_llm_config, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index feddaaa0eac..40addf086b6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -52,6 +52,7 @@ describe("buildComplexityRouterConfig", () => { const expected = { tiers, classifier_type: "heuristic", + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, escalation_keywords: ["LITELLM ESCALATE"], @@ -790,6 +791,20 @@ describe("heuristic_first", () => { }); }); +describe("classification_mode", () => { + it("emits user_turn", () => { + const config = buildComplexityRouterConfig({ ...baseParams, classificationMode: "user_turn" }); + expect(config.classification_mode).toBe("user_turn"); + }); + + it("writes every_request explicitly, so a saved router never depends on the backend default", () => { + expect( + buildComplexityRouterConfig({ ...baseParams, classificationMode: "every_request" }).classification_mode, + ).toBe("every_request"); + expect(buildComplexityRouterConfig(baseParams).classification_mode).toBe("every_request"); + }); +}); + describe("buildComplexityRouterConfig with an edited tier set", () => { const customTierSet = { tiers: [ @@ -902,6 +917,10 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); }); + it("keeps classification_mode, which the backend accepts beside tier_definitions", () => { + expect(build({ classificationMode: "user_turn" }).classification_mode).toBe("user_turn"); + }); + it("carries the plan-mode floor as the row's name, not the row id the form holds", () => { expect(build({ planModeMinTier: "sec" }).plan_mode_min_tier).toBe("SECURITY_REVIEW"); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 9a14e956207..6a087649c00 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -19,10 +19,12 @@ import { import { AdaptiveEligible, AdaptiveRouterWeights, + ClassificationMode, ClassifierFallback, ClassifierLLMConfig, ClassifierType, ComplexityTierLabels, + DEFAULT_CLASSIFICATION_MODE, ComplexityRouterConfigValue, ComplexityTiers, DimensionWeights, @@ -105,6 +107,7 @@ export interface BuildComplexityRouterConfigParams { classifierFallback: ClassifierFallback | undefined; classificationPrompt: string | undefined; heuristicFirstMaxTier: string | undefined; + classificationMode: ClassificationMode | undefined; sessionAffinity: boolean; deploymentAffinity: boolean; customTechnicalKeywords: string[]; @@ -159,6 +162,7 @@ export interface ComplexityRouterConfigPayload { classifier_fallback?: ClassifierFallback; classification_prompt?: string; heuristic_first_max_tier?: string; + classification_mode: ClassificationMode; session_affinity: boolean; deployment_affinity: boolean; custom_technical_keywords?: string[]; @@ -393,6 +397,7 @@ export const buildComplexityRouterConfig = ({ classifierFallback, classificationPrompt, heuristicFirstMaxTier, + classificationMode, sessionAffinity, deploymentAffinity, customTechnicalKeywords, @@ -452,6 +457,7 @@ export const buildComplexityRouterConfig = ({ ...(cleanedTierLabels && { tier_labels: cleanedTierLabels }), classifier_type: classifierType, ...classifierWireFields(effectiveType, classifierInputs), + classification_mode: classificationMode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: sessionAffinity, deployment_affinity: deploymentAffinity, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index d8c2987ead5..3e63cbd3b32 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -257,6 +257,33 @@ describe("buildUpdatedComplexityRouterConfig session affinity", () => { }); }); +describe("buildUpdatedComplexityRouterConfig classification mode", () => { + it("round-trips a stored user_turn through hydrate then save", () => { + const stored = { ...STORED, classification_mode: "user_turn" }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + + expect(hydrated.classification_mode).toBe("user_turn"); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).classification_mode).toBe("user_turn"); + }); + + it("round-trips an explicitly stored every_request, so an untouched save leaves it as written", () => { + const stored = { ...STORED, classification_mode: "every_request" }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + + expect(hydrated.classification_mode).toBe("every_request"); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).classification_mode).toBe("every_request"); + }); + + it("rewrites a stored user_turn to every_request once the operator picks the default back", () => { + const stored = { ...STORED, classification_mode: "user_turn" }; + const result = buildUpdatedComplexityRouterConfig(stored, { + ...FORM_VALUE, + classification_mode: "every_request", + }); + expect(result.classification_mode).toBe("every_request"); + }); +}); + describe("buildUpdatedComplexityRouterConfig deployment affinity", () => { it("writes deployment_affinity=false when the toggle is off", () => { const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, deployment_affinity: false }); @@ -476,6 +503,7 @@ describe("managed keys survive an untouched open-and-save", () => { classifier_context_budget_chars: 4000, classifier_context_include_assistant_turns: true, classifier_fallback: "default_model", + classification_mode: "user_turn", session_affinity: true, deployment_affinity: false, adaptive: true, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 027e01a9351..d199af51b41 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -47,6 +47,7 @@ const expectedClassifiedTierConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, adaptive: true, @@ -68,6 +69,7 @@ const expectedAdaptiveDisabledConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, }; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index a4921fcfcb5..96e8549eac8 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -364,7 +364,7 @@ describe("EditAutoRouterModal assistant turns", () => { }); }); -describe("EditAutoRouterModal session affinity", () => { +describe("EditAutoRouterModal classification frequency", () => { beforeEach(() => { modelPatchUpdateCall.mockClear(); }); @@ -381,15 +381,15 @@ describe("EditAutoRouterModal session affinity", () => { />, ); - // A stored config with no session_affinity key now runs with affinity OFF, because the backend - // field defaults to False. The toggle has to render what the router actually does, and an - // untouched save must not flip it. - it("shows a stored config with no session_affinity key as off", async () => { + // A stored config with neither key now runs with affinity OFF, because both backend fields + // default that way. The picker has to render what the router actually does, and an untouched + // save must not flip it. + it("shows a stored config with neither key as every request", async () => { const user = userEvent.setup(); renderWithStoredConfig(STORED_CONFIG); - await user.click(await screen.findByText("Advanced: Affinity")); - expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Every request/ })).toBeChecked(); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -397,12 +397,12 @@ describe("EditAutoRouterModal session affinity", () => { expect(savedConfig().session_affinity).toBe(false); }); - it("shows a stored session_affinity=true as on and preserves it through an untouched save", async () => { + it("shows a stored session_affinity=true as once per session and preserves it through an untouched save", async () => { const user = userEvent.setup(); renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); - await user.click(await screen.findByText("Advanced: Affinity")); - expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).toBeChecked(); + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Once per session/ })).toBeChecked(); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -410,12 +410,12 @@ describe("EditAutoRouterModal session affinity", () => { expect(savedConfig().session_affinity).toBe(true); }); - it("persists turning session affinity on", async () => { + it("persists picking once per session", async () => { const user = userEvent.setup(); renderWithStoredConfig(STORED_CONFIG); - await user.click(await screen.findByText("Advanced: Affinity")); - await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Once per session/ })); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -423,18 +423,71 @@ describe("EditAutoRouterModal session affinity", () => { expect(savedConfig().session_affinity).toBe(true); }); - it("persists turning session affinity back off", async () => { + it("persists picking every request back over a stored session pin", async () => { const user = userEvent.setup(); renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); - await user.click(await screen.findByText("Advanced: Affinity")); - await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every request/ })); await user.click(screen.getByRole("button", { name: /save changes/i })); await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); expect(savedConfig().session_affinity).toBe(false); }); + + it("clears a stored session pin when the operator moves to every new user message", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every new user message/ })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(false); + expect(savedConfig().classification_mode).toBe("user_turn"); + }); + + it("shows a stored user_turn as selected and preserves it through an untouched save", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, classification_mode: "user_turn" }); + + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Every new user message/ })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classification_mode).toBe("user_turn"); + }); + + it("persists switching a stored config to every new user message", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every new user message/ })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classification_mode).toBe("user_turn"); + }); + + it("rewrites the stored mode to every_request when the operator picks it back", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, classification_mode: "user_turn" }); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every request/ })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classification_mode).toBe("every_request"); + }); }); describe("EditAutoRouterModal deployment affinity", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 4751c2e64b7..2aecf0b3483 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -96,6 +96,7 @@ export interface StoredComplexityRouterConfig { classifier_context_budget_chars?: unknown; classifier_context_include_assistant_turns?: unknown; classifier_fallback?: unknown; + classification_mode?: unknown; tier_boundaries?: unknown; token_thresholds?: unknown; dimension_weights?: unknown; @@ -165,6 +166,10 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== "" ? parsedConfig.heuristic_first_max_tier : undefined, + classification_mode: + parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request" + ? parsedConfig.classification_mode + : undefined, tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries), token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds), dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights), @@ -207,6 +212,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_fallback", "classification_prompt", "heuristic_first_max_tier", + "classification_mode", "session_affinity", "deployment_affinity", "adaptive", @@ -294,6 +300,7 @@ export const buildUpdatedComplexityRouterConfig = ( planModeMinTier: value.plan_mode_min_tier, classificationPrompt: value.classification_prompt, heuristicFirstMaxTier: value.heuristic_first_max_tier, + classificationMode: value.classification_mode, tierLabels: value.tier_labels, classifierType: value.classifier_type, classifierLlmConfig: value.classifier_llm_config, diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index f14d6279e32..5632d6d947f 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -230,6 +230,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -273,6 +274,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -287,6 +289,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -320,6 +323,7 @@ describe("autorouter_presets", () => { const simpleTierConfig = (presetModel: string) => ({ tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }); @@ -563,6 +567,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, match_threshold: 0, @@ -578,6 +583,7 @@ describe("autorouter_presets", () => { { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic", + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, enable_context_window_escalation: false, @@ -589,11 +595,29 @@ describe("autorouter_presets", () => { expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); }); + it("carries a preset's classification_mode and defaults it when the preset omits one", () => { + const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }; + const base = { + tiers, + classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, + session_affinity: false, + deployment_affinity: true, + }; + const availability = groupsOnly(["gpt-5-nano"]); + expect( + buildPresetPrefill({ ...base, classification_mode: "user_turn" }, availability).complexityRouterConfig + .classification_mode, + ).toBe("user_turn"); + expect(buildPresetPrefill(base, availability).complexityRouterConfig.classification_mode).toBe("every_request"); + }); + it("falls back to the defaults when a preset omits match_threshold and escalation_keywords", () => { const prefill = buildPresetPrefill( { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic", + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }, @@ -610,6 +634,7 @@ describe("autorouter_presets", () => { const base = { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -625,6 +650,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["claude-sonnet-4-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -639,6 +665,7 @@ describe("autorouter_presets", () => { REASONING: [{ model_name: "o3", litellm_params: { reasoning_effort: "high" } }], }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -658,6 +685,7 @@ describe("autorouter_presets", () => { REASONING: [{ model_name: "claude-sonnet-4-5", litellm_params: { reasoning_effort: "high" } }], }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -680,6 +708,9 @@ describe("autorouter_presets", () => { ], }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, + session_affinity: false, + deployment_affinity: true, }; const prefill = buildPresetPrefill(config, groupsOnly(["claude-sonnet-4.5"])); // temperature survives from the spelling that would otherwise have been overwritten; @@ -693,6 +724,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index f96dd5ddb4c..4c1f08b62e9 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -6,6 +6,7 @@ import { ComplexityRouterConfigValue, ClassifierType, ClassifierLLMConfig, + DEFAULT_CLASSIFICATION_MODE, DEFAULT_SESSION_AFFINITY, DEFAULT_DEPLOYMENT_AFFINITY, usesLlmClassifier, @@ -288,6 +289,7 @@ export const buildPresetPrefill = ( classifier_context_budget_chars: config.classifier_context_budget_chars, classifier_context_per_turn_chars: config.classifier_context_per_turn_chars, classifier_context_include_assistant_turns: config.classifier_context_include_assistant_turns, + classification_mode: config.classification_mode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: config.session_affinity ?? DEFAULT_SESSION_AFFINITY, deployment_affinity: config.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, adaptive: config.adaptive,