From 06a58efb2e42a78c88eed2d4e23fdd6d215fd777 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 27 Jul 2026 16:30:27 -0700 Subject: [PATCH 01/33] feat(mcp): manual authorization-code delivery for headless MCP clients The aggregate gateway DCR flow ends in a 303 to the client's loopback redirect_uri. When the MCP client runs on a browserless machine (EC2, SSH box, container) the user authorizes from a browser on another machine, so the 303 dereferences the wrong loopback and the code never reaches the client. The connect banner now offers manual delivery for loopback clients: the finish form posts delivery=manual and /authorize/complete renders the callback URL on a no-store page instead of redirecting. The user pastes it into the client (Claude Code v2.1.191+ accepts a pasted callback URL) or fetches it from the client machine's terminal. Manual codes keep the same sealing, PKCE binding, and single-use guard, with a 5 minute expiry instead of 2 to survive the copy-paste hop; the used-code marker TTL derives from the code's own remaining lifetime so the single-use property holds for the full 5 minutes. The default redirect path is unchanged. Resolves LIT-4863 --- .../mcp_server/discoverable_endpoints.py | 9 +- .../mcp_server/gateway_dcr_flow.py | 70 +++++- .../mcp_server/test_gateway_dcr_flow.py | 207 ++++++++++++++++++ .../chat/ConnectFlowBanner.test.tsx | 35 ++- .../src/components/chat/ConnectFlowBanner.tsx | 17 ++ 5 files changed, 329 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index caa5c65894c..cdc3ac15b1a 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1778,10 +1778,12 @@ async def token_endpoint( @router.post("/authorize/complete") -async def authorize_complete(request: Request, flow: str = Form(...)): +async def authorize_complete(request: Request, flow: str = Form(...), delivery: str | None = Form(None)): """Finish an aggregate connect flow: mint the gateway authorization code for the - signed-in user and redirect back to the DCR client. POST plus the per-flow HttpOnly - cookie set at /authorize; an anonymous or bad-flow request just 400s.""" + signed-in user and hand it back to the DCR client, by 303 redirect (default) or, for + a loopback client on a different machine, as a copyable callback URL + (``delivery=manual``). POST plus the per-flow HttpOnly cookie set at /authorize; an + anonymous or bad-flow request just 400s.""" from litellm.proxy.proxy_server import user_api_key_cache # noqa: PLC0415 # circular import at module load return await complete_connect_flow( @@ -1789,6 +1791,7 @@ async def authorize_complete(request: Request, flow: str = Form(...)): flow_handle=flow, session_user_id=_session_cookie_user_id(request), cache=user_api_key_cache, + delivery=delivery, ) diff --git a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py index 58233c4c9e5..7177b798c5f 100644 --- a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py +++ b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py @@ -39,6 +39,7 @@ from __future__ import annotations import hashlib import hmac +import html import secrets from base64 import urlsafe_b64encode from collections.abc import Mapping @@ -47,7 +48,7 @@ from typing import Awaitable, Callable, Literal, TypeVar from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse from fastapi import HTTPException, Request -from fastapi.responses import JSONResponse, RedirectResponse, Response +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response from pydantic import BaseModel, ConfigDict, Field, ValidationError from typing_extensions import assert_never @@ -94,6 +95,13 @@ server-side session store, and the sealed value never appears in a URL).""" CONNECT_FLOW_TTL_SECONDS = 600 GATEWAY_AUTH_CODE_TTL_SECONDS = 120 +MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS = 300 +"""Lifetime of a code the user delivers by hand (headless/remote client, LIT-4863 class): +copy-pasting a callback URL from a laptop browser to an SSH session is slower than a +browser redirect, so manual-delivery codes get 5 minutes instead of 2, still well under +the 10-minute ceiling RFC 6749 section 4.1.2 recommends. Single-use and PKCE binding are +unchanged, so the longer window only extends how long the legitimate holder has to paste +it, not what an observer could do with it.""" _CLAIM_TTL_BUFFER_SECONDS = 60 _USED_CODE_CACHE_PREFIX = "mcp_gateway_dcr_code_used:" _USED_FLOW_CACHE_PREFIX = "mcp_gateway_dcr_flow_used:" @@ -390,6 +398,7 @@ async def complete_connect_flow( flow_handle: str, session_user_id: str | None, cache: DualCache, + delivery: str | None = None, ) -> Response: """The deliberate finish step of the connect flow: mint the gateway authorization code and send the browser back to the client. @@ -399,7 +408,24 @@ async def complete_connect_flow( into the flow: a link crafted by another party dies here with ``access_denied`` instead of minting a code for the victim's identity. The flow is single-use (an atomic claim on its ``jti``), so a double-submit cannot mint two codes from one sign-in. + + ``delivery`` chooses how the code reaches the client. Default (absent or + ``"redirect"``) is the 303 to the client's registered redirect URI. ``"manual"`` + renders the callback URL on a page instead, for a client whose redirect URI is a + loopback host but which runs on a DIFFERENT machine than the browser (EC2/SSH box, + container): the 303 would dereference the browser machine's loopback and the code + would never arrive, so the user carries it over by pasting the URL into the client or + fetching it from the client machine's terminal. Manual delivery is honored only for + loopback redirect URIs; a routable redirect URI works from any browser by + construction, so those flows always redirect. The user who sees the page is exactly + the user the 303 would have carried the code to, and the same user already sees the + code today in the dead redirect's address bar, so the page exposes the code to no new + party. Unknown ``delivery`` values are rejected rather than defaulted: a client that + asked for manual delivery and got a dead redirect instead would silently lose its + code. """ + if delivery not in (None, "redirect", "manual"): + return _oauth_error(400, "invalid_request", "delivery must be 'redirect' or 'manual'") sealed_flow = request.cookies.get(_flow_cookie_name(flow_handle)) if sealed_flow is None: return _oauth_error(400, "invalid_request", "unknown or expired connect flow") @@ -417,6 +443,8 @@ async def complete_connect_flow( f"{_USED_FLOW_CACHE_PREFIX}{flow.jti}", CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS ): return _oauth_error(400, "invalid_request", "this connect flow was already completed; restart the connection") + manual_delivery = delivery == "manual" and is_loopback_redirect_host(urlparse(flow.redirect_uri)) + code_ttl = MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS if manual_delivery else GATEWAY_AUTH_CODE_TTL_SECONDS code = _seal( GATEWAY_AUTH_CODE_PREFIX, _GatewayAuthCode( @@ -426,16 +454,46 @@ async def complete_connect_flow( code_challenge=flow.code_challenge, jti=secrets.token_urlsafe(24), iat=int(now.timestamp()), - exp=int(now.timestamp()) + GATEWAY_AUTH_CODE_TTL_SECONDS, + exp=int(now.timestamp()) + code_ttl, ), ) params = {"code": code, **({"state": flow.state} if flow.state else {})} - response = RedirectResponse(_append_query_params(flow.redirect_uri, params), status_code=303) + callback_url = _append_query_params(flow.redirect_uri, params) + response: Response = ( + _manual_delivery_response(callback_url) if manual_delivery else RedirectResponse(callback_url, status_code=303) + ) path, secure = _cookie_path_and_secure(request) response.delete_cookie(key=_flow_cookie_name(flow_handle), path=path, secure=secure, httponly=True, samesite="lax") return response +def _manual_delivery_response(callback_url: str) -> Response: + """The manual code-delivery page: the callback URL the 303 would have followed, + rendered for the user to carry to the machine the client actually runs on (paste into + the client's prompt, or fetch with curl from that machine's terminal). Served + no-store because the body holds a live single-use code, and the URL is HTML-escaped + because it is client-influenced. The page renders the URL as data only, never as a + ready-to-paste shell command: no single quoting of an attacker-influenced string is + correct across POSIX shells, cmd.exe, and PowerShell (cmd.exe ignores single quotes + and percent-expands inside double quotes), so any command string this page suggested + would be wrong for some shell the user might paste it into.""" + safe_url = html.escape(callback_url, quote=True) + minutes = MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS // 60 + body = ( + "Finish connecting" + "

Almost done

" + "

Your MCP client runs on a different machine, so this browser cannot deliver the" + " authorization code to it. On the machine where the client runs, paste this URL into" + " the client's prompt (Claude Code accepts the pasted callback URL), or pass it as the" + " quoted argument of a curl command from that machine's terminal:

" + f'

' + f"

The code is single-use and expires in {minutes} minutes. You can close this window" + " once the client confirms it is connected.

" + "" + ) + return HTMLResponse(body, headers=TOKEN_NO_CACHE_HEADERS) + + def _pkce_verifier_matches(code_verifier: str, code_challenge: str) -> bool: """RFC 7636 S256 verification, total over hostile input. The comparison is over bytes so a non-ASCII ``code_challenge`` (which reaches here unvalidated from the client's @@ -601,9 +659,11 @@ async def _authorization_code_grant( if failure is not None: return _reload_failure_response(failure) # Atomic single-use claim is the gate: on a concurrent double-redeem exactly one caller - # wins, and a claim that cannot be recorded fails closed. + # wins, and a claim that cannot be recorded fails closed. The marker's TTL derives from + # the code's own remaining lifetime so it outlives whichever lifetime the code was minted with. if not await guard.claim( - f"{_USED_CODE_CACHE_PREFIX}{parsed.jti}", GATEWAY_AUTH_CODE_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS + f"{_USED_CODE_CACHE_PREFIX}{parsed.jti}", + parsed.exp - int(now.timestamp()) + _CLAIM_TTL_BUFFER_SECONDS, ): return _oauth_error(400, "invalid_grant", "the authorization code was already used") return _session_token_pair(SessionPrincipal(user_id=parsed.user_id, client_id=client_id), keys, now) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py index 375ec022115..85a19331777 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py @@ -1,6 +1,7 @@ """Tests for the aggregate gateway DCR flow (register, authorize, complete, token).""" import hashlib +import html import json from base64 import urlsafe_b64encode from datetime import datetime, timedelta, timezone @@ -16,7 +17,10 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( GATEWAY_AUTH_CODE_PREFIX, GATEWAY_AUTH_CODE_TTL_SECONDS, GATEWAY_DCR_CLIENT_ID_PREFIX, + MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS, + _AUTH_CODE_DEBUG_KEY, _GatewayAuthCode, + _open_sealed, _seal, aggregate_authorize, aggregate_token, @@ -588,3 +592,206 @@ async def test_single_use_guard_fails_closed_when_redis_errors(): guard = _SingleUseGuard(cache) assert await guard.claim("jti-fault", 60) is False # fail closed, not a fallback count of 1 + + +LOOPBACK_REDIRECT_URI = "http://localhost:3118/callback" + + +async def _complete(redirect_uri: str, delivery, cookies=None, handle=None, session_user_id="u1"): + client_id = (await _register([redirect_uri]))["client_id"] + if cookies is None: + handle, cookies = _flow_cookie_from(_authorize(client_id, session_user_id="u1", redirect_uri=redirect_uri)) + response = await complete_connect_flow( + request=_request("/authorize/complete", cookies=cookies, method="POST"), + flow_handle=handle, + session_user_id=session_user_id, + cache=DualCache(), + delivery=delivery, + ) + return client_id, response + + +def _callback_url_from_page(response) -> str: + import html as html_lib + import re + + match = re.search(r'value="([^"]+)"', response.body.decode()) + assert match is not None + return html_lib.unescape(match.group(1)) + + +@pytest.mark.asyncio +async def test_manual_delivery_renders_pasteable_callback_url_for_loopback_client(): + """The LIT-4863 headless path: a loopback client on another machine gets the callback + URL on a page instead of a dead 303, and the code on that page is a full-fidelity + authorization code (PKCE-bound, single-use, redeemable at /token).""" + client_id, response = await _complete(LOOPBACK_REDIRECT_URI, delivery="manual") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + assert response.headers["cache-control"] == "no-store" + assert f"{CONNECT_FLOW_COOKIE_PREFIX}" in response.headers["set-cookie"] + + callback_url = _callback_url_from_page(response) + parsed = urlparse(callback_url) + assert f"{parsed.scheme}://{parsed.netloc}{parsed.path}" == LOOPBACK_REDIRECT_URI + params = parse_qs(parsed.query) + assert params["state"] == ["client-state-123"] + code = params["code"][0] + assert code.startswith(GATEWAY_AUTH_CODE_PREFIX) + + cache = DualCache() + token_response = await aggregate_token( + request=_request("/token", method="POST"), + grant_type="authorization_code", + code=code, + redirect_uri=LOOPBACK_REDIRECT_URI, + client_id=client_id, + code_verifier=CODE_VERIFIER, + refresh_token=None, + master_key=MASTER_KEY, + reload_user=_reload_user_active, + cache=cache, + ) + assert token_response.status_code == 200 + + replay = await aggregate_token( + request=_request("/token", method="POST"), + grant_type="authorization_code", + code=code, + redirect_uri=LOOPBACK_REDIRECT_URI, + client_id=client_id, + code_verifier=CODE_VERIFIER, + refresh_token=None, + master_key=MASTER_KEY, + reload_user=_reload_user_active, + cache=cache, + ) + assert json.loads(replay.body)["error"] == "invalid_grant" + + +@pytest.mark.asyncio +async def test_manual_delivery_code_gets_the_longer_ttl_and_redirect_code_does_not(): + _, manual = await _complete(LOOPBACK_REDIRECT_URI, delivery="manual") + manual_code = parse_qs(urlparse(_callback_url_from_page(manual)).query)["code"][0] + opened_manual = _open_sealed(manual_code, GATEWAY_AUTH_CODE_PREFIX, _GatewayAuthCode, _AUTH_CODE_DEBUG_KEY) + assert opened_manual is not None + assert opened_manual.exp - opened_manual.iat == MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS + + _, redirected = await _complete(LOOPBACK_REDIRECT_URI, delivery=None) + redirect_code = parse_qs(urlparse(redirected.headers["location"]).query)["code"][0] + opened_redirect = _open_sealed(redirect_code, GATEWAY_AUTH_CODE_PREFIX, _GatewayAuthCode, _AUTH_CODE_DEBUG_KEY) + assert opened_redirect is not None + assert opened_redirect.exp - opened_redirect.iat == GATEWAY_AUTH_CODE_TTL_SECONDS + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delivery", [None, "redirect"]) +async def test_loopback_client_still_redirects_when_manual_not_requested(delivery): + _, response = await _complete(LOOPBACK_REDIRECT_URI, delivery=delivery) + assert response.status_code == 303 + assert response.headers["location"].startswith(LOOPBACK_REDIRECT_URI) + + +@pytest.mark.asyncio +async def test_manual_delivery_is_ignored_for_routable_redirect_uri(): + """A routable redirect URI works from any browser by construction, so manual is a + no-op there and the flow keeps its normal shape.""" + _, response = await _complete(REDIRECT_URI, delivery="manual") + assert response.status_code == 303 + assert response.headers["location"].startswith(REDIRECT_URI) + + +@pytest.mark.asyncio +async def test_unknown_delivery_value_is_rejected_before_the_flow_is_consumed(): + """A typo'd delivery must not burn the single-use flow: the user fixes the form and + finishes normally.""" + client_id = (await _register([LOOPBACK_REDIRECT_URI]))["client_id"] + handle, cookies = _flow_cookie_from(_authorize(client_id, session_user_id="u1", redirect_uri=LOOPBACK_REDIRECT_URI)) + + rejected = await complete_connect_flow( + request=_request("/authorize/complete", cookies=cookies, method="POST"), + flow_handle=handle, + session_user_id="u1", + cache=DualCache(), + delivery="carrier-pigeon", + ) + assert rejected.status_code == 400 + assert json.loads(rejected.body)["error"] == "invalid_request" + + retried = await complete_connect_flow( + request=_request("/authorize/complete", cookies=cookies, method="POST"), + flow_handle=handle, + session_user_id="u1", + cache=DualCache(), + delivery="manual", + ) + assert retried.status_code == 200 + + +@pytest.mark.asyncio +async def test_manual_delivery_page_escapes_client_influenced_values(): + """redirect_uri (and everything else on the page) is client-registered input; a quote + or tag in its path must render inert.""" + hostile_uri = 'http://127.0.0.1:9/cb">' + _, response = await _complete(hostile_uri, delivery="manual") + assert response.status_code == 200 + body = response.body.decode() + assert "" not in body + assert "<script>" in body + + +class _TtlRecordingCache(DualCache): + """Captures the TTL of every single-use claim recorded through the in-memory arm.""" + + def __init__(self): + super().__init__() + self.claim_ttls: dict = {} + + async def async_increment_cache(self, key, value, ttl=None, **kwargs): + self.claim_ttls[key] = ttl + return await super().async_increment_cache(key, value, ttl=ttl, **kwargs) + + +@pytest.mark.asyncio +async def test_used_code_marker_outlives_the_manually_delivered_code(): + """Veria review finding on the LIT-4863 change: a manual code lives 300s, but the + used-code marker was retained for the 120s redirect lifetime plus buffer, so a client + could redeem, wait out the marker, and redeem the still-valid code again. The marker's + TTL must cover the code's own remaining lifetime plus the claim buffer.""" + client_id, response = await _complete(LOOPBACK_REDIRECT_URI, delivery="manual") + code = parse_qs(urlparse(_callback_url_from_page(response)).query)["code"][0] + + cache = _TtlRecordingCache() + token_response = await aggregate_token( + request=_request("/token", method="POST"), + grant_type="authorization_code", + code=code, + redirect_uri=LOOPBACK_REDIRECT_URI, + client_id=client_id, + code_verifier=CODE_VERIFIER, + refresh_token=None, + master_key=MASTER_KEY, + reload_user=_reload_user_active, + cache=cache, + ) + assert token_response.status_code == 200 + + marker_ttls = [ttl for key, ttl in cache.claim_ttls.items() if key.startswith("mcp_gateway_dcr_code_used:")] + assert len(marker_ttls) == 1 + assert marker_ttls[0] >= MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redirect_uri", [LOOPBACK_REDIRECT_URI, "http://127.0.0.1:9/cb$(whoami)&calc& rem x"]) +async def test_manual_delivery_page_renders_the_url_as_data_never_as_a_shell_command(redirect_uri): + """Two review rounds proved no single command string is safe across POSIX shells, + cmd.exe, and PowerShell (single quotes are not quoting in cmd.exe; percent expands + there even inside double quotes), so the page must render the callback URL as data + only and never as a ready-to-paste command.""" + _, response = await _complete(redirect_uri, delivery="manual") + assert response.status_code == 200 + body = response.body.decode() + assert "" not in body + assert 'curl "' not in body + assert "curl '" not in body + assert 'value="' in body diff --git a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx index a565ae5db08..4833b4ad8a4 100644 --- a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx +++ b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { render, screen } from "@testing-library/react"; -import ConnectFlowBanner from "./ConnectFlowBanner"; +import ConnectFlowBanner, { isLoopbackOrigin } from "./ConnectFlowBanner"; vi.mock("@/components/networking", () => ({ getProxyBaseUrl: () => "https://gateway.example.com", @@ -36,6 +36,39 @@ describe("ConnectFlowBanner", () => { expect(screen.getAllByText(/the application/).length).toBeGreaterThan(0); }); + it("offers manual delivery for a loopback client, posted only when checked", () => { + const { container } = render( + , + ); + + const checkbox = container.querySelector('input[type="checkbox"][name="delivery"]') as HTMLInputElement; + expect(checkbox).not.toBeNull(); + expect(checkbox.value).toBe("manual"); + expect(checkbox.checked).toBe(false); + expect(screen.getByText(/remote or SSH machine/i)).toBeInTheDocument(); + }); + + it("does not offer manual delivery for a routable client origin or an unknown one", () => { + const routable = render(); + expect(routable.container.querySelector('input[name="delivery"]')).toBeNull(); + + const unknown = render(); + expect(unknown.container.querySelector('input[name="delivery"]')).toBeNull(); + }); + + it("classifies loopback origins like the server does", () => { + expect(isLoopbackOrigin("http://localhost:3118")).toBe(true); + expect(isLoopbackOrigin("http://127.0.0.1:8080")).toBe(true); + expect(isLoopbackOrigin("http://127.5.4.3:1")).toBe(true); + expect(isLoopbackOrigin("http://[::1]:9000")).toBe(true); + expect(isLoopbackOrigin("http://[0:0:0:0:0:0:0:1]:9000")).toBe(true); + expect(isLoopbackOrigin("https://claude.ai")).toBe(false); + expect(isLoopbackOrigin("http://127.evil.com")).toBe(false); + expect(isLoopbackOrigin("http://localhost.evil.com")).toBe(false); + expect(isLoopbackOrigin(null)).toBe(false); + expect(isLoopbackOrigin("not a url")).toBe(false); + }); + it("does NOT complete the flow on pagehide (completion requires the explicit button)", () => { // Security regression: an attacker could lure a signed-in victim to their own client's // authorize URL; the victim merely closing the tab must NOT deliver a victim-bound code. diff --git a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx index ac2c508e815..cea42f916f8 100644 --- a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx +++ b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx @@ -26,9 +26,20 @@ interface Props { * (no click). Merely visiting the authorize URL is attacker-inducible, so completion has to be a * deliberate user action, not a side effect of leaving the page. */ +export function isLoopbackOrigin(origin: string | null): boolean { + if (!origin) return false; + try { + const hostname = new URL(origin).hostname.replace(/^\[|\]$/g, ""); + return hostname === "localhost" || hostname === "::1" || /^127(\.\d{1,3}){3}$/.test(hostname); + } catch { + return false; + } +} + const ConnectFlowBanner: React.FC = ({ flowHandle, clientOrigin }) => { const action = `${getProxyBaseUrl()}/authorize/complete`; const clientLabel = clientOrigin ?? "the application"; + const loopbackClient = isLoopbackOrigin(clientOrigin); return (
@@ -50,6 +61,12 @@ const ConnectFlowBanner: React.FC = ({ flowHandle, clientOrigin }) => { > Finish connecting + {loopbackClient && ( + + )}
From f9c5be8ebfb13b20d671de12f76182ab1714ff15 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 30 Jul 2026 01:43:59 +0000 Subject: [PATCH 02/33] fix(fireworks_ai): correct Kimi K2.5/K2.6/K2.7 max output token limits Fireworks publishes a 262144-token context window for the Kimi K2.5, K2.6 and K2.7 models but caps generation well below that. Every fireworks_ai Kimi K2.5/K2.6/K2.7 alias had max_output_tokens/max_tokens flattened to 262144 (equal to the context window), so the pre-call context-window check admitted requests asking for a full 262144-token completion that Fireworks rejects. Correct max_output_tokens/max_tokens to 32768 while keeping max_input_tokens at 262144, and add a regression test pinning the limits for all ten aliases. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 40 +++++----- model_prices_and_context_window.json | 40 +++++----- .../test_fireworks_ai_kimi_model_metadata.py | 76 +++++++++++++++++++ 3 files changed, 116 insertions(+), 40 deletions(-) create mode 100644 tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87d9b6afc18..5b2a5efc714 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16703,8 +16703,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -16717,8 +16717,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -16733,8 +16733,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17077,8 +17077,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -17091,8 +17091,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17107,8 +17107,8 @@ "input_cost_per_token": 2e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17123,8 +17123,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17139,8 +17139,8 @@ "input_cost_per_token": 1.9e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -42501,8 +42501,8 @@ "input_cost_per_token": 2e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -42517,8 +42517,8 @@ "input_cost_per_token": 1.9e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0edd3bd5f30..b375b770aea 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16703,8 +16703,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -16717,8 +16717,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -16733,8 +16733,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17077,8 +17077,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -17091,8 +17091,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17107,8 +17107,8 @@ "input_cost_per_token": 2e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17123,8 +17123,8 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -17139,8 +17139,8 @@ "input_cost_per_token": 1.9e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -42622,8 +42622,8 @@ "input_cost_per_token": 2e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -42638,8 +42638,8 @@ "input_cost_per_token": 1.9e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py new file mode 100644 index 00000000000..5641439aa54 --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py @@ -0,0 +1,76 @@ +""" +Regression test for Fireworks Kimi K2.5 / K2.6 / K2.7 context and output limits. + +Fireworks publishes a 262144-token context window for every Kimi K2.5, K2.6 and +K2.7 model, but caps generation well below that. A previous bulk edit had flattened +max_output_tokens/max_tokens to 262144 (equal to the context window), which let the +pre-call context-window check admit requests asking for a full 262144-token +completion that Fireworks then rejects. These assertions pin the corrected per-alias +limits so a future bulk edit can't silently flatten them again. +""" + +import json +from importlib.resources import files + +import pytest + +CONTEXT_WINDOW = 262144 +OUTPUT_LIMIT = 32768 + +KIMI_ALIASES = ( + "fireworks_ai/kimi-k2p5", + "fireworks_ai/kimi-k2p6", + "fireworks_ai/kimi-k2p6-fast", + "fireworks_ai/kimi-k2p7-code", + "fireworks_ai/kimi-k2p7-code-fast", + "fireworks_ai/accounts/fireworks/models/kimi-k2p5", + "fireworks_ai/accounts/fireworks/models/kimi-k2p6", + "fireworks_ai/accounts/fireworks/models/kimi-k2p7-code", + "fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast", + "fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast", +) + + +@pytest.fixture(scope="module") +def use_local_model_cost_map(): + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + import litellm + from litellm.utils import _invalidate_model_cost_lowercase_map + + original_model_cost = litellm.model_cost + litellm.model_cost = json.loads( + files("litellm") + .joinpath("model_prices_and_context_window_backup.json") + .read_text(encoding="utf-8") + ) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + try: + yield litellm + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + monkeypatch.undo() + + +@pytest.mark.parametrize("alias", KIMI_ALIASES) +def test_fireworks_kimi_raw_cost_entry_limits(use_local_model_cost_map, alias): + entry = use_local_model_cost_map.model_cost[alias] + + assert entry["litellm_provider"] == "fireworks_ai" + assert entry["max_input_tokens"] == CONTEXT_WINDOW + assert entry["max_output_tokens"] == OUTPUT_LIMIT + assert entry["max_tokens"] == OUTPUT_LIMIT + assert entry["max_output_tokens"] < entry["max_input_tokens"] + + +@pytest.mark.parametrize("alias", KIMI_ALIASES) +def test_fireworks_kimi_get_model_info_limits(use_local_model_cost_map, alias): + model_info = use_local_model_cost_map.get_model_info(model=alias) + + assert model_info["max_input_tokens"] == CONTEXT_WINDOW + assert model_info["max_output_tokens"] == OUTPUT_LIMIT + assert model_info["max_tokens"] == OUTPUT_LIMIT From b0a48d516c53c4d44eb096fe61a77853404fceed Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 30 Jul 2026 01:59:25 +0000 Subject: [PATCH 03/33] test(fireworks_ai): align Kimi output-limit expectations with cost map fix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index b22e69f0942..3b2bc7e647d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4432,7 +4432,7 @@ _FIREWORKS_MODELS = [ 4e-06, 1.9e-07, 262144, - 262144, + 32768, True, True, ), @@ -4442,7 +4442,7 @@ _FIREWORKS_MODELS = [ 8e-06, 3.8e-07, 262144, - 262144, + 32768, True, True, ), @@ -4452,7 +4452,7 @@ _FIREWORKS_MODELS = [ 4e-06, 1.6e-07, 262144, - 262144, + 32768, True, True, ), @@ -4462,7 +4462,7 @@ _FIREWORKS_MODELS = [ 8e-06, 3e-07, 262144, - 262144, + 32768, True, True, ), From e0946ccf0d02bb558cd6aab98815ae30d525890f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 02:43:54 +0000 Subject: [PATCH 04/33] fix(pricing): correct gpt-5.4-mini and gpt-5.4-nano token limits gpt-5.4-mini and gpt-5.4-nano are 400K-context models (272K input, 128K output), but their cost map entries carried gpt-5.4's 1.05M window. The router's pre-call context window check therefore admitted prompts far past what the models accept, so oversized requests were dispatched to the provider and failed there instead of being caught locally or routed through context_window_fallbacks. The azure_ai entries also inherited gpt-5.4's above-272K tiered pricing. OpenAI applies that surcharge to the 1.05M-window models only, so those keys are removed. Limits per OpenAI's model reference and Azure AI Foundry's model table: gpt-5.4-mini and gpt-5.4-nano are 400,000 context / 272,000 input / 128,000 output --- ...odel_prices_and_context_window_backup.json | 40 ++------- model_prices_and_context_window.json | 48 +++-------- .../test_gpt_5_4_model_metadata.py | 81 +++++++++++++++++++ 3 files changed, 101 insertions(+), 68 deletions(-) create mode 100644 tests/test_litellm/test_gpt_5_4_model_metadata.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87d9b6afc18..cc418c9c428 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3454,22 +3454,16 @@ }, "azure_ai/gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, "cache_read_input_token_cost_priority": 1.5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, "input_cost_per_token": 7.5e-07, - "input_cost_per_token_above_272k_tokens": 1.5e-06, "input_cost_per_token_priority": 1.5e-06, - "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, - "output_cost_per_token_above_272k_tokens": 6.75e-06, "output_cost_per_token_priority": 9e-06, - "output_cost_per_token_above_272k_tokens_priority": 1.35e-05, "source": "https://ai.azure.com/catalog/models/gpt-5.4-mini", "supported_endpoints": [ "/v1/chat/completions", @@ -3500,22 +3494,16 @@ }, "azure_ai/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, "cache_read_input_token_cost_priority": 1.5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, "input_cost_per_token": 7.5e-07, - "input_cost_per_token_above_272k_tokens": 1.5e-06, "input_cost_per_token_priority": 1.5e-06, - "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, - "output_cost_per_token_above_272k_tokens": 6.75e-06, "output_cost_per_token_priority": 9e-06, - "output_cost_per_token_above_272k_tokens_priority": 1.35e-05, "source": "https://ai.azure.com/catalog/models/gpt-5.4-mini", "supported_endpoints": [ "/v1/chat/completions", @@ -3546,22 +3534,16 @@ }, "azure_ai/gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_priority": 4e-08, - "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_priority": 4e-07, - "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "output_cost_per_token_above_272k_tokens": 1.875e-06, "output_cost_per_token_priority": 2.5e-06, - "output_cost_per_token_above_272k_tokens_priority": 3.75e-06, "source": "https://ai.azure.com/catalog/models/gpt-5.4-nano", "supported_endpoints": [ "/v1/chat/completions", @@ -3592,22 +3574,16 @@ }, "azure_ai/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_priority": 4e-08, - "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_priority": 4e-07, - "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "output_cost_per_token_above_272k_tokens": 1.875e-06, "output_cost_per_token_priority": 2.5e-06, - "output_cost_per_token_above_272k_tokens_priority": 3.75e-06, "source": "https://ai.azure.com/catalog/models/gpt-5.4-nano", "supported_endpoints": [ "/v1/chat/completions", @@ -7201,7 +7177,7 @@ "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7236,7 +7212,7 @@ "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7271,7 +7247,7 @@ "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7306,7 +7282,7 @@ "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0edd3bd5f30..c4628fecdb8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3454,22 +3454,16 @@ }, "azure_ai/gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, "cache_read_input_token_cost_priority": 1.5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, "input_cost_per_token": 7.5e-07, - "input_cost_per_token_above_272k_tokens": 1.5e-06, "input_cost_per_token_priority": 1.5e-06, - "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, - "output_cost_per_token_above_272k_tokens": 6.75e-06, "output_cost_per_token_priority": 9e-06, - "output_cost_per_token_above_272k_tokens_priority": 1.35e-05, "source": "https://ai.azure.com/catalog/models/gpt-5.4-mini", "supported_endpoints": [ "/v1/chat/completions", @@ -3500,22 +3494,16 @@ }, "azure_ai/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_above_272k_tokens": 1.5e-07, "cache_read_input_token_cost_priority": 1.5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 3e-07, "input_cost_per_token": 7.5e-07, - "input_cost_per_token_above_272k_tokens": 1.5e-06, "input_cost_per_token_priority": 1.5e-06, - "input_cost_per_token_above_272k_tokens_priority": 3e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, - "output_cost_per_token_above_272k_tokens": 6.75e-06, "output_cost_per_token_priority": 9e-06, - "output_cost_per_token_above_272k_tokens_priority": 1.35e-05, "source": "https://ai.azure.com/catalog/models/gpt-5.4-mini", "supported_endpoints": [ "/v1/chat/completions", @@ -3546,22 +3534,16 @@ }, "azure_ai/gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_priority": 4e-08, - "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_priority": 4e-07, - "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "output_cost_per_token_above_272k_tokens": 1.875e-06, "output_cost_per_token_priority": 2.5e-06, - "output_cost_per_token_above_272k_tokens_priority": 3.75e-06, "source": "https://ai.azure.com/catalog/models/gpt-5.4-nano", "supported_endpoints": [ "/v1/chat/completions", @@ -3592,22 +3574,16 @@ }, "azure_ai/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_priority": 4e-08, - "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_priority": 4e-07, - "input_cost_per_token_above_272k_tokens_priority": 8e-07, "litellm_provider": "azure_ai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, - "output_cost_per_token_above_272k_tokens": 1.875e-06, "output_cost_per_token_priority": 2.5e-06, - "output_cost_per_token_above_272k_tokens_priority": 3.75e-06, "source": "https://ai.azure.com/catalog/models/gpt-5.4-nano", "supported_endpoints": [ "/v1/chat/completions", @@ -7201,7 +7177,7 @@ "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7236,7 +7212,7 @@ "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7271,7 +7247,7 @@ "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7306,7 +7282,7 @@ "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -24364,7 +24340,7 @@ "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -24410,7 +24386,7 @@ "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -24454,7 +24430,7 @@ "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -24497,7 +24473,7 @@ "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", - "max_input_tokens": 1050000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", diff --git a/tests/test_litellm/test_gpt_5_4_model_metadata.py b/tests/test_litellm/test_gpt_5_4_model_metadata.py new file mode 100644 index 00000000000..f93e6187dcb --- /dev/null +++ b/tests/test_litellm/test_gpt_5_4_model_metadata.py @@ -0,0 +1,81 @@ +import json +from functools import lru_cache +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +DOCUMENTED_MAX_INPUT_TOKENS = 272000 +DOCUMENTED_MAX_OUTPUT_TOKENS = 128000 + +SMALL_MODEL_NAMES = ( + "gpt-5.4-mini", + "gpt-5.4-mini-2026-03-17", + "gpt-5.4-nano", + "gpt-5.4-nano-2026-03-17", +) +SMALL_MODELS = tuple(f"{prefix}{name}" for prefix in ("", "azure/", "azure_ai/") for name in SMALL_MODEL_NAMES) + +STANDARD_PRICING = { + "gpt-5.4-mini": (7.5e-07, 4.5e-06, 7.5e-08), + "gpt-5.4-nano": (2e-07, 1.25e-06, 2e-08), +} + +LONG_CONTEXT_MODELS = ("gpt-5.4", "gpt-5.4-pro") + + +@lru_cache(maxsize=2) +def _load(path: Path) -> dict[str, dict[str, object]]: + with open(path) as f: + return json.load(f) + + +def _pricing_key(model: str) -> str: + return "gpt-5.4-nano" if "nano" in model else "gpt-5.4-mini" + + +@pytest.mark.parametrize("model", SMALL_MODELS) +def test_gpt_5_4_small_models_use_documented_token_limits(model: str) -> None: + """gpt-5.4-mini/nano are 400K-window models: 272K in, 128K out, not gpt-5.4's 1.05M window.""" + info = _load(MAIN_PATH).get(model) + assert info is not None, f"{model} not found in model_prices_and_context_window.json" + + assert info["max_input_tokens"] == DOCUMENTED_MAX_INPUT_TOKENS + assert info["max_output_tokens"] == DOCUMENTED_MAX_OUTPUT_TOKENS + assert info["max_tokens"] == DOCUMENTED_MAX_OUTPUT_TOKENS + + +@pytest.mark.parametrize("model", SMALL_MODELS) +def test_gpt_5_4_small_models_have_no_long_context_surcharge(model: str) -> None: + """OpenAI prices prompts above 272K at 2x input / 1.5x output for the 1.05M-window models only.""" + info = _load(MAIN_PATH)[model] + assert [key for key in info if "above_272k" in key] == [] + + +@pytest.mark.parametrize("model", SMALL_MODELS) +def test_gpt_5_4_small_models_standard_pricing(model: str) -> None: + info = _load(MAIN_PATH)[model] + input_cost, output_cost, cache_read_cost = STANDARD_PRICING[_pricing_key(model)] + + assert info["input_cost_per_token"] == input_cost + assert info["output_cost_per_token"] == output_cost + assert info["cache_read_input_token_cost"] == cache_read_cost + + +@pytest.mark.parametrize("model", LONG_CONTEXT_MODELS) +def test_gpt_5_4_long_context_models_keep_surcharge(model: str) -> None: + """The mini/nano correction must leave gpt-5.4 and gpt-5.4-pro tiered pricing intact.""" + info = _load(MAIN_PATH)[model] + + assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(info["input_cost_per_token"] * 2) + assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(info["output_cost_per_token"] * 1.5) + + +@pytest.mark.parametrize("model", SMALL_MODELS) +def test_gpt_5_4_small_models_backup_matches_main(model: str) -> None: + assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model), ( + f"{model} differs between main and backup model cost maps" + ) From fec7f5f2461ab610858f870b65bae5108b3844df Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 28 Jul 2026 15:20:07 -0700 Subject: [PATCH 05/33] feat(ui): give auto-routers their own tab on Models + Endpoints Auto-routers had no home and no list. The create form was mounted in two unrelated places, inside Models + Endpoints > Add Model and again under Cost Optimization, and neither showed which auto routers already existed; seeing or editing one meant finding its row in the models table and drilling in. They now get a dedicated Auto-Routers tab beside All Models, listing every auto_router/* deployment with create, edit and delete in one place, and both former entry points are removed. Creating opens in a shadcn dialog rather than swapping the whole panel out, so the list stays on screen behind it; the dialog caps its height and scrolls, since the complexity form is long. The form's own heading goes with it, the dialog header owning that now. An auto router is a routing construct rather than a deployment, so it also comes off the All Models table. That table pages server-side off total_count, so a client-side filter would page over a total including rows it never renders; /v2/model/info therefore gains exclude_auto_routers (default false, so every existing caller is unaffected) and the filter runs before the count. /v1/models is untouched, so clients still see auto-routers as models. Clicking a router opens the same `?model=` drill-in the All Models table uses, so it lands in ModelInfoView with the full Model Settings, Edit Settings, Edit Auto Router and Delete. An earlier revision had a bespoke detail page here, which was a partial reimplementation of that view and showed the router's type twice, once as a Type pill and again as a "Routing strategy" field saying the same thing. Both are gone. The auto-router list is keyed under the same `models/list` namespace as the models table rather than a private one. It reads the same /v2/model/info data, and six call sites across the app already invalidate ["models","list"] after a write; a separate key meant an edit made through ModelInfoView left the tab stale until a full reload, and every future writer would have had to remember a second key. An auto router has no upstream credential, so its detail header drops Update API Key and Re-use Credentials, and the destructive action names what it removes rather than saying model. Test Connection was gated on the editor-aware predicate, which let adaptive and quality routers through to a check that builds its targets from complexity config they do not have; it now gates on the deployment predicate. The edit modal also applies the semantic-matching guard the create form has. It renders those controls now, and the backend raises on semantic_keyword_matching without an embedding model or keyword rules, so skipping the shared validator turned an inline message into a raw 400. Whether a row is writable has two independent axes and the dashboard needs both. STRATEGY: there are four auto_router/* kinds and only complexity and semantic have a form here, so adaptive and quality must not be handed an editor that would write auto_router_config onto a deployment storing its settings elsewhere. ORIGIN: a config.yaml row reports db_model false and the API refuses it whatever its strategy (PATCH /model/{id}/update 404s, POST /model/delete 400s). Capability is derived per capability rather than as one editable flag, because the constraints differ: editing needs an editor, deleting removes a row by id and never reads its config, so a DB-created adaptive router stays deletable. Both axes live in add_model/auto_router_strategies.ts as a declarative table, one record per strategy, so a fifth strategy is a table row rather than another branch. That also retired four copies of "is this a complexity router", one of which was written twice in a row in model_info_view. Creation narrows to the complexity router, which the UI calls Auto-Router v2; the semantic option was already badged "to be deprecated" in the picker, so the picker goes away along with the semantic submit path and its validation helper. Existing semantic routers stay editable. The edit modal mounted ComplexityRouterConfig without the keyword, escalation and semantic-matching handlers, so those sections never rendered and could only be set at create time. It now hydrates them from the stored config, and the five keys become managed only when a caller supplies that state, so a caller rendering no such control still carries them through. A component-level round-trip test covers it: a payload-builder test cannot see a hydration bug. A complexity tier is str | list[str] on the backend, and the UI carried three readers of that rule, one of which dropped a pinned string. They collapse into one owner, add_model/complexity_router_tiers.ts. --- litellm/proxy/proxy_server.py | 29 ++ .../proxy_server/test_routes_model_info.py | 146 ++++++++ ui/litellm-dashboard/eslint-suppressions.json | 27 +- .../_components/AutorouterTab.tsx | 28 -- .../CostOptimizationView.activity.test.tsx | 1 - .../_components/CostOptimizationView.test.tsx | 7 +- .../_components/CostOptimizationView.tsx | 9 +- .../hooks/models/useModels.test.ts | 27 ++ .../app/(dashboard)/hooks/models/useModels.ts | 92 ++++- .../components/AllModelsTab.tsx | 3 + .../AutoRouters/AutoRoutersPanel.test.tsx | 252 ++++++++++++++ .../AutoRouters/AutoRoutersPanel.tsx | 117 +++++++ .../AutoRouters/AutoRoutersTable.tsx | 68 ++++ .../AutoRouters/AutoRoutersTableColumns.tsx | 173 ++++++++++ .../AutoRouters/autoRouterRows.test.ts | 196 +++++++++++ .../components/AutoRouters/autoRouterRows.ts | 103 ++++++ .../components/AutoRouters/fitPills.test.ts | 43 +++ .../components/AutoRouters/fitPills.ts | 52 +++ .../models-and-endpoints/page.test.tsx | 31 ++ .../(dashboard)/models-and-endpoints/page.tsx | 22 +- .../panels/AddModelPanel.tsx | 8 +- .../panels/AutoRoutersTabPanel.tsx | 23 ++ .../_components/general_settings.test.tsx | 17 +- .../_components/general_settings.tsx | 152 +++++---- .../add_model/AddModelForm.test.tsx | 13 + .../add_model/add_auto_router_tab.test.tsx | 6 +- .../add_model/add_auto_router_tab.tsx | 197 ++--------- .../add_model/add_model_tab.test.tsx | 318 ------------------ .../components/add_model/add_model_tab.tsx | 98 ------ .../add_model/auto_router_strategies.ts | 143 ++++++++ .../build_complexity_router_config.ts | 8 +- .../build_semantic_router_validation.test.ts | 67 ---- .../build_semantic_router_validation.ts | 29 -- .../add_model/complexity_router_keywords.ts | 40 +++ .../add_model/complexity_router_tiers.test.ts | 30 ++ .../add_model/complexity_router_tiers.ts | 14 + ...d_updated_complexity_router_config.test.ts | 88 +++++ .../edit_auto_router_modal.test.tsx | 121 +++++++ .../edit_auto_router_modal.tsx | 122 +++++-- .../src/components/leftnav.test.tsx | 21 ++ .../src/components/model_info_view.test.tsx | 85 +++++ .../src/components/model_info_view.tsx | 78 +++-- .../src/components/networking.tsx | 4 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 44 files changed, 2198 insertions(+), 912 deletions(-) delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AutoRoutersTabPanel.tsx delete mode 100644 ui/litellm-dashboard/src/components/add_model/add_model_tab.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/auto_router_strategies.ts delete mode 100644 ui/litellm-dashboard/src/components/add_model/build_semantic_router_validation.test.ts delete mode 100644 ui/litellm-dashboard/src/components/add_model/build_semantic_router_validation.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/complexity_router_keywords.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts create mode 100644 ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts create mode 100644 ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 70484eb1e4e..bf010c7f249 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11773,6 +11773,22 @@ def _sort_models( return all_models +def _is_auto_router_model(model: Mapping[str, object]) -> bool: + """ + True for any auto-router deployment, i.e. every `auto_router/*` strategy + (semantic, complexity, adaptive, quality). + + Router._is_auto_router_deployment is deliberately narrower; it answers "is this the + *semantic* auto-router strategy" and returns False for the complexity and adaptive + prefixes, so it is not reusable here. + """ + litellm_params = model.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return False + litellm_model = litellm_params.get("model") + return isinstance(litellm_model, str) and litellm_model.startswith("auto_router/") + + def _paginate_models_response( all_models: List[Dict[str, Any]], page: int, @@ -12067,6 +12083,14 @@ async def model_info_v2( "asc", description="Sort order. Options: asc, desc", ), + exclude_auto_routers: bool | None = fastapi.Query( + False, + description=( + "Omit auto-router deployments (litellm model prefixed `auto_router/`). " + "They are routing constructs rather than deployments, and are managed on the " + "Router Settings page. Defaults to false, so existing callers are unaffected" + ), + ), ): """ Paginated model metadata for proxy deployments (pricing, provider, team access). @@ -12234,6 +12258,11 @@ async def model_info_v2( user_api_key_dict=user_api_key_dict, ) + # `is True` because direct-call tests bypass FastAPI, so the Query default arrives as a + # truthy sentinel object rather than False. + if exclude_auto_routers is True: + all_models = [m for m in all_models if not _is_auto_router_model(m)] + # Update total count to include agents search_total_count = len(all_models) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py index 00c3c5b1e74..b0e8a85d3fa 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py @@ -292,3 +292,149 @@ def test_model_group_info_invalid_method(client, auth_as, null_router): response = client.post("/model_group/info", json={}) assert response.status_code == 405 assert len(response.content) > 0 + + +# --------------------------------------------------------------------------- +# GET /v2/model/info?exclude_auto_routers +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mixed_auto_router_router(monkeypatch): + """Router carrying one ordinary deployment per auto-router strategy plus two plain ones.""" + model_list = [ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"id": "plain-1", "db_model": False}, + }, + { + "model_name": "tri-tier-router", + "litellm_params": {"model": "auto_router/complexity_router"}, + "model_info": {"id": "auto-complexity", "db_model": True}, + }, + { + "model_name": "support-router", + "litellm_params": {"model": "auto_router/support-router"}, + "model_info": {"id": "auto-semantic", "db_model": True}, + }, + { + "model_name": "adaptive-router", + "litellm_params": {"model": "auto_router/adaptive_router"}, + "model_info": {"id": "auto-adaptive", "db_model": True}, + }, + { + "model_name": "claude-opus", + "litellm_params": {"model": "anthropic/claude-opus-4-6"}, + "model_info": {"id": "plain-2", "db_model": False}, + }, + ] + from unittest.mock import AsyncMock + + router = MagicMock() + router.model_list = model_list + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(return_value={})) + monkeypatch.setattr( + proxy_server, + "_apply_search_filter_to_models", + AsyncMock(side_effect=lambda all_models, **kw: (all_models, len(all_models))), + ) + monkeypatch.setattr(proxy_server, "_enrich_model_info_with_litellm_data", lambda model, **kw: model) + + import litellm.proxy.agent_endpoints.model_list_helpers as mlh + + monkeypatch.setattr(mlh, "append_agents_to_model_info", AsyncMock(side_effect=lambda models, **kw: models)) + yield router + + +def _model_names(payload) -> list: + return [m["model_name"] for m in payload["data"]] + + +def test_v2_model_info_includes_auto_routers_by_default(client, auth_as, mixed_auto_router_router): + """The new param is opt-in; omitting it must not change what any existing caller sees.""" + with auth_as(): + response = client.get("/v2/model/info") + assert response.status_code == 200 + payload = response.json() + assert "tri-tier-router" in _model_names(payload) + assert payload["total_count"] == 5 + + +def test_v2_model_info_excludes_every_auto_router_strategy(client, auth_as, mixed_auto_router_router): + """All four `auto_router/*` strategies go, not just the semantic one that + Router._is_auto_router_deployment recognises.""" + with auth_as(): + response = client.get("/v2/model/info", params={"exclude_auto_routers": "true"}) + assert response.status_code == 200 + payload = response.json() + assert _model_names(payload) == ["gpt-4o-mini", "claude-opus"] + + +def test_v2_model_info_exclude_auto_routers_shrinks_total_count(client, auth_as, mixed_auto_router_router): + """The filter must run before the count, or the table pages off a total that + includes rows it never renders (49 shown, 50 claimed).""" + with auth_as(): + response = client.get("/v2/model/info", params={"exclude_auto_routers": "true"}) + payload = response.json() + assert payload["total_count"] == 2 + assert len(payload["data"]) == payload["total_count"] + + +def test_v2_model_info_exclude_auto_routers_paginates_over_the_filtered_set( + client, auth_as, mixed_auto_router_router +): + """Page size applies to the filtered list, so no page silently comes back short.""" + with auth_as(): + response = client.get( + "/v2/model/info", params={"exclude_auto_routers": "true", "page": 1, "size": 1} + ) + payload = response.json() + assert payload["total_count"] == 2 + assert payload["total_pages"] == 2 + assert len(payload["data"]) == 1 + + +@pytest.mark.asyncio +async def test_model_info_v2_query_sentinel_does_not_filter(monkeypatch, mixed_auto_router_router): + """Called directly (not through FastAPI) the default arrives as a truthy Query object. + Guarding on `is True` is what stops every direct-call test from silently filtering.""" + from unittest.mock import AsyncMock + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(return_value={})) + monkeypatch.setattr( + proxy_server, + "_apply_search_filter_to_models", + AsyncMock(side_effect=lambda all_models, **kw: (all_models, len(all_models))), + ) + monkeypatch.setattr(proxy_server, "_enrich_model_info_with_litellm_data", lambda model, **kw: model) + + import litellm.proxy.agent_endpoints.model_list_helpers as mlh + + monkeypatch.setattr(mlh, "append_agents_to_model_info", AsyncMock(side_effect=lambda models, **kw: models)) + + admin = UserAPIKeyAuth(user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + # Deliberately omit exclude_auto_routers, exactly as the pre-existing direct-call tests do. + resp = await proxy_server.model_info_v2( + user_api_key_dict=admin, + model=None, + user_models_only=False, + include_team_models=False, + debug=False, + page=1, + size=50, + search=None, + modelId=None, + teamId=None, + sortBy=None, + sortOrder="asc", + ) + + assert "tri-tier-router" in [m["model_name"] for m in resp["data"]] diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 7686cc05fa6..c0d24d3a1e6 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -210,11 +210,6 @@ "count": 1 } }, - "src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx": { "no-restricted-imports": { "count": 1 @@ -1696,7 +1691,7 @@ "count": 1 }, "no-restricted-imports": { - "count": 3 + "count": 2 }, "prefer-const": { "count": 2 @@ -2550,17 +2545,12 @@ "count": 1 } }, - "src/components/add_model/add_auto_router_tab.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/add_auto_router_tab.tsx": { "local/filename-pascal-case": { "count": 1 }, "no-restricted-imports": { - "count": 3 + "count": 2 } }, "src/components/add_model/add_model_modes.tsx": { @@ -2568,19 +2558,6 @@ "count": 1 } }, - "src/components/add_model/add_model_tab.test.tsx": { - "no-restricted-imports": { - "count": 2 - } - }, - "src/components/add_model/add_model_tab.tsx": { - "local/filename-pascal-case": { - "count": 1 - }, - "no-restricted-imports": { - "count": 4 - } - }, "src/components/add_model/advanced_settings.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx deleted file mode 100644 index 3474036528a..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx +++ /dev/null @@ -1,28 +0,0 @@ -"use client"; - -import React from "react"; -import { Form } from "antd"; - -import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab"; - -interface AutorouterTabProps { - accessToken: string | null; - userId: string | null; - userRole: string; -} - -const AutorouterTab: React.FC = ({ accessToken, userRole }) => { - const [form] = Form.useForm(); - - if (!accessToken) { - return null; - } - - return ( -
- form.resetFields()} accessToken={accessToken} userRole={userRole} /> -
- ); -}; - -export default AutorouterTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 363525c48af..ca7adf07941 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -27,7 +27,6 @@ vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => })); vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); -vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () =>
})); import CostOptimizationView from "./CostOptimizationView"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx index 46aa23fcfc0..42dc7719144 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx @@ -3,7 +3,6 @@ import { describe, expect, it, vi } from "vitest"; vi.mock("./UsageTab", () => ({ __esModule: true, default: () =>
})); vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); -vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () =>
})); vi.mock("./PromptCachingTab", () => ({ __esModule: true, default: () =>
})); import CostOptimizationView from "./CostOptimizationView"; @@ -11,13 +10,13 @@ import CostOptimizationView from "./CostOptimizationView"; const renderView = () => render(); describe("CostOptimizationView", () => { - it("renders all four cost-optimization tabs", () => { - const { getByText } = renderView(); + it("renders the three cost-optimization tabs and no autorouter tab", () => { + const { getByText, queryByText } = renderView(); expect(getByText("Usage")).toBeInTheDocument(); expect(getByText("Prompt Compression")).toBeInTheDocument(); - expect(getByText("Autorouter")).toBeInTheDocument(); expect(getByText("Prompt Caching")).toBeInTheDocument(); + expect(queryByText("Autorouter")).not.toBeInTheDocument(); }); it("defaults to the Usage tab and switches the active tab on click", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index 3bab6afee57..9c2d0b20b56 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -6,7 +6,6 @@ import { Alert, Tabs } from "antd"; import UsageTab from "./UsageTab"; import PromptCompressionTab from "./PromptCompressionTab"; -import AutorouterTab from "./AutorouterTab"; import PromptCachingTab from "./PromptCachingTab"; import { useDailyActivityRange } from "./useDailyActivityRange"; @@ -30,11 +29,6 @@ const CostOptimizationView: React.FC = ({ accessToken label: "Prompt Compression", children: , }, - { - key: "autorouter", - label: "Autorouter", - children: , - }, { key: "caching", label: "Prompt Caching", @@ -50,7 +44,8 @@ const CostOptimizationView: React.FC = ({ accessToken

Cost Optimization

- Track and configure the mechanisms that save you money: prompt compression, prompt caching, and auto routing + Track and configure the mechanisms that save you money: prompt compression and prompt caching. Auto routers + live on the Router Settings page

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts index f83ebd2622a..f04c4b7bfcd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts @@ -7,6 +7,7 @@ import { selectAutoRouterModelGroups, useAllProxyModels, useAutoRouterModelGroups, + useAutoRouters, useInfiniteModelInfo, useModelHub, useModelsInfo, @@ -113,6 +114,9 @@ describe("useModelsInfo", () => { undefined, undefined, undefined, + // exclude_auto_routers defaults off: only the Models + Endpoints table opts in, so + // every other consumer of this hook keeps seeing auto-routers. + false, ); expect(modelInfoCall).toHaveBeenCalledTimes(1); }); @@ -137,6 +141,9 @@ describe("useModelsInfo", () => { undefined, undefined, undefined, + // exclude_auto_routers defaults off: only the Models + Endpoints table opts in, so + // every other consumer of this hook keeps seeing auto-routers. + false, ); }); @@ -1079,4 +1086,24 @@ describe("useAutoRouterModelGroups", () => { await waitFor(() => expect(modelInfoCall).toHaveBeenCalled()); expect(result.current.size).toBe(0); }); + + // The Auto-Routers tab and the models table read the same /v2/model/info data. Six call + // sites across the app invalidate ["models","list"] after a write; if the auto-router query + // sits in its own namespace, an edit through ModelInfoView leaves the tab stale until a full + // reload, and every future writer has to remember a second key. + describe("auto-router cache namespace", () => { + it("keys the auto-router list under models/list so existing invalidations reach it", async () => { + (modelInfoCall as any).mockResolvedValue(mockPaginatedModelInfoResponse); + const { result } = renderHook(() => useAutoRouters(), { wrapper }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + + const keys = queryClient + .getQueryCache() + .findAll({ queryKey: ["models", "list"] }) + .map((query) => query.queryKey); + + expect(keys.some((key) => JSON.stringify(key).includes("autoRouters"))).toBe(true); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index ad5e3c91ec3..88c4836f112 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -1,4 +1,4 @@ -import { useQuery, useInfiniteQuery, UseQueryResult } from "@tanstack/react-query"; +import { useQuery, useInfiniteQuery, useQueryClient, UseQueryResult } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { modelInfoCall, modelHubCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "../useAuthorized"; @@ -24,7 +24,6 @@ export interface PaginatedModelInfoResponse { const modelKeys = createQueryKeys("models"); const modelHubKeys = createQueryKeys("modelHub"); -const autoRouterKeys = createQueryKeys("autoRouterModelGroups"); const allProxyModelsKeys = createQueryKeys("allProxyModels"); const selectedTeamModelsKeys = createQueryKeys("selectedTeamModels"); const infiniteModelKeys = createQueryKeys("infiniteModels"); @@ -38,6 +37,7 @@ export const useModelsInfo = ( teamId?: string, sortBy?: string, sortOrder?: string, + excludeAutoRouters: boolean = false, ) => { const { accessToken, userId, userRole } = useAuthorized(); return useQuery({ @@ -52,10 +52,25 @@ export const useModelsInfo = ( ...(teamId && { teamId }), ...(sortBy && { sortBy }), ...(sortOrder && { sortOrder }), + // Part of the key: callers that exclude auto-routers must not share a cache entry + // with callers that keep them. + ...(excludeAutoRouters && { excludeAutoRouters: "true" }), }, }), queryFn: async () => - await modelInfoCall(accessToken!, userId!, userRole!, page, size, search, modelId, teamId, sortBy, sortOrder), + await modelInfoCall( + accessToken!, + userId!, + userRole!, + page, + size, + search, + modelId, + teamId, + sortBy, + sortOrder, + excludeAutoRouters, + ), enabled: Boolean(accessToken && userId && userRole), }); }; @@ -69,6 +84,29 @@ export interface AutoRouterCandidateDeployment { litellm_params?: { model?: string | null } | null; } +export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { + litellm_params?: { + model?: string | null; + complexity_router_config?: unknown; + complexity_router_default_model?: string | null; + auto_router_config?: unknown; + auto_router_default_model?: string | null; + auto_router_embedding_model?: string | null; + adaptive_router_config?: unknown; + adaptive_router_default_model?: string | null; + quality_router_config?: unknown; + quality_router_default_model?: string | null; + } | null; + model_info?: { + id?: string | null; + /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ + db_model?: boolean | null; + created_at?: string | null; + updated_at?: string | null; + team_id?: string | null; + } | null; +} + export const isAutoRouterDeployment = (deployment: AutoRouterCandidateDeployment): boolean => Boolean(deployment?.litellm_params?.model?.startsWith(AUTO_ROUTER_MODEL_PREFIX)); @@ -80,11 +118,14 @@ export const selectAutoRouterModelGroups = (deployments: AutoRouterCandidateDepl .filter((modelName): modelName is string => Boolean(modelName)), ); +export const selectAutoRouterDeployments = (deployments: AutoRouterDeployment[]): AutoRouterDeployment[] => + deployments.filter(isAutoRouterDeployment); + const fetchAllModelDeployments = async ( accessToken: string, userId: string, userRole: string, -): Promise => { +): Promise => { const firstPage: PaginatedModelInfoResponse = await modelInfoCall( accessToken, userId, @@ -100,18 +141,28 @@ const fetchAllModelDeployments = async ( ); return [firstPage, ...remainingPages].flatMap( (page: PaginatedModelInfoResponse) => page?.data ?? [], - ) as AutoRouterCandidateDeployment[]; + ) as AutoRouterDeployment[]; }; +/** + * Deliberately under the same `models/list` namespace as useModelsInfo: it is the same + * /v2/model/info data, and every writer in the app already invalidates ["models","list"]. + * A private namespace meant an edit through ModelInfoView left this list stale, and every + * future writer would have had to remember a second key. + */ +const autoRouterListKey = (userId: string | null, userRole: string | null) => + modelKeys.list({ + filters: { + scope: "autoRouters", + ...(userId && { userId }), + ...(userRole && { userRole }), + }, + }); + export const useAutoRouterModelGroups = (): ReadonlySet => { const { accessToken, userId, userRole } = useAuthorized(); - const { data } = useQuery>({ - queryKey: autoRouterKeys.list({ - filters: { - ...(userId && { userId }), - ...(userRole && { userRole }), - }, - }), + const { data } = useQuery>({ + queryKey: autoRouterListKey(userId, userRole), queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), enabled: Boolean(accessToken && userId && userRole), select: selectAutoRouterModelGroups, @@ -119,6 +170,23 @@ export const useAutoRouterModelGroups = (): ReadonlySet => { return data ?? NO_AUTO_ROUTERS; }; +export const useAutoRouters = (): UseQueryResult => { + const { accessToken, userId, userRole } = useAuthorized(); + return useQuery({ + queryKey: autoRouterListKey(userId, userRole), + queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), + enabled: Boolean(accessToken && userId && userRole), + select: selectAutoRouterDeployments, + }); +}; + +export const useInvalidateAutoRouters = (): (() => Promise) => { + const queryClient = useQueryClient(); + return async () => { + await queryClient.invalidateQueries({ queryKey: modelKeys.lists() }); + }; +}; + export const useModelHub = () => { const { accessToken } = useAuthorized(); return useQuery({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx index 1dc7736d5ac..1a65109eac8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx @@ -104,6 +104,9 @@ const AllModelsTab = ({ teamIdForQuery, sortBy, sortOrder, + // Auto-routers are routing constructs, not deployments; they are listed and managed on + // the Router Settings page. Excluded server-side so total_count stays honest. + true, ); const isLoading = isLoadingModelsInfo || isLoadingModelCostMap; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx new file mode 100644 index 00000000000..7d026f067fb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx @@ -0,0 +1,252 @@ +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { renderWithProviders, screen, testQueryClient, waitFor } from "@/../tests/test-utils"; + +import { AutoRoutersPanel } from "./AutoRoutersPanel"; + +const { modelInfoCall, modelDeleteCall } = vi.hoisted(() => ({ + modelInfoCall: vi.fn(), + modelDeleteCall: vi.fn().mockResolvedValue({}), +})); + +vi.mock("@/components/networking", () => ({ + modelInfoCall, + modelDeleteCall, + modelHubCall: vi.fn(), + modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([]), +})); + +const { openModel } = vi.hoisted(() => ({ openModel: vi.fn() })); + +vi.mock("@/app/(dashboard)/models-and-endpoints/detailNavigation", () => ({ + useModelDetailRouting: () => ({ openModel, modelId: null, teamId: null, openTeam: vi.fn(), close: vi.fn() }), +})); + +vi.mock("@/components/edit_auto_router/edit_auto_router_modal", () => ({ + __esModule: true, + default: ({ modelData }: { modelData: { model_name?: string; model_info?: { id?: string } } }) => ( +
+ edit:{modelData.model_name}:{modelData.model_info?.id} +
+ ), +})); + +vi.mock("@/components/add_model/add_auto_router_tab", () => ({ + __esModule: true, + default: ({ handleOk }: { handleOk: () => void }) => ( + + ), +})); + +// A realistic /v2/model/info page: two auto-routers among ordinary deployments. The panel must +// render exactly the auto_router/* rows; a view that renders page.data unfiltered passes a +// "renders a table" assertion but fails this one. +const DEPLOYMENTS = [ + { + // DB-created adaptive router: no editor for its shape, but it must stay deletable, since + // auto-routers are excluded from Models + Endpoints and this tab is the only delete path. + model_name: "adaptive-router", + litellm_params: { model: "auto_router/adaptive_router" }, + model_info: { id: "auto-3", db_model: true }, + }, + { + // config.yaml row: the API refuses both update and delete, so neither control may appear. + model_name: "config-router", + litellm_params: { + model: "auto_router/complexity_router", + complexity_router_config: { tiers: {}, classifier_type: "llm" }, + }, + model_info: { id: "auto-4", db_model: false }, + }, + { + model_name: "gpt-4o-mini", + litellm_params: { model: "openai/gpt-4o-mini" }, + model_info: { id: "plain-1" }, + }, + { + model_name: "tri-tier-router", + litellm_params: { + model: "auto_router/complexity_router", + complexity_router_config: { tiers: { SIMPLE: ["gpt-4o-mini"] }, classifier_type: "heuristic" }, + complexity_router_default_model: "gpt-4o-mini", + }, + model_info: { id: "auto-1", db_model: true, created_at: "2026-07-28T21:40:09.900000+00:00" }, + }, + { + model_name: "anthropic-opus-4-6", + litellm_params: { model: "anthropic/claude-opus-4-6" }, + model_info: { id: "plain-2" }, + }, + { + model_name: "support-router", + litellm_params: { + model: "auto_router/support-router", + auto_router_config: JSON.stringify({ routes: [{ name: "gpt-4o-mini" }] }), + auto_router_default_model: "gpt-4o-mini", + }, + model_info: { id: "auto-2", db_model: true, created_at: "2026-07-27T10:00:00.000000+00:00" }, + }, +]; + +const pageOf = (data: typeof DEPLOYMENTS) => ({ + data, + total_count: data.length, + current_page: 1, + total_pages: 1, + size: 1000, +}); + +const mockDeploymentsPage = () => { + modelInfoCall.mockResolvedValue(pageOf(DEPLOYMENTS)); +}; + +const renderPanel = (canModify = true) => + renderWithProviders(); + +describe("AutoRoutersPanel", () => { + beforeEach(() => { + // The shared test client caches with staleTime: Infinity and refetchOnMount: false, so + // without this every test after the first reads the previous test's deployment page. + testQueryClient.clear(); + modelInfoCall.mockReset(); + modelDeleteCall.mockClear(); + openModel.mockClear(); + mockDeploymentsPage(); + }); + + it("lists only auto_router deployments, not every model on the proxy", async () => { + renderPanel(); + + expect(await screen.findByText("tri-tier-router")).toBeInTheDocument(); + expect(await screen.findByText("support-router")).toBeInTheDocument(); + expect(screen.queryByText("gpt-4o-mini", { selector: "span.text-sm.font-medium" })).not.toBeInTheDocument(); + expect(screen.queryByText("anthropic-opus-4-6", { selector: "span.text-sm.font-medium" })).not.toBeInTheDocument(); + }); + + it("labels Type by classifier rather than by router family", async () => { + renderPanel(); + + expect(await screen.findByText("Heuristic")).toBeInTheDocument(); + expect(await screen.findByText("Semantic")).toBeInTheDocument(); + }); + + // Reuses the models-page drill-in, so an auto router opens the full ModelInfoView with + // Model Settings and Edit Settings, not a parallel detail view that reimplements part of it. + it("opens the shared model detail view on row click", async () => { + const user = userEvent.setup(); + renderPanel(); + + await user.click(await screen.findByRole("button", { name: "support-router" })); + + expect(openModel).toHaveBeenCalledWith("auto-2"); + }); + + it("opens the create form in a dialog and refetches the list after a create", async () => { + const user = userEvent.setup(); + renderPanel(); + + await screen.findByText("tri-tier-router"); + const callsBeforeCreate = modelInfoCall.mock.calls.length; + + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Add Auto Router" })); + + // A dialog, not a full-panel swap: the list stays mounted behind it. + const dialog = await screen.findByRole("dialog"); + expect(dialog).toHaveTextContent("Add Auto Router"); + expect(screen.getByText("tri-tier-router")).toBeInTheDocument(); + + await user.click(await screen.findByRole("button", { name: "Submit auto router" })); + + // Back on the list, and the deployment query was invalidated so a new router shows up + // without a manual page reload. + expect(await screen.findByText("tri-tier-router")).toBeInTheDocument(); + await waitFor(() => expect(modelInfoCall.mock.calls.length).toBeGreaterThan(callsBeforeCreate)); + }); + + // The page decides who may write (proxy admin or team admin); the panel just has to make + // every write affordance absent when told no, rather than let a submit 403 later. Reading + // stays open: a read-only caller can still drill into the detail view. + it("shows the list but no write affordances when canModify is false", async () => { + renderPanel(false); + + expect(await screen.findByText("tri-tier-router")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Add Auto Router" })).not.toBeInTheDocument(); + expect(screen.queryByTestId("auto-router-actions-auto-1")).not.toBeInTheDocument(); + // Still navigable, because opening the detail view is a read. + expect(screen.getByRole("button", { name: "tri-tier-router" })).toBeInTheDocument(); + }); + + // Auto-routers are hidden from Models + Endpoints, which used to be the only route to the + // delete action, so this tab is now the only place an auto router can be removed. + it("deletes the chosen router by its model id and refetches", async () => { + const user = userEvent.setup(); + renderPanel(); + + await screen.findByText("support-router"); + const callsBeforeDelete = modelInfoCall.mock.calls.length; + + await user.click(screen.getByTestId("auto-router-actions-auto-2")); + await user.click(await screen.findByTestId("auto-router-action-delete")); + await user.click(await screen.findByRole("button", { name: /^delete$/i })); + + await waitFor(() => expect(modelDeleteCall).toHaveBeenCalledWith("token", "auto-2")); + await waitFor(() => expect(modelInfoCall.mock.calls.length).toBeGreaterThan(callsBeforeDelete)); + }); + + it("does not delete when the confirmation is dismissed", async () => { + const user = userEvent.setup(); + renderPanel(); + + await screen.findByText("support-router"); + + await user.click(screen.getByTestId("auto-router-actions-auto-2")); + await user.click(await screen.findByTestId("auto-router-action-delete")); + await user.click(await screen.findByRole("button", { name: /cancel/i })); + + expect(modelDeleteCall).not.toHaveBeenCalled(); + }); + + it("gives a read-only caller no delete affordance", async () => { + renderPanel(false); + + await screen.findByText("support-router"); + expect(screen.queryByTestId("auto-router-actions-auto-2")).not.toBeInTheDocument(); + }); + + it("renders an empty state when the proxy has models but no auto routers", async () => { + modelInfoCall.mockResolvedValue( + pageOf(DEPLOYMENTS.filter((d) => !d.litellm_params.model.startsWith("auto_router/"))), + ); + + renderPanel(); + + expect(await screen.findByText("No auto routers yet")).toBeInTheDocument(); + }); + + it("keeps delete available on a DB-created adaptive router that has no editor", async () => { + const user = userEvent.setup(); + renderPanel(); + + await screen.findByText("adaptive-router"); + await user.click(screen.getByTestId("auto-router-actions-auto-3")); + await user.click(await screen.findByTestId("auto-router-action-delete")); + await user.click(await screen.findByRole("button", { name: /^delete$/i })); + + await waitFor(() => expect(modelDeleteCall).toHaveBeenCalledWith("token", "auto-3")); + }); + + it("offers no delete on a config-defined router, which the API would refuse", async () => { + renderPanel(); + + await screen.findByText("config-router"); + expect(screen.queryByTestId("auto-router-actions-auto-4")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx new file mode 100644 index 00000000000..3fbce1bd4a0 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { Plus } from "lucide-react"; +import { useMemo, useState } from "react"; + +import { useAutoRouters, useInvalidateAutoRouters } from "@/app/(dashboard)/hooks/models/useModels"; +import { useModelDetailRouting } from "@/app/(dashboard)/models-and-endpoints/detailNavigation"; +import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab"; +import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { modelDeleteCall } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; + +import { AutoRoutersTable } from "./AutoRoutersTable"; +import { AutoRouterRow, toAutoRouterRows } from "./autoRouterRows"; + +interface AutoRoutersPanelProps { + accessToken: string; + userRole: string; + /** Owned by the page, which knows whether the caller may write. */ + canModify: boolean; +} + +export function AutoRoutersPanel({ accessToken, userRole, canModify }: AutoRoutersPanelProps) { + const { data: deployments, isLoading } = useAutoRouters(); + const invalidateAutoRouters = useInvalidateAutoRouters(); + // Clicking a router opens the same ?model= drill-in the All Models table uses, so an auto + // router gets the full ModelInfoView: Model Settings, Edit Settings, Edit Auto Router and + // Delete. A separate detail view here would be a worse copy of it. + const { openModel } = useModelDetailRouting(); + const [isCreating, setIsCreating] = useState(false); + const [deletingRouter, setDeletingRouter] = useState(null); + const [isDeleting, setIsDeleting] = useState(false); + + const routers = useMemo(() => toAutoRouterRows(deployments ?? []), [deployments]); + + const handleCreated = () => { + setIsCreating(false); + void invalidateAutoRouters(); + }; + + const handleConfirmDelete = async () => { + if (!deletingRouter) return; + setIsDeleting(true); + try { + await modelDeleteCall(accessToken, deletingRouter.id); + NotificationsManager.success(`Deleted auto router: ${deletingRouter.name}`); + setDeletingRouter(null); + await invalidateAutoRouters(); + } catch (error) { + NotificationsManager.fromBackend(`Failed to delete auto router: ${error}`); + } finally { + setIsDeleting(false); + } + }; + + return ( +
+
+
+

Auto routers

+

+ Auto routers sit above your deployments and pick a model per request. They are called like any other model, + so clients keep using a single model name. +

+
+ {canModify && ( + + )} +
+ + openModel(row.id)} + onDeleteClick={setDeletingRouter} + /> + + + {/* The form is long, so the dialog caps its height and scrolls its body rather than + growing past the viewport. */} + + + Add Auto Router + + Routes each request to a model by classifying its complexity. Called like any other model, so clients keep + using a single model name. + + + + + + + {deletingRouter && ( + setDeletingRouter(null)} + onOk={handleConfirmDelete} + confirmLoading={isDeleting} + /> + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx new file mode 100644 index 00000000000..943388f8535 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTable.tsx @@ -0,0 +1,68 @@ +"use client"; + +import { SortingState } from "@tanstack/react-table"; +import { useMemo, useState } from "react"; + +import { DataTable } from "@/components/shared/DataTable"; +import { AutoRouterIcon } from "@/components/shared/table_cells"; + +import { getAutoRoutersTableColumns } from "./AutoRoutersTableColumns"; +import { AutoRouterRow } from "./autoRouterRows"; + +interface AutoRoutersTableProps { + routers: AutoRouterRow[]; + isLoading: boolean; + canModify: boolean; + onRouterClick: (row: AutoRouterRow) => void; + onDeleteClick: (row: AutoRouterRow) => void; +} + +const PAGE_SIZE_OPTIONS = [10, 25, 50]; + +function EmptyState({ canModify }: { canModify: boolean }) { + return ( +
+
+ +
+
No auto routers yet
+
+ {canModify + ? "Create an auto router to pick the right model per request instead of pinning one." + : "An auto router picks the right model per request instead of pinning one."} +
+
+ ); +} + +export function AutoRoutersTable({ + routers, + isLoading, + canModify, + onRouterClick, + onDeleteClick, +}: AutoRoutersTableProps) { + const [sorting, setSorting] = useState([]); + + const columns = useMemo( + () => getAutoRoutersTableColumns({ canModify, onRouterClick, onDeleteClick }), + [canModify, onRouterClick, onDeleteClick], + ); + + return ( + router.id} + sortingMode="client" + sorting={sorting} + onSortingChange={setSorting} + paginationMode="client" + pageSizeOptions={PAGE_SIZE_OPTIONS} + isLoading={isLoading} + loadingMessage="Loading auto routers…" + noDataMessage={} + size="compact" + /> + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx new file mode 100644 index 00000000000..995ba634c34 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersTableColumns.tsx @@ -0,0 +1,173 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { MoreHorizontal, Trash2 } from "lucide-react"; + +import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DateCell, IdentityCell } from "@/components/shared/table_cells"; +import { Badge } from "@/components/ui/badge"; +import { buttonVariants } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/cva.config"; + +import { AutoRouterRow } from "./autoRouterRows"; +import { fitPills } from "./fitPills"; + +function TypeCell({ row }: { row: AutoRouterRow }) { + return ( + + {row.typeLabel} + + ); +} + +function TargetsCell({ targets }: { targets: string[] }) { + const containerRef = useRef(null); + const [width, setWidth] = useState(0); + + useEffect(() => { + const node = containerRef.current; + if (!node || typeof ResizeObserver === "undefined") return; + const observer = new ResizeObserver((entries) => { + const measured = entries[0]?.contentRect.width; + if (typeof measured === "number") setWidth(measured); + }); + observer.observe(node); + return () => observer.disconnect(); + }, []); + + const { visible, overflow } = useMemo(() => fitPills(targets, width), [targets, width]); + + if (targets.length === 0) { + return -; + } + + return ( +
+ {visible.map((target) => ( + + {target} + + ))} + {overflow > 0 && ( + + +{overflow} + + )} +
+ ); +} + +function AutoRouterRowActions({ + row, + onDeleteClick, +}: { + row: AutoRouterRow; + onDeleteClick: (row: AutoRouterRow) => void; +}) { + return ( + + + + + + onDeleteClick(row)} + > + + Delete auto router + + + + ); +} + +interface AutoRoutersTableColumnsDeps { + canModify: boolean; + onRouterClick: (row: AutoRouterRow) => void; + onDeleteClick: (row: AutoRouterRow) => void; +} + +export const getAutoRoutersTableColumns = ({ + canModify, + onRouterClick, + onDeleteClick, +}: AutoRoutersTableColumnsDeps): ColumnDef[] => [ + { + id: "name", + accessorKey: "name", + meta: { title: "Name" }, + header: ({ column }) => , + size: 260, + enableSorting: true, + cell: ({ row }) => onRouterClick(row.original)} />, + }, + { + id: "kind", + accessorKey: "kind", + meta: { title: "Type" }, + header: "Type", + size: 180, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "targets", + meta: { title: "Routes to" }, + header: "Routes to", + size: 320, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "defaultModel", + accessorKey: "defaultModel", + meta: { title: "Default model" }, + header: "Default model", + size: 200, + enableSorting: false, + cell: ({ row }) => + row.original.defaultModel ? ( + + {row.original.defaultModel} + + ) : ( + - + ), + }, + { + id: "createdAt", + accessorKey: "createdAt", + meta: { title: "Created" }, + header: ({ column }) => , + size: 150, + enableSorting: true, + sortingFn: "datetime", + cell: ({ row }) => , + }, + ...(canModify + ? [ + { + id: "actions", + meta: { title: "" }, + header: "", + size: 60, + enableSorting: false, + cell: ({ row }) => + row.original.canDelete ? : null, + } satisfies ColumnDef, + ] + : []), +]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts new file mode 100644 index 00000000000..924391d16fb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it } from "vitest"; + +import { autoRouterStrategy, isComplexityRouter } from "@/components/add_model/auto_router_strategies"; +import { toAutoRouterRow, toAutoRouterRows } from "./autoRouterRows"; + +const complexityDeployment = { + model_name: "tri-tier-router", + litellm_params: { + model: "auto_router/complexity_router", + complexity_router_config: { + tiers: { + SIMPLE: ["gpt-4o-mini"], + MEDIUM: ["anthropic-sonnet-4-6"], + COMPLEX: ["anthropic-opus-4-6", "gpt-4o-mini"], + REASONING: [], + }, + classifier_type: "heuristic", + }, + complexity_router_default_model: "gpt-4o-mini", + }, + model_info: { id: "cid-1", db_model: true, created_at: "2026-07-28T21:40:09.900000+00:00" }, +}; + +const semanticDeployment = { + model_name: "support-router", + litellm_params: { + model: "auto_router/support-router", + auto_router_config: JSON.stringify({ + routes: [ + { name: "gpt-4o-mini", utterances: ["reset my password"] }, + { name: "anthropic-opus-4-6", utterances: ["design a distributed system"] }, + ], + }), + auto_router_default_model: "gpt-4o-mini", + }, + model_info: { id: "sid-1", db_model: true, created_at: "2026-07-27T10:00:00.000000+00:00" }, +}; + +describe("autoRouterRows", () => { + it("classifies a complexity router and unions its tier models as targets", () => { + const row = toAutoRouterRow(complexityDeployment, 0); + + expect(row.kind).toBe("complexity"); + expect(row.typeLabel).toBe("Heuristic"); + // Union across tiers, de-duplicated: gpt-4o-mini appears in both SIMPLE and COMPLEX. + expect(row.targets).toEqual(["gpt-4o-mini", "anthropic-sonnet-4-6", "anthropic-opus-4-6"]); + expect(row.defaultModel).toBe("gpt-4o-mini"); + expect(row.id).toBe("cid-1"); + }); + + it("parses a semantic router whose config arrives as a JSON string", () => { + const row = toAutoRouterRow(semanticDeployment, 0); + + expect(row.kind).toBe("semantic"); + expect(row.typeLabel).toBe("Semantic"); + expect(row.targets).toEqual(["gpt-4o-mini", "anthropic-opus-4-6"]); + expect(row.defaultModel).toBe("gpt-4o-mini"); + }); + + it("shows a tier pinned as a bare string, which the backend accepts as `str | list[str]`", () => { + const row = toAutoRouterRow( + { + ...complexityDeployment, + litellm_params: { + ...complexityDeployment.litellm_params, + complexity_router_config: { + tiers: { SIMPLE: "gpt-4o-mini", MEDIUM: ["anthropic-sonnet-4-6"], COMPLEX: "", REASONING: [] }, + classifier_type: "heuristic", + }, + }, + }, + 0, + ); + + expect(row.targets).toEqual(["gpt-4o-mini", "anthropic-sonnet-4-6"]); + }); + + it("labels a router using the LLM classifier", () => { + const row = toAutoRouterRow( + { + ...complexityDeployment, + litellm_params: { + ...complexityDeployment.litellm_params, + complexity_router_config: { tiers: {}, classifier_type: "llm", adaptive: true }, + }, + }, + 0, + ); + + expect(row.typeLabel).toBe("LLM Classifier"); + }); + + it("treats a deployment carrying complexity_router_config as complexity even off the canonical model string", () => { + expect(isComplexityRouter({ model: "auto_router/legacy", complexity_router_config: { tiers: {} } })).toBe(true); + }); + + it("survives an unparseable config instead of throwing", () => { + const row = toAutoRouterRow( + { + model_name: "broken", + litellm_params: { model: "auto_router/broken", auto_router_config: "{not json" }, + model_info: { id: "bid-1" }, + }, + 0, + ); + + expect(row.kind).toBe("semantic"); + expect(row.targets).toEqual([]); + }); + + it("falls back to a stable synthetic id when the deployment has no model_info id", () => { + const rows = toAutoRouterRows([ + { model_name: "a", litellm_params: { model: "auto_router/a" } }, + { model_name: "b", litellm_params: { model: "auto_router/b" } }, + ]); + + expect(rows.map((row) => row.id)).toEqual(["a-0", "b-1"]); + }); + // Regression: adaptive and quality routers used to fall through to the semantic branch, + // which read the wrong config key and reported an empty route list and a null default. + it("classifies an adaptive router as adaptive, not semantic", () => { + const row = toAutoRouterRow( + { + model_name: "smart-router", + litellm_params: { + model: "auto_router/adaptive_router", + adaptive_router_default_model: "gpt-4o-mini", + adaptive_router_config: { available_models: ["gpt-4o", "gpt-4o-mini"] }, + }, + model_info: { id: "ad-1" }, + }, + 0, + ); + + expect(row.kind).toBe("adaptive"); + expect(row.typeLabel).toBe("Adaptive"); + expect(row.targets).toEqual(["gpt-4o", "gpt-4o-mini"]); + expect(row.defaultModel).toBe("gpt-4o-mini"); + }); + + it("classifies a quality router as quality, not semantic", () => { + const row = toAutoRouterRow( + { + model_name: "quality-router", + litellm_params: { + model: "auto_router/quality_router", + quality_router_default_model: "gpt-4o", + quality_router_config: { available_models: ["gpt-4o"] }, + }, + model_info: { id: "q-1" }, + }, + 0, + ); + + expect(row.kind).toBe("quality"); + expect(row.typeLabel).toBe("Quality"); + expect(row.targets).toEqual(["gpt-4o"]); + }); + + it("mirrors the backend prefix ordering, so a named strategy never reads as semantic", () => { + const kindOf = (model: string) => autoRouterStrategy({ model }).kind; + expect(kindOf("auto_router/complexity_router")).toBe("complexity"); + expect(kindOf("auto_router/adaptive_router")).toBe("adaptive"); + expect(kindOf("auto_router/quality_router")).toBe("quality"); + expect(kindOf("auto_router/my-own-router")).toBe("semantic"); + }); + + // The capability matrix. Origin and strategy constrain DIFFERENT capabilities, and + // collapsing them into one "editable" flag is what stranded DB-created adaptive routers + // with no delete control. Live-verified: for a config row PATCH /model/{id}/update 404s + // and POST /model/delete 400s. + const rowFor = (model: string, dbModel: boolean) => + toAutoRouterRow({ model_name: "r", litellm_params: { model }, model_info: { id: "x", db_model: dbModel } }, 0); + + it.each([ + { model: "auto_router/complexity_router", db: true, canEdit: true, canDelete: true, reason: null }, + { model: "auto_router/my-semantic", db: true, canEdit: true, canDelete: true, reason: null }, + // No editor for its shape, but deleting never reads the config, so delete stays. + { model: "auto_router/adaptive_router", db: true, canEdit: false, canDelete: true, reason: "no-editor" }, + { model: "auto_router/quality_router", db: true, canEdit: false, canDelete: true, reason: "no-editor" }, + // config.yaml rows: the API refuses both, whatever the strategy. + { model: "auto_router/complexity_router", db: false, canEdit: false, canDelete: false, reason: "config-managed" }, + { model: "auto_router/adaptive_router", db: false, canEdit: false, canDelete: false, reason: "config-managed" }, + ])("$model (db_model=$db) -> canEdit=$canEdit canDelete=$canDelete", (spec) => { + const row = rowFor(spec.model, spec.db); + expect(row.canEdit).toBe(spec.canEdit); + expect(row.canDelete).toBe(spec.canDelete); + expect(row.editBlockedReason).toBe(spec.reason); + }); + + it("treats a missing db_model as config-defined rather than assuming it is writable", () => { + const row = toAutoRouterRow({ ...complexityDeployment, model_info: { id: "unknown-1" } }, 0); + expect(row.canEdit).toBe(false); + expect(row.canDelete).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts new file mode 100644 index 00000000000..4817711f469 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts @@ -0,0 +1,103 @@ +import { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; +import { + AutoRouterKind, + EditBlockedReason, + autoRouterCapabilities, + autoRouterStrategy, +} from "@/components/add_model/auto_router_strategies"; +import { normalizeTierModels } from "@/components/add_model/complexity_router_tiers"; + +export type { AutoRouterKind }; + +export interface AutoRouterRow { + id: string; + name: string; + kind: AutoRouterKind; + typeLabel: string; + /** Edit needs an API-created row AND a strategy the dashboard has a form for. */ + canEdit: boolean; + /** Delete only needs an API-created row; removing by id never reads the config. */ + canDelete: boolean; + editBlockedReason: EditBlockedReason | null; + targets: string[]; + defaultModel: string | null; + createdAt: string | null; + deployment: AutoRouterDeployment; +} + +const safeParse = (value: string): unknown => { + try { + return JSON.parse(value); + } catch { + return null; + } +}; + +const asRecord = (value: unknown): Record => { + const parsed: unknown = typeof value === "string" ? safeParse(value) : value; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : {}; +}; + +const asStringArray = (value: unknown): string[] => + Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : []; + +const dedupe = (models: string[]): string[] => Array.from(new Set(models)); + +export const complexityTypeLabel = (config: Record): string => + config.classifier_type === "llm" ? "LLM Classifier" : "Heuristic"; + +interface Presentation { + typeLabel: string; + targets: string[]; +} + +// Adaptive and quality both declare a flat pool and have no editor here, so the row reports +// what is configured rather than interpreting it. +const configManaged = (label: string, config: Record): Presentation => ({ + typeLabel: label, + targets: asStringArray(config.available_models), +}); + +/** How each strategy renders itself, given its own config object. */ +const PRESENTERS: Record) => Presentation> = { + complexity: (config) => ({ + typeLabel: complexityTypeLabel(config), + targets: dedupe(Object.values(asRecord(config.tiers)).flatMap(normalizeTierModels)), + }), + semantic: (config) => { + const routes = dedupe( + (Array.isArray(config.routes) ? config.routes : []) + .map((route) => asRecord(route).name) + .filter((name): name is string => typeof name === "string" && name.length > 0), + ); + return { typeLabel: "Semantic", targets: routes }; + }, + adaptive: (config) => configManaged("Adaptive", config), + quality: (config) => configManaged("Quality", config), +}; + +export const toAutoRouterRow = (deployment: AutoRouterDeployment, index: number): AutoRouterRow => { + const params = deployment.litellm_params ?? {}; + const info = deployment.model_info ?? {}; + const name = deployment.model_name ?? ""; + const strategy = autoRouterStrategy(params); + const { canEdit, canDelete, editBlockedReason } = autoRouterCapabilities(params, info); + + return { + id: info.id ?? `${name}-${index}`, + name, + kind: strategy.kind, + canEdit, + canDelete, + editBlockedReason, + createdAt: info.created_at ?? null, + defaultModel: (params[strategy.defaultModelKey] as string | null | undefined) ?? null, + deployment, + ...PRESENTERS[strategy.kind](asRecord(params[strategy.configKey])), + }; +}; + +export const toAutoRouterRows = (deployments: AutoRouterDeployment[]): AutoRouterRow[] => + deployments.map(toAutoRouterRow); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.test.ts new file mode 100644 index 00000000000..9d702810c35 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; + +import { fitPills, pillWidth } from "./fitPills"; + +const TARGETS = ["anthropic-sonnet-4-6", "gpt-4o-mini", "anthropic-opus-4-6", "voyage-4-large"]; + +describe("fitPills", () => { + it("keeps everything on one row when it all fits", () => { + const wide = TARGETS.reduce((total, label) => total + pillWidth(label) + 4, 0) + 40; + expect(fitPills(TARGETS, wide)).toEqual({ visible: TARGETS, overflow: 0 }); + }); + + it("shows more pills as the column gets wider", () => { + const narrow = fitPills(TARGETS, 200); + const wider = fitPills(TARGETS, 420); + + expect(narrow.visible.length).toBeLessThan(wider.visible.length); + expect(narrow.visible.length + narrow.overflow).toBe(TARGETS.length); + expect(wider.visible.length + wider.overflow).toBe(TARGETS.length); + }); + + it("reserves room for the +N counter so the row never overflows", () => { + const { visible } = fitPills(TARGETS, 220); + const used = visible.reduce((total, label, index) => total + pillWidth(label) + (index === 0 ? 0 : 4), 0); + // 28px counter + its 4px gap must still fit alongside the visible pills. + expect(used + 32).toBeLessThanOrEqual(220); + }); + + it("always shows at least one pill, even when a single name is wider than the column", () => { + expect(fitPills(["an-extremely-long-deployment-name-that-never-fits"], 40)).toEqual({ + visible: ["an-extremely-long-deployment-name-that-never-fits"], + overflow: 0, + }); + }); + + it("shows one pill before the first measurement rather than flashing every pill", () => { + expect(fitPills(TARGETS, 0)).toEqual({ visible: [TARGETS[0]], overflow: 3 }); + }); + + it("handles an empty target list", () => { + expect(fitPills([], 300)).toEqual({ visible: [], overflow: 0 }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.ts new file mode 100644 index 00000000000..fba116af0fd --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/fitPills.ts @@ -0,0 +1,52 @@ +/** + * How many pills fit on ONE row of a given width, leaving room for a "+N" counter. + * + * jsdom reports no layout and the shared ResizeObserver mock only fires inside chart + * subtrees, so this stays a pure width-in / count-out function: the component measures and + * this decides, which keeps the overflow rule unit-testable. + */ + +const CHAR_WIDTH = 6.5; +const PILL_PADDING = 18; +const PILL_GAP = 4; +const OVERFLOW_WIDTH = 28; + +export const pillWidth = (label: string): number => label.length * CHAR_WIDTH + PILL_PADDING; + +export interface FittedPills { + visible: string[]; + overflow: number; +} + +export const fitPills = (labels: string[], availableWidth: number): FittedPills => { + if (labels.length === 0) return { visible: [], overflow: 0 }; + + // Unmeasured (0 or negative) means the first paint before ResizeObserver reports. Show one + // pill rather than all of them, so the row never flashes multi-line and then collapses. + if (availableWidth <= 0) { + return { visible: labels.slice(0, 1), overflow: labels.length - 1 }; + } + + const fitted: string[] = []; + let used = 0; + + for (const [index, label] of labels.entries()) { + const remaining = labels.length - index - 1; + const gap = fitted.length === 0 ? 0 : PILL_GAP; + // Anything still queued after this pill needs room for the "+N" counter beside it. + const reserve = remaining > 0 ? PILL_GAP + OVERFLOW_WIDTH : 0; + + if (used + gap + pillWidth(label) + reserve > availableWidth) break; + + used += gap + pillWidth(label); + fitted.push(label); + } + + // Always show at least one pill; a single over-long name truncates via CSS instead of + // collapsing the cell to a bare "+N". + if (fitted.length === 0) { + return { visible: labels.slice(0, 1), overflow: labels.length - 1 }; + } + + return { visible: fitted, overflow: labels.length - fitted.length }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx index b9dd09d5a71..222a2e04117 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx @@ -7,6 +7,7 @@ import ModelsAndEndpointsPage from "./page"; vi.mock("./panels/AllModelsPanel", () => ({ default: () =>
})); vi.mock("./panels/AddModelPanel", () => ({ default: () =>
})); +vi.mock("./panels/AutoRoutersTabPanel", () => ({ default: () =>
})); vi.mock("./panels/LlmCredentialsPanel", () => ({ default: () =>
})); vi.mock("./panels/PassThroughPanel", () => ({ default: () =>
})); vi.mock("./panels/HealthStatusPanel", () => ({ default: () =>
})); @@ -97,4 +98,34 @@ describe("ModelsAndEndpointsPage", () => { expect(queryByRole("tab", { name: "LLM Credentials" })).toBeNull(); expect(queryByRole("tab", { name: "Health Status" })).toBeNull(); }); + + // Auto-routers are excluded from the All Models table, so this tab is their home: the only + // place in the product to list, create, edit or delete one. + describe("Auto-Routers tab", () => { + it("sits third, after All Models and Add Model", () => { + const { getAllByRole } = renderPage(); + + const tabs = getAllByRole("tab").map((tab) => tab.textContent); + expect(tabs[0]).toContain("All Models"); + expect(tabs[1]).toBe("Add Model"); + expect(tabs[2]).toContain("Auto-Routers"); + // Badged Beta while the tab settles; BetaBadge renders the label text. + expect(tabs[2]).toContain("Beta"); + }); + + it("renders its panel when selected", async () => { + const user = userEvent.setup(); + const { getByRole, getByTestId } = renderPage(); + + await user.click(getByRole("tab", { name: /Auto-Routers/ })); + expect(getByTestId("panel-auto-routers")).toBeInTheDocument(); + }); + + it("is hidden from non-admins, who cannot write models", () => { + mockUseAuthorized.mockReturnValue(NON_ADMIN); + const { queryByRole } = renderPage(); + + expect(queryByRole("tab", { name: /Auto-Routers/ })).toBeNull(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx index cb173367459..b7b16caf2ad 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx @@ -8,12 +8,14 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import { all_admin_roles, internalUserRoles, isProxyAdminRole, isUserTeamAdminForAnyTeam } from "@/utils/roles"; +import BetaBadge from "@/components/BetaBadge"; import CostOptimizationFeedbackBanner from "@/components/molecules/cost_optimization_feedback_banner"; import ModelInfoView from "@/components/model_info_view"; import TeamInfoView from "@/components/team/TeamInfo"; import { useModelDetailRouting } from "@/app/(dashboard)/models-and-endpoints/detailNavigation"; import { useModelDashboardData } from "@/app/(dashboard)/models-and-endpoints/useModelDashboardData"; import AllModelsPanel from "@/app/(dashboard)/models-and-endpoints/panels/AllModelsPanel"; +import AutoRoutersTabPanel from "@/app/(dashboard)/models-and-endpoints/panels/AutoRoutersTabPanel"; import AddModelPanel from "@/app/(dashboard)/models-and-endpoints/panels/AddModelPanel"; import LlmCredentialsPanel from "@/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel"; import PassThroughPanel from "@/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel"; @@ -24,6 +26,7 @@ import PriceDataPanel from "@/app/(dashboard)/models-and-endpoints/panels/PriceD type ModelTabSlug = | "add" + | "auto-routers" | "llm-credentials" | "pass-through" | "health" @@ -35,6 +38,7 @@ const BASE_TAB_KEY = "all-models"; const TAB_LABELS: Record = { add: "Add Model", + "auto-routers": "Auto-Routers", "llm-credentials": "LLM Credentials", "pass-through": "Pass-Through Endpoints", health: "Health Status", @@ -47,6 +51,8 @@ const renderPanel = (key: string) => { switch (key) { case BASE_TAB_KEY: return ; + case "auto-routers": + return ; case "add": return ; case "llm-credentials": @@ -89,6 +95,7 @@ export default function ModelsAndEndpointsPage() { () => [ "", ...(shouldHideAddModelTab ? [] : (["add"] as const)), + ...(isAdmin ? (["auto-routers"] as const) : []), ...(isAdmin ? (["llm-credentials", "pass-through", "health", "retry-settings", "model-group-alias", "price-data"] as const) : []), @@ -97,11 +104,24 @@ export default function ModelsAndEndpointsPage() { ); const allModelsLabel = isAdmin ? "All Models" : "Your Models"; + // Auto-Routers carries a Beta badge; BetaBadge honours the admin setting that hides these. + const tabLabel = (slug: "" | ModelTabSlug): React.ReactNode => { + if (!slug) return allModelsLabel; + if (slug === "auto-routers") { + return ( + + {TAB_LABELS[slug]} + + ); + } + return TAB_LABELS[slug]; + }; + const tabItems = visibleSlugs.map((slug) => { const key = slug || BASE_TAB_KEY; return { key, - label: slug ? TAB_LABELS[slug] : allModelsLabel, + label: tabLabel(slug), children: key === activeKey ? renderPanel(key) : null, }; }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 26dcc60d717..4d7bcc921af 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -3,7 +3,7 @@ import { Form } from "antd"; import { useState } from "react"; import { useQueryClient } from "@tanstack/react-query"; -import AddModelTab from "@/components/add_model/add_model_tab"; +import AddModelForm from "@/components/add_model/AddModelForm"; import { handleAddModelSubmit } from "@/components/add_model/handle_add_model_submit"; import { Providers, getPlaceholder, getProviderModels } from "@/components/provider_info_helpers"; import NotificationsManager from "@/components/molecules/notifications_manager"; @@ -14,7 +14,7 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload"; export default function AddModelPanel() { - const { accessToken, userRole } = useAuthorized(); + const { accessToken } = useAuthorized(); const [form] = Form.useForm(); const queryClient = useQueryClient(); const { data: modelCostMapData } = useModelCostMap(); @@ -39,7 +39,7 @@ export default function AddModelPanel() { }; return ( - ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AutoRoutersTabPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AutoRoutersTabPanel.tsx new file mode 100644 index 00000000000..8b620e22bf9 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AutoRoutersTabPanel.tsx @@ -0,0 +1,23 @@ +"use client"; + +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { isProxyAdminRole } from "@/utils/roles"; + +import { AutoRoutersPanel } from "../components/AutoRouters/AutoRoutersPanel"; + +/** + * Owns the permission decision for the Auto-Routers tab so the panel stays a renderer. + * Creating or editing an auto router is a POST /model/new or PATCH /model/{id}/update, both + * proxy-admin gated, so viewer roles read the list without write affordances. + */ +export default function AutoRoutersTabPanel() { + const { accessToken, userRole } = useAuthorized(); + + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.test.tsx index c4cfa98b2a1..9ffcbfc9975 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.test.tsx @@ -13,7 +13,6 @@ vi.mock("@/components/networking", () => ({ vi.mock("@/components/router_settings", () => ({ default: () => null })); vi.mock("@/components/Settings/RouterSettings/Fallbacks/Fallbacks", () => ({ default: () => null })); vi.mock("@/components/routing_groups", () => ({ default: () => null })); - // Mirrors the /config/list ordering: the two prompt-caching rows sit between the // General-tab rows in the unfiltered response but are filtered out of the General // tab's table, so any index-based lookup into the unfiltered array reads the wrong @@ -99,3 +98,19 @@ describe("GeneralSettings General tab", () => { expect(within(row).getByRole("spinbutton")).toHaveValue("1.00"); }); }); + +// The five tabs here are proxy-wide settings. Auto-routers moved to Models + Endpoints. +describe("GeneralSettings tabs", () => { + beforeEach(() => { + vi.mocked(getGeneralSettingsCall).mockResolvedValue([]); + }); + + it("renders the proxy-wide tabs and no auto-router tab", async () => { + renderWithProviders(); + + for (const name of ["Loadbalancing", "Routing Groups", "Fallbacks", "Prompt Caching", "General"]) { + expect(await screen.findByRole("tab", { name })).toBeInTheDocument(); + } + expect(screen.queryByRole("tab", { name: /auto.?router/i })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index fa3447e0cbf..ed7b17067d5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -13,7 +13,7 @@ import { Icon, Switch, } from "@tremor/react"; -import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "@/components/networking"; import { InputNumber, Select as AntdSelect } from "antd"; import { TrashIcon } from "@heroicons/react/outline"; @@ -232,82 +232,80 @@ const GeneralSettings: React.FC = ({ accessToken, user return (
- - - Loadbalancing - Routing Groups - Fallbacks - Prompt Caching - General - - - - - - - - - - - - - - - - - - - - Setting - Value - Status - Action - - - - {generalSettings - .filter((value) => value.field_type !== "TypedDictionary" && value.field_tab !== PROMPT_CACHING_TAB) - .map((value, index) => ( - - - {value.field_name} -

- {value.field_description} -

- - - - - - {value.stored_in_db == true ? ( - - ) : value.stored_in_db == false ? ( - - ) : ( - - )} - - - - handleResetField(value.field_name)}> - Reset - - - - ))} - -
- - - - + + + Loadbalancing + Routing Groups + Fallbacks + Prompt Caching + General + + + + + + + + + + + + + + + + + + + Setting + Value + Status + Action + + + + {generalSettings + .filter((value) => value.field_type !== "TypedDictionary" && value.field_tab !== PROMPT_CACHING_TAB) + .map((value, index) => ( + + + {value.field_name} +

+ {value.field_description} +

+ + + + + + {value.stored_in_db == true ? ( + + ) : value.stored_in_db == false ? ( + + ) : ( + + )} + + + + handleResetField(value.field_name)}> + Reset + + + + ))} + +
+ + +
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx index b4c22a834a4..040ea2cb440 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx @@ -297,4 +297,17 @@ describe("AddModelForm", () => { expect(screen.queryByRole("switch")).not.toBeInTheDocument(); }); + + it("should display the provider field and the Test Connect / Add Model buttons", async () => { + const mockUseAuthorized = vi.mocked(await import("@/app/(dashboard)/hooks/useAuthorized")); + mockUseAuthorized.default.mockReturnValue(mockAuthorizedUser("proxy_admin", "user-1", true)); + + const props = createTestProps(); + + renderWithProviders(); + + expect(await screen.findByText("Provider")).toBeInTheDocument(); + expect((await screen.findAllByRole("button", { name: "Test Connect" })).length).toBeGreaterThan(0); + expect(await screen.findByRole("button", { name: "Add Model" })).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 4713f8c6869..3328c1703a8 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -1,7 +1,6 @@ import { renderWithProviders, screen } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import { vi } from "vitest"; -import { Form } from "antd"; import AddAutoRouterTab from "./add_auto_router_tab"; import NotificationManager from "../molecules/notifications_manager"; @@ -21,10 +20,7 @@ vi.mock("../molecules/notifications_manager", () => ({ default: { fromBackend: vi.fn() }, })); -const Harness = () => { - const [form] = Form.useForm(); - return ; -}; +const Harness = () => ; describe("AddAutoRouterTab", () => { it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index e7826e09dce..6e946bf9106 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -1,13 +1,10 @@ import React, { useEffect, useState } from "react"; -import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Radio, Badge, Space, Modal } from "antd"; -import type { FormInstance } from "antd"; -import { ThunderboltOutlined, BranchesOutlined } from "@ant-design/icons"; -import { Text, TextInput } from "@tremor/react"; +import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal } from "antd"; +import { TextInput } from "@tremor/react"; import { modelAvailableCall } from "../networking"; import { all_admin_roles } from "@/utils/roles"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; -import RouterConfigBuilder from "./RouterConfigBuilder"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, DEFAULT_ADAPTIVE_WEIGHTS, @@ -22,27 +19,22 @@ import { getSemanticConfigError, } from "./build_complexity_router_config"; import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; -import { getSemanticRouterError } from "./build_semantic_router_validation"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import NotificationManager from "../molecules/notifications_manager"; interface AddAutoRouterTabProps { - form: FormInstance; handleOk: () => void; accessToken: string; userRole: string; } -type RouterType = "recommended" | "semantic"; - const { Title } = Typography; -const AddAutoRouterTab: React.FC = ({ form, handleOk, accessToken, userRole }) => { +const AddAutoRouterTab: React.FC = ({ handleOk, accessToken, userRole }) => { + const [form] = Form.useForm(); const [modelAccessGroups, setModelAccessGroups] = useState([]); const [modelInfo, setModelInfo] = useState([]); - const [routerType, setRouterType] = useState("recommended"); - const [complexityRouterConfig, setComplexityRouterConfig] = useState({ tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic", @@ -56,9 +48,6 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc const [escalationKeywords, setEscalationKeywords] = useState(DEFAULT_ESCALATION_KEYWORDS); const [showValidationErrors, setShowValidationErrors] = useState(false); - // Semantic router config (existing) - const [routerConfig, setRouterConfig] = useState(null); - const [isTestModalVisible, setIsTestModalVisible] = useState(false); const [isTestingConnection, setIsTestingConnection] = useState(false); const [connectionTestId, setConnectionTestId] = useState(0); @@ -169,40 +158,6 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc }); }; - const submitSemanticRouter = (name: string) => { - const validationError = getSemanticRouterError({ - defaultModel: form.getFieldValue("auto_router_default_model"), - embeddingModel: form.getFieldValue("auto_router_embedding_model"), - routerConfig, - }); - if (validationError) { - NotificationManager.fromBackend(validationError); - return; - } - - form.setFieldsValue({ - custom_llm_provider: "auto_router", - model: name, - api_key: "not_required_for_auto_router", - }); - - form - .validateFields() - .then((values) => { - const submitValues = { - ...values, - auto_router_name: name, - auto_router_config: routerConfig, - model_type: "semantic_router", - }; - handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); - }) - .catch((error) => { - console.error("Validation failed:", error); - NotificationManager.fromBackend("Please fill in all required fields"); - }); - }; - const handleAutoRouterSubmit = () => { const name = form.getFieldValue("auto_router_name"); if (!name) { @@ -212,11 +167,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc return; } - if (routerType === "recommended") { - submitRecommendedRouter(name); - } else { - submitSemanticRouter(name); - } + submitRecommendedRouter(name); }; const handleTestConnection = () => { @@ -239,53 +190,6 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc return ( <> - Add Auto Router - - Create an auto router that automatically selects the best model based on request complexity or semantic - matching. Use in place of a single default model. - - - -
- Router Type - { - setRouterType(e.target.value); - setShowValidationErrors(false); - }} - className="w-full" - > - - -
- - Auto-Router v2 - -
-
- Routes by request complexity across four tiers, with optional keyword-to-tier overrides and semantic - keyword matching. No training data needed. -
- - -
- - Semantic Router [to be deprecated] -
-
- Routes based on semantic similarity to example utterances. Requires an embedding model and example - utterances. -
- - - -
- -
= ({ form, handleOk, acc - {routerType === "recommended" ? ( -
- -
- ) : ( - <> -
- { - setRouterConfig(config); - form.setFieldValue("auto_router_config", config); - }} - /> -
- - - - - - - - - - )} +
+ +
@@ -408,7 +265,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc Need Help?
- {routerType === "recommended" && ( + { - )} + }
- {(!isAutoRouter || isComplexityRouter) && ( + {(!isAnyAutoRouter || isComplexityRouterModel) && ( )} - + {!isAnyAutoRouter && ( + <> + - + + + )}
@@ -721,7 +725,7 @@ export default function ModelInfoView({
Model Settings
- {isAutoRouter && canEditModel && !isEditing && ( + {isAutoRouterModel && canEditModel && !isEditing && ( setIsAutoRouterModalOpen(true)} className="flex items-center"> Edit Auto Router @@ -1414,9 +1418,9 @@ export default function ModelInfoView({ { /** * Get all models on proxy @@ -1595,6 +1596,9 @@ export const modelInfoCall = async ( if (sortOrder && sortOrder.trim()) { params.append("sortOrder", sortOrder.trim()); } + if (excludeAutoRouters) { + params.append("exclude_auto_routers", "true"); + } if (params.toString()) { url += `?${params.toString()}`; } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ed975c6be0a..121cd79eccb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -58089,6 +58089,8 @@ export interface operations { sortBy?: string | null; /** @description Sort order. Options: asc, desc */ sortOrder?: string | null; + /** @description Omit auto-router deployments (litellm model prefixed `auto_router/`). They are routing constructs rather than deployments, and are managed on the Router Settings page. Defaults to false, so existing callers are unaffected */ + exclude_auto_routers?: boolean | null; }; header?: never; path?: never; From 692a812f13183b92f9d175abe8dcd5fad33f33ab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:17:39 -0700 Subject: [PATCH 06/33] fix(batches): calculate cost and usage for completed Vertex AI batches --- litellm/batches/batch_utils.py | 3 - .../test_litellm/batches/test_batch_utils.py | 114 +++++++++++++++++- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 2fcb8455e90..36076d59a35 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -212,9 +212,6 @@ async def _get_batch_output_file_content_as_dictionary( _is_base64_encoded_unified_file_id, ) - if custom_llm_provider == "vertex_ai": - raise ValueError("Vertex AI does not support file content retrieval") - if batch.output_file_id is None: raise ValueError("Output file id is None cannot retrieve file content") diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index c7aecac477e..fea897e9372 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -14,6 +14,7 @@ maps (litellm.completion_cost, batch_cost_calculator), the tokenizer deterministic stand-ins so the arithmetic under test is the only variable. """ +import json import os import sys @@ -615,10 +616,117 @@ def _batch(output_file_id): ) +def _vertex_openai_row(custom_id, model, prompt_tokens, completion_tokens): + return { + "id": f"batch_req_{custom_id}", + "custom_id": custom_id, + "response": { + "status_code": 200, + "request_id": custom_id, + "body": { + "id": f"chatcmpl-{custom_id}", + "object": "chat.completion", + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": _usage(prompt_tokens, completion_tokens), + }, + }, + "error": None, + } + + +def _vertex_jsonl(rows): + return "\n".join(json.dumps(row) for row in rows).encode() + + @pytest.mark.asyncio -async def test_output_file_content_vertex_raises(): - with pytest.raises(ValueError, match="Vertex AI does not support"): - await bu._get_batch_output_file_content_as_dictionary(_batch("of"), custom_llm_provider="vertex_ai") +async def test_output_file_content_vertex_fetches_via_afile_content(monkeypatch): + import litellm.files.main as files_main + + rows = [_vertex_openai_row("request-1", "gemini-3.6-flash", 10, 5)] + captured: dict = {} + + async def fake_afile_content(**kw): + captured.update(kw) + return type("R", (), {"content": _vertex_jsonl(rows)})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + + result = await bu._get_batch_output_file_content_as_dictionary( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={ + "vertex_project": "proj-1", + "vertex_location": "us-central1", + "vertex_credentials": "/path/to/creds.json", + "model": "vertex_ai/gemini-3.6-flash", + }, + ) + + assert result == rows + assert captured["file_id"] == "gs://litellm-bucket/output/predictions.jsonl" + assert captured["custom_llm_provider"] == "vertex_ai" + assert captured["vertex_project"] == "proj-1" + assert captured["vertex_location"] == "us-central1" + assert captured["vertex_credentials"] == "/path/to/creds.json" + assert "model" not in captured + + +@pytest.mark.asyncio +async def test_output_file_content_vertex_unified_file_id_extracts_gcs_uri(monkeypatch): + import base64 + + import litellm.files.main as files_main + + captured: dict = {} + + async def fake_afile_content(**kw): + captured.update(kw) + return type("R", (), {"content": b'{"a": 1}'})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + unified_id = ( + "litellm_proxy:application/jsonl;unified_id,uuid-1;target_model_names,vertex-model;" + "llm_output_file_id,gs://litellm-bucket/output/predictions.jsonl;llm_output_file_model_id,model-1" + ) + encoded_id = base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=") + + await bu._get_batch_output_file_content_as_dictionary(_batch(encoded_id), custom_llm_provider="vertex_ai") + + assert captured["file_id"] == "gs://litellm-bucket/output/predictions.jsonl" + assert captured["custom_llm_provider"] == "vertex_ai" + + +@pytest.mark.asyncio +async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monkeypatch): + import litellm.files.main as files_main + + rows = [ + _vertex_openai_row("request-1", "gemini-3.6-flash", 10, 5), + _vertex_openai_row("request-2", "gemini-3.6-flash", 20, 10), + ] + + async def fake_afile_content(**kw): + return type("R", (), {"content": _vertex_jsonl(rows)})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + + cost, usage, models = await bu._handle_completed_batch( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"}, + ) + + assert cost > 0 + assert cost == pytest.approx(30 * 7.5e-07 + 15 * 3.75e-06) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (30, 15, 45) + assert models == ["gemini-3.6-flash", "gemini-3.6-flash"] @pytest.mark.asyncio From 27ccc444715ff9c8e98ddc2d102ff92c4e703962 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:44:31 -0700 Subject: [PATCH 07/33] fix(batches): forward gcs_bucket_name so vertex batch cost logging can read the output file --- litellm/batches/batch_utils.py | 2 ++ tests/test_litellm/batches/test_batch_utils.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 36076d59a35..b62331b904b 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -267,6 +267,8 @@ def _extract_file_access_credentials(litellm_params: Optional[dict]) -> dict: "vertex_project", "vertex_location", "vertex_credentials", + "gcs_bucket_name", + "bucket_name", "timeout", "max_retries", ] diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index fea897e9372..70c89f70ceb 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -236,6 +236,8 @@ def test_extract_credentials_only_known_keys(): "api_key": "sk-1", "api_base": "https://b", "vertex_project": "proj", + "gcs_bucket_name": "my-bucket", + "bucket_name": "my-alias-bucket", "model": "gpt-4o", # not a credential key "unrelated": "x", } @@ -243,6 +245,8 @@ def test_extract_credentials_only_known_keys(): "api_key": "sk-1", "api_base": "https://b", "vertex_project": "proj", + "gcs_bucket_name": "my-bucket", + "bucket_name": "my-alias-bucket", } @@ -262,6 +266,8 @@ def test_extract_credentials_all_supported_keys(): "vertex_project", "vertex_location", "vertex_credentials", + "gcs_bucket_name", + "bucket_name", "timeout", "max_retries", } @@ -665,6 +671,7 @@ async def test_output_file_content_vertex_fetches_via_afile_content(monkeypatch) "vertex_project": "proj-1", "vertex_location": "us-central1", "vertex_credentials": "/path/to/creds.json", + "gcs_bucket_name": "litellm-bucket", "model": "vertex_ai/gemini-3.6-flash", }, ) @@ -675,6 +682,7 @@ async def test_output_file_content_vertex_fetches_via_afile_content(monkeypatch) assert captured["vertex_project"] == "proj-1" assert captured["vertex_location"] == "us-central1" assert captured["vertex_credentials"] == "/path/to/creds.json" + assert captured["gcs_bucket_name"] == "litellm-bucket" assert "model" not in captured From 0b09588685d8bc4b227e1dcdb11d43f4d379adab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:00:01 -0700 Subject: [PATCH 08/33] refactor(batches): aggregate batch output cost, usage, and models in a single pass Completed-batch cost tracking parsed the whole output file into a list of dicts, pretty-printed it into debug strings even with debug logging off, and walked the list three times (cost, usage, models), so a large batch output could pin a worker's memory. The output is now folded line by line into small per-line stats records via _aggregate_batch_cost_usage_models, the eager json.dumps debug calls are gone, and the raw-vertex path computes cost and usage in one call instead of two. _get_batch_output_file_content_as_dictionary becomes _fetch_batch_output_file_content (returns bytes); the superseded three-pass helpers are deleted and their tests migrated --- basedpyright-code-budget.json | 14 +- litellm/batches/batch_utils.py | 294 +++++++----------- ruff-strict-budget.json | 12 +- .../test_batch_custom_pricing.py | 27 +- tests/batches_tests/test_batch_rate_limits.py | 6 +- .../test_batches_logging_unit_tests.py | 22 +- .../test_litellm/batches/test_batch_utils.py | 223 +++++++------ .../test_vertex_ai_batch_passthrough.py | 38 +-- type-discipline-budget.json | 2 +- 9 files changed, 294 insertions(+), 344 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index db3c2502e94..3259f519f66 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 33216 + "limit": 33210 }, "reportArgumentType": { "limit": 2648 @@ -57,7 +57,7 @@ "limit": 5893 }, "reportMissingTypeArgument": { - "limit": 15886 + "limit": 15883 }, "reportMissingTypeStubs": { "limit": 41 @@ -105,19 +105,19 @@ "limit": 113 }, "reportUnknownMemberType": { - "limit": 40525 + "limit": 40523 }, "reportUnknownParameterType": { - "limit": 20384 + "limit": 20381 }, "reportUnknownVariableType": { - "limit": 32099 + "limit": 32095 }, "reportUnnecessaryCast": { "limit": 177 }, "reportUnnecessaryComparison": { - "limit": 1023 + "limit": 1022 }, "reportUnnecessaryContains": { "limit": 7 @@ -135,7 +135,7 @@ "limit": 33 }, "reportUnusedFunction": { - "limit": 206 + "limit": 205 }, "reportUnusedImport": { "limit": 1005 diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index b62331b904b..eef4cf8d87f 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -1,5 +1,6 @@ import json -from typing import Any, Iterator, List, Literal, Optional, Tuple +from dataclasses import dataclass +from typing import Any, Iterable, Iterator, List, Literal, Optional, Tuple import litellm from litellm._logging import verbose_logger @@ -24,20 +25,20 @@ async def calculate_batch_cost_and_usage( deployment-specific pricing (e.g. input_cost_per_token_batches) is used instead of the global cost map. """ - batch_cost = _batch_cost_calculator( + if ( + custom_llm_provider == "vertex_ai" + and model_name + and getattr(litellm, "disable_vertex_batch_output_transformation", False) + ): + batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name) + return batch_cost, batch_usage, [model_name] + + return _aggregate_batch_cost_usage_models( + entries=file_content_dictionary, custom_llm_provider=custom_llm_provider, - file_content_dictionary=file_content_dictionary, model_name=model_name, model_info=model_info, ) - batch_usage = _get_batch_job_total_usage_from_file_content( - file_content_dictionary=file_content_dictionary, - custom_llm_provider=custom_llm_provider, - model_name=model_name, - ) - batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider) - - return batch_cost, batch_usage, batch_models async def _handle_completed_batch( @@ -46,7 +47,9 @@ async def _handle_completed_batch( model_name: Optional[str] = None, litellm_params: Optional[dict] = None, ) -> Tuple[float, Usage, List[str]]: - """Helper function to process a completed batch and handle logging + """Fetch a completed batch's output file and aggregate its cost, usage, and + models in a single pass over the JSONL lines, so the parsed file content is + never materialized in memory. Args: batch: The batch object @@ -54,75 +57,109 @@ async def _handle_completed_batch( model_name: Optional model name litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.) """ - # Get batch results - file_content_dictionary = await _get_batch_output_file_content_as_dictionary( - batch, custom_llm_provider, litellm_params=litellm_params - ) + file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params) - # Calculate costs and usage - batch_cost = _batch_cost_calculator( - custom_llm_provider=custom_llm_provider, - file_content_dictionary=file_content_dictionary, - model_name=model_name, - ) - batch_usage = _get_batch_job_total_usage_from_file_content( - file_content_dictionary=file_content_dictionary, - custom_llm_provider=custom_llm_provider, - model_name=model_name, - ) - - batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider) - - return batch_cost, batch_usage, batch_models - - -def _get_batch_models_from_file_content( - file_content_dictionary: List[dict], - model_name: Optional[str] = None, - custom_llm_provider: str = "openai", -) -> List[str]: - """ - Get the models from the file content - """ - if model_name: - return [model_name] - batch_models = [] - for _item in file_content_dictionary: - if _batch_response_was_successful(_item, custom_llm_provider): - _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider) - _model = _response_body.get("model") - if _model: - batch_models.append(_model) - return batch_models - - -def _batch_cost_calculator( - file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai", - model_name: Optional[str] = None, - model_info: Optional[ModelInfo] = None, -) -> float: - """ - Calculate the cost of a batch based on the output file id - """ if ( custom_llm_provider == "vertex_ai" and model_name and getattr(litellm, "disable_vertex_batch_output_transformation", False) ): - batch_cost, _ = calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name) - verbose_logger.debug("vertex_ai_total_cost=%s", batch_cost) - return batch_cost + batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage( + _get_file_content_as_dictionary(file_content), model_name + ) + return batch_cost, batch_usage, [model_name] - # For other providers, use the existing logic - total_cost = _get_batch_job_cost_from_file_content( - file_content_dictionary=file_content_dictionary, + return _aggregate_batch_cost_usage_models( + entries=_iter_batch_input_entries(file_content), custom_llm_provider=custom_llm_provider, model_name=model_name, - model_info=model_info, ) - verbose_logger.debug("total_cost=%s", total_cost) - return total_cost + + +@dataclass(frozen=True, slots=True) +class _BatchOutputLineStats: + cost: float + prompt_tokens: int + completion_tokens: int + total_tokens: int + cache_read_tokens: int + cache_creation_tokens: int + model: Optional[str] + + +def _iter_successful_output_line_stats( + entries: Iterable[dict], + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"], + model_name: Optional[str], + model_info: Optional[ModelInfo], +) -> Iterator[_BatchOutputLineStats]: + from litellm.cost_calculator import batch_cost_calculator + + for entry in entries: + if not _batch_response_was_successful(entry, custom_llm_provider): + continue + response_body = _get_response_from_batch_job_output_file(entry, custom_llm_provider) + usage = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider) + prompt_details = _parse_prompt_tokens_details(usage) + raw_model = response_body.get("model") + response_model = raw_model if isinstance(raw_model, str) and raw_model else None + if model_info is not None or custom_llm_provider in ("anthropic", "bedrock"): + if custom_llm_provider == "bedrock" and model_name: + cost_model = model_name + else: + cost_model = response_model or model_name or "" + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, + model=cost_model, + custom_llm_provider=custom_llm_provider, + model_info=model_info, + ) + line_cost = prompt_cost + completion_cost + else: + line_cost = litellm.completion_cost( + completion_response=response_body, + custom_llm_provider=custom_llm_provider, + call_type=CallTypes.aretrieve_batch.value, + ) + yield _BatchOutputLineStats( + cost=line_cost, + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + cache_read_tokens=prompt_details["cache_hit_tokens"], + cache_creation_tokens=prompt_details["cache_creation_tokens"], + model=response_model, + ) + + +def _aggregate_batch_cost_usage_models( + entries: Iterable[dict], + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"], + model_name: Optional[str] = None, + model_info: Optional[ModelInfo] = None, +) -> Tuple[float, Usage, List[str]]: + """Aggregate cost, usage, and models from batch output entries in a single + pass, holding one small stats record per line instead of the parsed file.""" + line_stats = tuple(_iter_successful_output_line_stats(entries, custom_llm_provider, model_name, model_info)) + + cache_token_params = { + key: tokens + for key, tokens in ( + ("cache_read_input_tokens", sum(stats.cache_read_tokens for stats in line_stats)), + ("cache_creation_input_tokens", sum(stats.cache_creation_tokens for stats in line_stats)), + ) + if tokens > 0 + } + batch_usage = Usage( + total_tokens=sum(stats.total_tokens for stats in line_stats), + prompt_tokens=sum(stats.prompt_tokens for stats in line_stats), + completion_tokens=sum(stats.completion_tokens for stats in line_stats), + **cache_token_params, + ) + batch_models = [model_name] if model_name else [stats.model for stats in line_stats if stats.model] + total_cost = sum((stats.cost for stats in line_stats), 0.0) + verbose_logger.debug("batch output aggregate: cost=%s usage=%s models=%s", total_cost, batch_usage, batch_models) + return total_cost, batch_usage, batch_models def calculate_vertex_ai_batch_cost_and_usage( @@ -193,13 +230,13 @@ def calculate_vertex_ai_batch_cost_and_usage( ) -async def _get_batch_output_file_content_as_dictionary( +async def _fetch_batch_output_file_content( batch: Batch, custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai", litellm_params: Optional[dict] = None, -) -> List[dict]: +) -> bytes: """ - Get the batch output file content as a list of dictionaries + Fetch the batch output file and return its raw JSONL bytes Args: batch: The batch object @@ -237,7 +274,7 @@ async def _get_batch_output_file_content_as_dictionary( file_content_kwargs.update(credentials) _file_content = await afile_content(**file_content_kwargs) # type: ignore[reportArgumentType] - return _get_file_content_as_dictionary(_file_content.content) + return _file_content.content def _extract_file_access_credentials(litellm_params: Optional[dict]) -> dict: @@ -283,17 +320,7 @@ def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]: """ Get the file content as a list of dictionaries from JSON Lines format """ - try: - _file_content_str = file_content.decode("utf-8") - # Split by newlines and parse each line as a separate JSON object - json_objects = [] - for line in _file_content_str.strip().split("\n"): - if line: # Skip empty lines - json_objects.append(json.loads(line)) - verbose_logger.debug("json_objects=%s", json.dumps(json_objects, indent=4)) - return json_objects - except Exception as e: - raise e + return list(_iter_batch_input_entries(file_content)) def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]: @@ -360,101 +387,6 @@ def _count_entry_tokens( return 0 -def _get_batch_job_cost_from_file_content( - file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai", - model_name: Optional[str] = None, - model_info: Optional[ModelInfo] = None, -) -> float: - """ - Get the cost of a batch job from the file content - """ - from litellm.cost_calculator import batch_cost_calculator - - try: - total_cost: float = 0.0 - # parse the file content as json - verbose_logger.debug("file_content_dictionary=%s", json.dumps(file_content_dictionary, indent=4)) - for _item in file_content_dictionary: - if _batch_response_was_successful(_item, custom_llm_provider): - _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider) - if model_info is not None or custom_llm_provider in ("anthropic", "bedrock"): - usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider) - # Bedrock batch output lines report a short internal model id - # (e.g. "claude-sonnet-4-6") that is not in the cost map; use the - # deployment model name for pricing when available. - if custom_llm_provider == "bedrock" and model_name: - model = model_name - else: - model = _response_body.get("model") or model_name or "" - prompt_cost, completion_cost = batch_cost_calculator( - usage=usage, - model=model, - custom_llm_provider=custom_llm_provider, - model_info=model_info, - ) - total_cost += prompt_cost + completion_cost - else: - total_cost += litellm.completion_cost( - completion_response=_response_body, - custom_llm_provider=custom_llm_provider, - call_type=CallTypes.aretrieve_batch.value, - ) - verbose_logger.debug("total_cost=%s", total_cost) - return total_cost - except Exception as e: - verbose_logger.error("error in _get_batch_job_cost_from_file_content", e) - raise e - - -def _get_batch_job_total_usage_from_file_content( - file_content_dictionary: List[dict], - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai", - model_name: Optional[str] = None, -) -> Usage: - """ - Get the tokens of a batch job from the file content - """ - if ( - custom_llm_provider == "vertex_ai" - and model_name - and getattr(litellm, "disable_vertex_batch_output_transformation", False) - ): - _, batch_usage = calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name) - return batch_usage - - # For other providers, use the existing logic - total_tokens: int = 0 - prompt_tokens: int = 0 - completion_tokens: int = 0 - cache_read_tokens: int = 0 - cache_creation_tokens: int = 0 - for _item in file_content_dictionary: - if _batch_response_was_successful(_item, custom_llm_provider): - _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider) - usage: Usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider) - total_tokens += usage.total_tokens - prompt_tokens += usage.prompt_tokens - completion_tokens += usage.completion_tokens - prompt_details = _parse_prompt_tokens_details(usage) - cache_read_tokens += prompt_details["cache_hit_tokens"] - cache_creation_tokens += prompt_details["cache_creation_tokens"] - cache_token_params = { - key: tokens - for key, tokens in ( - ("cache_read_input_tokens", cache_read_tokens), - ("cache_creation_input_tokens", cache_creation_tokens), - ) - if tokens > 0 - } - return Usage( - total_tokens=total_tokens, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - **cache_token_params, - ) - - def _count_prompt_or_input_tokens(model: str, value: Any) -> int: """Token-count a ``prompt`` / ``input`` field that the OpenAI batch schema allows in four shapes: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index f3b4fce97d3..6323dde9a9f 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -135,7 +135,7 @@ "limit": 30 }, "PERF401": { - "limit": 144 + "limit": 143 }, "PERF402": { "limit": 9 @@ -315,16 +315,16 @@ "limit": 98 }, "TRY201": { - "limit": 424 + "limit": 422 }, "TRY203": { - "limit": 123 + "limit": 122 }, "TRY300": { - "limit": 883 + "limit": 881 }, "UP006": { - "limit": 12147 + "limit": 12143 }, "UP007": { "limit": 2526 @@ -363,6 +363,6 @@ "limit": 105 }, "UP045": { - "limit": 17824 + "limit": 17823 } } diff --git a/tests/batches_tests/test_batch_custom_pricing.py b/tests/batches_tests/test_batch_custom_pricing.py index 3dc1d116e8d..c2159b564a8 100644 --- a/tests/batches_tests/test_batch_custom_pricing.py +++ b/tests/batches_tests/test_batch_custom_pricing.py @@ -12,8 +12,7 @@ import litellm import pytest from litellm.batches.batch_utils import ( - _batch_cost_calculator, - _get_batch_job_cost_from_file_content, + _aggregate_batch_cost_usage_models, calculate_batch_cost_and_usage, ) from litellm.cost_calculator import batch_cost_calculator @@ -113,28 +112,12 @@ def test_batch_cost_calculator_uses_custom_model_info(): ), f"Expected completion cost {expected_completion}, got {completion_cost}" -def test_get_batch_job_cost_from_file_content_uses_custom_model_info(): - """_get_batch_job_cost_from_file_content should thread model_info to completion_cost.""" +def test_aggregate_batch_cost_uses_custom_model_info(): + """_aggregate_batch_cost_usage_models should thread model_info to batch_cost_calculator.""" file_content = [_make_batch_output_line(prompt_tokens=10, completion_tokens=5)] - cost = _get_batch_job_cost_from_file_content( - file_content_dictionary=file_content, - custom_llm_provider="openai", - model_info=CUSTOM_MODEL_INFO, - ) - - expected = (10 * 0.00125) + (5 * 0.005) - assert cost == pytest.approx( - expected - ), f"Expected total cost {expected}, got {cost}" - - -def test_batch_cost_calculator_func_uses_custom_model_info(): - """_batch_cost_calculator should thread model_info.""" - file_content = [_make_batch_output_line(prompt_tokens=10, completion_tokens=5)] - - cost = _batch_cost_calculator( - file_content_dictionary=file_content, + cost, _, _ = _aggregate_batch_cost_usage_models( + entries=file_content, custom_llm_provider="openai", model_info=CUSTOM_MODEL_INFO, ) diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py index e1fe8782ef9..2c804d21ace 100644 --- a/tests/batches_tests/test_batch_rate_limits.py +++ b/tests/batches_tests/test_batch_rate_limits.py @@ -913,7 +913,7 @@ async def test_batch_logging_azure_credentials_regression(): from unittest.mock import AsyncMock, MagicMock, patch from litellm.batches.batch_utils import ( _extract_file_access_credentials, - _get_batch_output_file_content_as_dictionary, + _fetch_batch_output_file_content, _handle_completed_batch, ) from litellm.types.llms.openai import Batch, HttpxBinaryResponseContent @@ -996,7 +996,7 @@ async def test_batch_logging_azure_credentials_regression(): with patch( "litellm.files.main.afile_content", side_effect=mock_afile_content_tracker ): - result = await _get_batch_output_file_content_as_dictionary( + result = await _fetch_batch_output_file_content( batch=mock_batch, custom_llm_provider="azure", litellm_params=azure_credentials, @@ -1092,7 +1092,7 @@ async def test_batch_logging_azure_credentials_regression(): ) # Call without litellm_params (should still work for OpenAI) - result = await _get_batch_output_file_content_as_dictionary( + result = await _fetch_batch_output_file_content( batch=mock_batch, custom_llm_provider="openai", litellm_params=None, diff --git a/tests/batches_tests/test_batches_logging_unit_tests.py b/tests/batches_tests/test_batches_logging_unit_tests.py index 33a0a87dd92..62b6f5b08e4 100644 --- a/tests/batches_tests/test_batches_logging_unit_tests.py +++ b/tests/batches_tests/test_batches_logging_unit_tests.py @@ -19,10 +19,8 @@ import litellm from litellm import create_batch, create_file from litellm._logging import verbose_logger from litellm.batches.batch_utils import ( - _batch_cost_calculator, + _aggregate_batch_cost_usage_models, _get_file_content_as_dictionary, - _get_batch_job_cost_from_file_content, - _get_batch_job_total_usage_from_file_content, _get_batch_job_usage_from_response_body, _get_response_from_batch_job_output_file, _batch_response_was_successful, @@ -139,9 +137,10 @@ def test_get_file_content_as_dictionary(sample_file_content): def test_get_batch_job_total_usage_from_file_content(sample_file_content_dict): - usage = _get_batch_job_total_usage_from_file_content( - sample_file_content_dict, custom_llm_provider="openai" - ) + with patch("litellm.completion_cost", return_value=0.0): + _, usage, _ = _aggregate_batch_cost_usage_models( + entries=sample_file_content_dict, custom_llm_provider="openai" + ) assert usage.total_tokens == 62 # 30 + 32 assert usage.prompt_tokens == 42 # 20 + 22 assert usage.completion_tokens == 20 # 10 + 10 @@ -157,8 +156,8 @@ async def test_batch_cost_calculator(sample_file_content_dict): so we expect the cost to be 0.5 * 2 = 1.0 """ with patch("litellm.completion_cost", return_value=0.5): - cost = _batch_cost_calculator( - file_content_dictionary=sample_file_content_dict, + cost, _, _ = _aggregate_batch_cost_usage_models( + entries=sample_file_content_dict, custom_llm_provider="openai", ) assert cost == 1.0 # 0.5 * 2 successful responses @@ -278,9 +277,12 @@ async def test_handle_completed_batch_computes_real_cost_from_output_file( created_at=1234567890, ) + sample_file_content_bytes = "\n".join( + json.dumps(row) for row in sample_file_content_dict + ).encode() with patch( - "litellm.batches.batch_utils._get_batch_output_file_content_as_dictionary", - new=AsyncMock(return_value=sample_file_content_dict), + "litellm.batches.batch_utils._fetch_batch_output_file_content", + new=AsyncMock(return_value=sample_file_content_bytes), ): cost, usage, models = await _handle_completed_batch( batch=batch, custom_llm_provider="openai" diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index 70c89f70ceb..fb9a9a9d040 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -201,29 +201,34 @@ def test_estimate_tokens_never_zero_for_short_rows(): # =========================================================================== # -# _get_batch_models_from_file_content (output file) +# _aggregate_batch_cost_usage_models: models (output file) # =========================================================================== # -def test_output_models_uses_model_name_override(): - # model_name short-circuits: content is ignored entirely. - assert bu._get_batch_models_from_file_content([_success_row(model="ignored")], model_name="forced-model") == [ - "forced-model" - ] +def test_output_models_uses_model_name_override(monkeypatch): + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0) + _, _, models = bu._aggregate_batch_cost_usage_models( + entries=[_success_row(model="ignored")], custom_llm_provider="openai", model_name="forced-model" + ) + assert models == ["forced-model"] -def test_output_models_collects_from_successful_only(): +def test_output_models_collects_from_successful_only(monkeypatch): + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0) rows = [ _success_row(model="gpt-4o"), _failed_row(model="should-be-skipped"), _success_row(model="claude-3"), ] - assert bu._get_batch_models_from_file_content(rows) == ["gpt-4o", "claude-3"] + _, _, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") + assert models == ["gpt-4o", "claude-3"] -def test_output_models_skips_successful_without_model(): +def test_output_models_skips_successful_without_model(monkeypatch): + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0) rows = [{"response": {"status_code": 200, "body": {}}}] - assert bu._get_batch_models_from_file_content(rows) == [] + _, _, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") + assert models == [] # =========================================================================== # @@ -379,17 +384,18 @@ def test_count_entry_uses_model_name_fallback(monkeypatch): # =========================================================================== # -# _get_batch_job_total_usage_from_file_content (output usage aggregation) +# _aggregate_batch_cost_usage_models: usage (output usage aggregation) # =========================================================================== # -def test_total_usage_sums_successful_only(): +def test_total_usage_sums_successful_only(monkeypatch): + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0) rows = [ _success_row(usage=_usage(10, 5)), # 15 _failed_row(), # excluded _success_row(usage=_usage(20, 10)), # 30 ] - usage = bu._get_batch_job_total_usage_from_file_content(rows) + _, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( 30, 15, @@ -398,7 +404,9 @@ def test_total_usage_sums_successful_only(): def test_total_usage_empty_is_zero(): - usage = bu._get_batch_job_total_usage_from_file_content([]) + cost, usage, models = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai") + assert cost == 0.0 + assert models == [] assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( 0, 0, @@ -407,7 +415,7 @@ def test_total_usage_empty_is_zero(): # =========================================================================== # -# _get_batch_job_cost_from_file_content (cost maps mocked) +# _aggregate_batch_cost_usage_models: cost (cost maps mocked) # =========================================================================== # @@ -426,7 +434,7 @@ def test_cost_from_content_completion_cost_path(monkeypatch): _success_row(usage=_usage(20, 10)), ] - total = bu._get_batch_job_cost_from_file_content(rows, custom_llm_provider="openai") + total, _, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") assert total == 1.0 # 2 successful * 0.5 assert len(calls) == 2 # failed row not costed @@ -442,8 +450,8 @@ def test_cost_from_content_model_info_path(monkeypatch): _success_row(usage=_usage(20, 10)), ] - total = bu._get_batch_job_cost_from_file_content( - rows, + total, _, _ = bu._aggregate_batch_cost_usage_models( + entries=rows, custom_llm_provider="openai", model_info={"input_cost_per_token": 0.0}, # type: ignore[arg-type] # truthy -> model_info path ) @@ -451,32 +459,65 @@ def test_cost_from_content_model_info_path(monkeypatch): assert total == pytest.approx(0.6) # 2 * (0.1 + 0.2) +def test_aggregate_consumes_entries_in_a_single_pass(monkeypatch): + """A one-shot generator: any implementation that iterates the entries twice + (e.g. separate cost and usage passes) sees nothing on the second pass and + returns wrong totals for at least one of cost/usage/models.""" + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.5) + one_shot = (row for row in [_success_row(usage=_usage(10, 5)), _failed_row(), _success_row(usage=_usage(20, 10))]) + + cost, usage, models = bu._aggregate_batch_cost_usage_models(entries=one_shot, custom_llm_provider="openai") + + assert cost == 1.0 + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (30, 15, 45) + assert models == ["gpt-4o", "gpt-4o"] + + # =========================================================================== # -# _batch_cost_calculator (dispatch: vertex-disable-transform vs generic) +# calculate_batch_cost_and_usage (dispatch: vertex-disable-transform vs generic) # =========================================================================== # -def test_batch_cost_calculator_generic_path(monkeypatch): - monkeypatch.setattr(bu, "_get_batch_job_cost_from_file_content", lambda **kw: 4.2) - assert bu._batch_cost_calculator([], custom_llm_provider="openai", model_name="gpt-4o") == 4.2 - - -def test_batch_cost_calculator_vertex_disable_transform_path(monkeypatch): +@pytest.mark.asyncio +async def test_calculate_vertex_disable_transform_path(monkeypatch): monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", True, raising=False) monkeypatch.setattr( bu, "calculate_vertex_ai_batch_cost_and_usage", - lambda content, model: (9.9, Usage()), + lambda content, model: (9.9, Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3)), ) # generic path must NOT be taken monkeypatch.setattr( bu, - "_get_batch_job_cost_from_file_content", + "_aggregate_batch_cost_usage_models", lambda **kw: pytest.fail("generic path should not run"), ) - cost = bu._batch_cost_calculator([], custom_llm_provider="vertex_ai", model_name="gemini-2.0-flash-001") + cost, usage, models = await bu.calculate_batch_cost_and_usage( + file_content_dictionary=[], custom_llm_provider="vertex_ai", model_name="gemini-2.0-flash-001" + ) assert cost == 9.9 + assert usage.total_tokens == 3 + assert models == ["gemini-2.0-flash-001"] + + +@pytest.mark.asyncio +async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch): + """Without a model_name the raw-vertex path cannot price lines; the generic + aggregation path must run even with the disable flag set.""" + monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", True, raising=False) + monkeypatch.setattr( + bu, + "calculate_vertex_ai_batch_cost_and_usage", + lambda content, model: pytest.fail("raw vertex path should not run"), + ) + + cost, usage, models = await bu.calculate_batch_cost_and_usage( + file_content_dictionary=[], custom_llm_provider="vertex_ai" + ) + assert cost == 0.0 + assert usage.total_tokens == 0 + assert models == [] # =========================================================================== # @@ -586,24 +627,19 @@ def test_vertex_cost_error_in_line_is_swallowed(monkeypatch): @pytest.mark.asyncio async def test_calculate_batch_cost_and_usage_orchestration(monkeypatch): rows = [_success_row(model="gpt-4o", usage=_usage(10, 5))] - monkeypatch.setattr(bu, "_batch_cost_calculator", lambda **kw: 2.5) - monkeypatch.setattr( - bu, - "_get_batch_job_total_usage_from_file_content", - lambda **kw: Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 2.5) cost, usage, models = await bu.calculate_batch_cost_and_usage( file_content_dictionary=rows, custom_llm_provider="openai" ) assert cost == 2.5 - assert usage.total_tokens == 15 - assert models == ["gpt-4o"] # real _get_batch_models_from_file_content + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (10, 5, 15) + assert models == ["gpt-4o"] # =========================================================================== # -# _get_batch_output_file_content_as_dictionary (file fetch + credential merge) +# _fetch_batch_output_file_content (file fetch + credential merge) # =========================================================================== # @@ -664,7 +700,7 @@ async def test_output_file_content_vertex_fetches_via_afile_content(monkeypatch) monkeypatch.setattr(files_main, "afile_content", fake_afile_content) - result = await bu._get_batch_output_file_content_as_dictionary( + result = await bu._fetch_batch_output_file_content( _batch("gs://litellm-bucket/output/predictions.jsonl"), custom_llm_provider="vertex_ai", litellm_params={ @@ -676,7 +712,7 @@ async def test_output_file_content_vertex_fetches_via_afile_content(monkeypatch) }, ) - assert result == rows + assert bu._get_file_content_as_dictionary(result) == rows assert captured["file_id"] == "gs://litellm-bucket/output/predictions.jsonl" assert captured["custom_llm_provider"] == "vertex_ai" assert captured["vertex_project"] == "proj-1" @@ -705,7 +741,7 @@ async def test_output_file_content_vertex_unified_file_id_extracts_gcs_uri(monke ) encoded_id = base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=") - await bu._get_batch_output_file_content_as_dictionary(_batch(encoded_id), custom_llm_provider="vertex_ai") + await bu._fetch_batch_output_file_content(_batch(encoded_id), custom_llm_provider="vertex_ai") assert captured["file_id"] == "gs://litellm-bucket/output/predictions.jsonl" assert captured["custom_llm_provider"] == "vertex_ai" @@ -740,7 +776,7 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk @pytest.mark.asyncio async def test_output_file_content_no_output_file_id_raises(): with pytest.raises(ValueError, match="Output file id is None"): - await bu._get_batch_output_file_content_as_dictionary(_batch(None), custom_llm_provider="openai") + await bu._fetch_batch_output_file_content(_batch(None), custom_llm_provider="openai") @pytest.mark.asyncio @@ -757,13 +793,13 @@ async def test_output_file_content_fetches_and_parses(monkeypatch): monkeypatch.setattr(files_main, "afile_content", fake_afile_content) monkeypatch.setattr(cu, "_is_base64_encoded_unified_file_id", lambda fid: False) - result = await bu._get_batch_output_file_content_as_dictionary( + result = await bu._fetch_batch_output_file_content( _batch("file-out"), custom_llm_provider="azure", litellm_params={"api_key": "sk-az", "api_base": "https://az", "model": "x"}, ) - assert result == [{"a": 1}, {"b": 2}] + assert result == b'{"a": 1}\n{"b": 2}' # afile_content received the file id + extracted credentials (not "model"). assert captured["file_id"] == "file-out" assert captured["custom_llm_provider"] == "azure" @@ -792,13 +828,13 @@ async def test_output_file_content_unified_file_id_extraction(monkeypatch): lambda fid: "litellm_proxy;llm_output_file_id,real-file-99;rest", ) - await bu._get_batch_output_file_content_as_dictionary(_batch("encoded-blob"), custom_llm_provider="openai") + await bu._fetch_batch_output_file_content(_batch("encoded-blob"), custom_llm_provider="openai") assert captured["file_id"] == "real-file-99" # =========================================================================== # -# _handle_completed_batch (async orchestrator: fetch -> cost/usage/models) +# _handle_completed_batch (async orchestrator: fetch -> single-pass aggregate) # =========================================================================== # @@ -806,49 +842,48 @@ async def test_output_file_content_unified_file_id_extraction(monkeypatch): async def test_handle_completed_batch_orchestration(monkeypatch): rows = [_success_row(model="gpt-4o", usage=_usage(10, 5))] - async def fake_get_content(batch, custom_llm_provider, litellm_params=None): - return rows + async def fake_fetch(batch, custom_llm_provider, litellm_params=None): + return _vertex_jsonl(rows) - monkeypatch.setattr(bu, "_get_batch_output_file_content_as_dictionary", fake_get_content) - monkeypatch.setattr(bu, "_batch_cost_calculator", lambda **kw: 3.3) - monkeypatch.setattr( - bu, - "_get_batch_job_total_usage_from_file_content", - lambda **kw: Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) + monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch) + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 3.3) cost, usage, models = await bu._handle_completed_batch(_batch("of"), custom_llm_provider="openai") assert cost == 3.3 - assert usage.total_tokens == 15 + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (10, 5, 15) assert models == ["gpt-4o"] -# =========================================================================== # -# Remaining branch: vertex usage disable-transform path. -# -# NOTE: the error path of _get_batch_job_cost_from_file_content (its `raise e`) -# is intentionally NOT tested: the preceding line logs via -# `verbose_logger.error("...", e)`, which passes the exception as a logging -# format-arg with no placeholder and itself raises TypeError under -# logging.raiseExceptions, masking the original error. Asserting that masked -# behavior would lock a source bug; left uncovered on purpose. -# =========================================================================== # +@pytest.mark.asyncio +async def test_handle_completed_batch_vertex_disable_transform_path(monkeypatch): + raw_rows = [{"response": {"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 2}}}] + async def fake_fetch(batch, custom_llm_provider, litellm_params=None): + return _vertex_jsonl(raw_rows) -def test_total_usage_vertex_disable_transform_path(monkeypatch): + monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch) monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", True, raising=False) - monkeypatch.setattr( - bu, - "calculate_vertex_ai_batch_cost_and_usage", - lambda content, model: ( - 0.0, - Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3), - ), + seen: dict = {} + + def fake_vertex_calc(content, model): + seen["content"] = content + seen["model"] = model + return 7.7, Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3) + + monkeypatch.setattr(bu, "calculate_vertex_ai_batch_cost_and_usage", fake_vertex_calc) + + cost, usage, models = await bu._handle_completed_batch( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + model_name="gemini-x", ) - usage = bu._get_batch_job_total_usage_from_file_content([], custom_llm_provider="vertex_ai", model_name="gemini-x") + assert cost == 7.7 assert usage.total_tokens == 3 + assert models == ["gemini-x"] + assert seen["content"] == raw_rows + assert seen["model"] == "gemini-x" def _anthropic_usage(input_tokens, output_tokens, cache_creation=0, cache_read=0): @@ -948,46 +983,54 @@ def test_bedrock_cost_uses_deployment_model_name(): "recordId": "1", "modelOutput": {"model": "claude-sonnet-4-6", "usage": {"input_tokens": 13, "output_tokens": 5}}, } - cost = bu._get_batch_job_cost_from_file_content( - file_content_dictionary=[row], + cost, _, models = bu._aggregate_batch_cost_usage_models( + entries=[row], custom_llm_provider="bedrock", model_name="us.anthropic.claude-sonnet-4-6", model_info={}, ) assert cost > 0 + assert models == ["us.anthropic.claude-sonnet-4-6"] -def test_anthropic_total_usage_sums_succeeded_only(): +def test_anthropic_total_usage_sums_succeeded_only(monkeypatch): + import litellm.cost_calculator as cc + + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.0, 0.0)) rows = [ _anthropic_succeeded_row(usage=_anthropic_usage(10, 5)), _anthropic_errored_row(), _anthropic_succeeded_row(usage=_anthropic_usage(20, 10, cache_read=100)), ] - usage = bu._get_batch_job_total_usage_from_file_content(rows, custom_llm_provider="anthropic") + _, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic") assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (130, 15, 145) -def test_anthropic_total_usage_aggregates_cache_token_details(): +def test_anthropic_total_usage_aggregates_cache_token_details(monkeypatch): + import litellm.cost_calculator as cc + + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.0, 0.0)) rows = [ _anthropic_succeeded_row(usage=_anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000)), _anthropic_errored_row(), _anthropic_succeeded_row(usage=_anthropic_usage(50, 20, cache_creation=300, cache_read=700)), ] - usage = bu._get_batch_job_total_usage_from_file_content(rows, custom_llm_provider="anthropic") + _, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic") assert usage.prompt_tokens_details.cached_tokens == 8700 assert usage.prompt_tokens_details.cache_creation_tokens == 2300 assert usage.cache_read_input_tokens == 8700 assert usage.cache_creation_input_tokens == 2300 -def test_total_usage_without_cache_tokens_has_no_prompt_details(): +def test_total_usage_without_cache_tokens_has_no_prompt_details(monkeypatch): + monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0) rows = [ { "custom_id": "req-1", "response": {"status_code": 200, "body": {"model": "gpt-5.2", "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}}, } ] - usage = bu._get_batch_job_total_usage_from_file_content(rows, custom_llm_provider="openai") + _, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (10, 5, 15) assert usage.prompt_tokens_details is None @@ -1000,8 +1043,8 @@ def test_anthropic_cost_applies_batch_discount_and_cache_pricing(): _anthropic_errored_row(), ] - total = bu._get_batch_job_cost_from_file_content( - rows, + total, _, _ = bu._aggregate_batch_cost_usage_models( + entries=rows, custom_llm_provider="anthropic", model_info=_ANTHROPIC_MODEL_INFO, # type: ignore[arg-type] ) @@ -1026,8 +1069,8 @@ def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatc lambda **kw: pytest.fail("anthropic rows must not go through completion_cost"), ) - total = bu._get_batch_job_cost_from_file_content( - [_anthropic_succeeded_row()], custom_llm_provider="anthropic" + total, _, _ = bu._aggregate_batch_cost_usage_models( + entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic" ) assert total == pytest.approx(0.3) @@ -1036,12 +1079,16 @@ def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatc assert seen[0]["usage"].prompt_tokens == 10 -def test_anthropic_batch_models_collected_from_succeeded_rows(): +def test_anthropic_batch_models_collected_from_succeeded_rows(monkeypatch): + import litellm.cost_calculator as cc + + monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.0, 0.0)) rows = [ _anthropic_succeeded_row(model="claude-sonnet-4-5-20250929"), _anthropic_errored_row(), ] - assert bu._get_batch_models_from_file_content(rows, None, "anthropic") == ["claude-sonnet-4-5-20250929"] + _, _, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic") + assert models == ["claude-sonnet-4-5-20250929"] @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py index 5de682ec8a0..52da7a4a81d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py @@ -590,22 +590,20 @@ class TestVertexAIBatchCostCalculation: assert usage.completion_tokens == 0 assert usage.total_tokens == 0 - def test_openai_shaped_output_records_nonzero_cost_and_usage(self): + @pytest.mark.asyncio + async def test_openai_shaped_output_records_nonzero_cost_and_usage(self): """ Regression test for the bug where Vertex batch cost/usage was always 0. After PR #25627 (transform_file_content_response), the GCS predictions.jsonl is rewritten into OpenAI batch shape before the cost-tracking path sees it. - With disable_vertex_batch_output_transformation=False (default), the content - is OpenAI-shaped, so _batch_cost_calculator must fall through to the generic - path rather than calling calculate_vertex_ai_batch_cost_and_usage (which only - reads raw usageMetadata fields). + With disable_vertex_batch_output_transformation=False (default), the cost + dispatch must fall through to the generic aggregation path rather than + calling calculate_vertex_ai_batch_cost_and_usage (which only reads raw + usageMetadata fields). """ import litellm - from litellm.batches.batch_utils import ( - _batch_cost_calculator, - _get_batch_job_total_usage_from_file_content, - ) + from litellm.batches.batch_utils import calculate_batch_cost_and_usage openai_shaped_responses = [ { @@ -668,12 +666,7 @@ class TestVertexAIBatchCostCalculation: try: litellm.disable_vertex_batch_output_transformation = False - cost = _batch_cost_calculator( - file_content_dictionary=openai_shaped_responses, - custom_llm_provider="vertex_ai", - model_name="gemini-2.0-flash-001", - ) - usage = _get_batch_job_total_usage_from_file_content( + cost, usage, _ = await calculate_batch_cost_and_usage( file_content_dictionary=openai_shaped_responses, custom_llm_provider="vertex_ai", model_name="gemini-2.0-flash-001", @@ -694,16 +687,14 @@ class TestVertexAIBatchCostCalculation: cost > 0 ), f"expected non-zero cost for completed Vertex batch, got {cost}" - def test_raw_vertex_output_still_works_when_transformation_disabled(self): + @pytest.mark.asyncio + async def test_raw_vertex_output_still_works_when_transformation_disabled(self): """ When disable_vertex_batch_output_transformation=True the GCS file is returned as raw Vertex predictions.jsonl; the specialized reader must be used. """ import litellm - from litellm.batches.batch_utils import ( - _batch_cost_calculator, - _get_batch_job_total_usage_from_file_content, - ) + from litellm.batches.batch_utils import calculate_batch_cost_and_usage raw_vertex_responses = [ { @@ -727,12 +718,7 @@ class TestVertexAIBatchCostCalculation: try: litellm.disable_vertex_batch_output_transformation = True - cost = _batch_cost_calculator( - file_content_dictionary=raw_vertex_responses, - custom_llm_provider="vertex_ai", - model_name="gemini-2.0-flash-001", - ) - usage = _get_batch_job_total_usage_from_file_content( + cost, usage, _ = await calculate_batch_cost_and_usage( file_content_dictionary=raw_vertex_responses, custom_llm_provider="vertex_ai", model_name="gemini-2.0-flash-001", diff --git a/type-discipline-budget.json b/type-discipline-budget.json index d56d5a6e305..a7ee2c261c3 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 23287 + "limit": 23279 }, "LIT002": { "limit": 27473 From 8a300929614cce9b31f6caaa748049590b63ffff Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:53:15 -0700 Subject: [PATCH 09/33] test(batches): exercise real GCS validation for vertex batch output reads --- .../test_litellm/batches/test_batch_utils.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index 70c89f70ceb..bf871623a94 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -18,7 +18,9 @@ import json import os import sys +import httpx import pytest +import respx sys.path.insert(0, os.path.abspath("../../../..")) @@ -711,6 +713,97 @@ async def test_output_file_content_vertex_unified_file_id_extracts_gcs_uri(monke assert captured["custom_llm_provider"] == "vertex_ai" +def _vertex_predictions_row(custom_id, prompt_tokens, completion_tokens): + return { + "request": { + "contents": [{"role": "user", "parts": [{"text": "hi"}]}], + "labels": {"litellm_custom_id": custom_id}, + }, + "status": "", + "response": { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "ok"}]}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": prompt_tokens, + "candidatesTokenCount": completion_tokens, + "totalTokenCount": prompt_tokens + completion_tokens, + }, + "modelVersion": "gemini-3.6-flash", + }, + "processed_time": "2026-07-30T00:00:00.000000+00:00", + } + + +@pytest.fixture +def respx_interceptable_httpx_client(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@respx.mock +async def test_output_file_content_vertex_managed_uri_accepted_by_real_validation(respx_interceptable_httpx_client): + managed_output_uri = ( + "gs://litellm-bucket/litellm-vertex-files/publishers/google/models/" + "gemini-3.6-flash/abc-123/prediction-model/predictions.jsonl" + ) + rows = [ + _vertex_predictions_row("request-1", 10, 5), + _vertex_predictions_row("request-2", 20, 10), + ] + route = respx.get(url__regex=r"https://storage\.googleapis\.com/storage/v1/b/litellm-bucket/o/.*").mock( + return_value=httpx.Response(200, content=_vertex_jsonl(rows)) + ) + + result = await bu._get_batch_output_file_content_as_dictionary( + _batch(managed_output_uri), + custom_llm_provider="vertex_ai", + litellm_params={ + "api_key": "test-token", + "vertex_project": "proj-1", + "vertex_location": "us-central1", + "gcs_bucket_name": "litellm-bucket", + }, + ) + + assert route.call_count == 1 + request = route.calls.last.request + assert request.url.raw_path == ( + b"/storage/v1/b/litellm-bucket/o/" + b"litellm-vertex-files%2Fpublishers%2Fgoogle%2Fmodels%2Fgemini-3.6-flash" + b"%2Fabc-123%2Fprediction-model%2Fpredictions.jsonl?alt=media" + ) + assert [row["custom_id"] for row in result] == ["request-1", "request-2"] + assert all(row["response"]["status_code"] == 200 for row in result) + assert all(row["response"]["body"]["model"] == "gemini-3.6-flash" for row in result) + assert [row["response"]["body"]["usage"]["prompt_tokens"] for row in result] == [10, 20] + assert [row["response"]["body"]["usage"]["completion_tokens"] for row in result] == [5, 10] + + +@pytest.mark.asyncio +@respx.mock +async def test_output_file_content_vertex_foreign_bucket_rejected_by_real_validation(): + with pytest.raises(Exception, match="does not match the configured storage bucket"): + await bu._get_batch_output_file_content_as_dictionary( + _batch("gs://attacker-bucket/litellm-vertex-files/x/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={ + "api_key": "test-token", + "vertex_project": "proj-1", + "vertex_location": "us-central1", + "gcs_bucket_name": "litellm-bucket", + }, + ) + + assert respx.mock.calls.call_count == 0 + + @pytest.mark.asyncio async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monkeypatch): import litellm.files.main as files_main From 516953b073abbbeadfa0f6be635cf1e87e65a9a4 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 30 Jul 2026 00:18:45 -0700 Subject: [PATCH 10/33] feat(ui): let team admins create auto-routers; authorize models by team, not created_by The Auto-Routers tab was proxy-admin only, while Add Model on the same page already admits team admins. The asymmetry was not a policy decision; the auto-router create form simply never mounted a team selector, so a team admin's submit was unscoped and POST /model/new rejects an unscoped create from any non-proxy-admin. Mounting the shared TeamDropdown closes it, and the tab now takes the same audience as its sibling. Fixing that surfaced a second, larger problem. The dashboard decided who may edit or delete a deployment with `(userRole === "Admin" || created_by === userID) && db_model`, but `created_by` is written at creation and never read by any backend auth check. The API authorizes on team-admin membership of model_info.team_id, so the dashboard was wrong in both directions: it hid controls from team admins the API accepts, and offered them to former team admins the API rejects. Verified against a live proxy; a model created by the proxy admin was PATCHed and DELETEd 200 by a team admin who did not create it, while the same key got 403 on another team's row and on an unscoped row. Both questions now have one owner in utils/modelPermissions.ts, deliberately shaped as a mirror of ModelManagementAuthChecks. Creation returns a tagged union rather than a pair of booleans, so "may not create" and "may create unscoped" cannot be confused, and the five places that had each invented their own spelling (the models page, the auto-routers tab and panel, the auto-router form, and both branches of AddModelForm) call it instead. Row affordances are now per row rather than per tab, because opening the tab to team admins puts routers they cannot act on in the same list. Note for reviewers: collapsing AddModelForm onto the shared owner changes behaviour for org_admin and Admin Viewer who also admin a team. They previously got the optional team selector, because all_admin_roles counts them as admins, and could submit an unscoped create that the API always 403s; they now get the required selector. Also corrects stale copy left by the auto-router move. The exclude_auto_routers API description named a dashboard page, which went stale inside a single PR; it now describes the concept so it cannot drift with the UI again. The eslint-suppressions prune includes one entry for caching/_components/cache_dashboard.tsx, which this branch does not touch. Its baseline was already stale; the gate measures the whole tree, so it could not be left behind. --- litellm/proxy/proxy_server.py | 5 +- ui/litellm-dashboard/eslint-suppressions.json | 7 +- .../_components/CostOptimizationView.tsx | 2 +- .../app/(dashboard)/hooks/models/useModels.ts | 1 + .../components/AllModelsTab.tsx | 4 +- .../AutoRouters/AutoRoutersPanel.test.tsx | 10 ++- .../AutoRouters/AutoRoutersPanel.tsx | 27 ++++-- .../AutoRouters/autoRouterRows.test.ts | 82 ++++++++++++++++-- .../components/AutoRouters/autoRouterRows.ts | 30 +++++-- .../(dashboard)/models-and-endpoints/page.tsx | 22 +++-- .../panels/AutoRoutersTabPanel.tsx | 27 ++++-- .../src/components/add_model/AddModelForm.tsx | 9 +- .../add_model/add_auto_router_tab.test.tsx | 80 ++++++++++++++++- .../add_model/add_auto_router_tab.tsx | 31 ++++++- .../src/components/model_info_view.tsx | 9 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- .../src/utils/modelPermissions.test.ts | 85 +++++++++++++++++++ .../src/utils/modelPermissions.ts | 80 +++++++++++++++++ 18 files changed, 459 insertions(+), 54 deletions(-) create mode 100644 ui/litellm-dashboard/src/utils/modelPermissions.test.ts create mode 100644 ui/litellm-dashboard/src/utils/modelPermissions.ts diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bf010c7f249..c6d9e28226f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12087,8 +12087,9 @@ async def model_info_v2( False, description=( "Omit auto-router deployments (litellm model prefixed `auto_router/`). " - "They are routing constructs rather than deployments, and are managed on the " - "Router Settings page. Defaults to false, so existing callers are unaffected" + "They select among deployments rather than being deployments themselves, so a " + "caller rendering a deployment list can leave them out. Defaults to false, so " + "existing callers are unaffected" ), ), ): diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index c0d24d3a1e6..6819b2851f5 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -151,14 +151,11 @@ "no-restricted-imports": { "count": 1 }, - "prefer-const": { - "count": 1 - }, "react-hooks/purity": { "count": 1 }, "react-hooks/set-state-in-effect": { - "count": 2 + "count": 1 } }, "src/app/(dashboard)/caching/_components/cache_health.tsx": { @@ -3320,7 +3317,7 @@ "count": 5 }, "no-restricted-syntax": { - "count": 153 + "count": 152 }, "prefer-const": { "count": 32 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index 9c2d0b20b56..f6593e80999 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -45,7 +45,7 @@ const CostOptimizationView: React.FC = ({ accessToken

Track and configure the mechanisms that save you money: prompt compression and prompt caching. Auto routers - live on the Router Settings page + live under Models + Endpoints, on the Auto-Routers tab

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 88c4836f112..52459d69b9a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -104,6 +104,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { created_at?: string | null; updated_at?: string | null; team_id?: string | null; + created_by?: string | null; } | null; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx index 1a65109eac8..1d206f81030 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx @@ -104,8 +104,8 @@ const AllModelsTab = ({ teamIdForQuery, sortBy, sortOrder, - // Auto-routers are routing constructs, not deployments; they are listed and managed on - // the Router Settings page. Excluded server-side so total_count stays honest. + // Auto-routers are routing constructs, not deployments; the sibling Auto-Routers tab + // lists and manages them. Excluded server-side so total_count stays honest. true, ); const isLoading = isLoadingModelsInfo || isLoadingModelCostMap; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx index 7d026f067fb..9ec551bc227 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.test.tsx @@ -108,7 +108,15 @@ const mockDeploymentsPage = () => { }; const renderPanel = (canModify = true) => - renderWithProviders(); + renderWithProviders( + , + ); describe("AutoRoutersPanel", () => { beforeEach(() => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index 3fbce1bd4a0..f27c1e1c44a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -11,6 +11,8 @@ import NotificationsManager from "@/components/molecules/notifications_manager"; import { modelDeleteCall } from "@/components/networking"; import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { type ModelWriteScope } from "@/utils/modelPermissions"; +import { Team } from "@/components/networking"; import { AutoRoutersTable } from "./AutoRoutersTable"; import { AutoRouterRow, toAutoRouterRows } from "./autoRouterRows"; @@ -18,11 +20,14 @@ import { AutoRouterRow, toAutoRouterRows } from "./autoRouterRows"; interface AutoRoutersPanelProps { accessToken: string; userRole: string; - /** Owned by the page, which knows whether the caller may write. */ - canModify: boolean; + userID: string | null; + teams: Team[] | null; + /** Owned by the page, which knows how this caller must scope what they create. */ + createScope: ModelWriteScope; } -export function AutoRoutersPanel({ accessToken, userRole, canModify }: AutoRoutersPanelProps) { +export function AutoRoutersPanel({ accessToken, userRole, userID, teams, createScope }: AutoRoutersPanelProps) { + const canCreate = createScope !== "forbidden"; const { data: deployments, isLoading } = useAutoRouters(); const invalidateAutoRouters = useInvalidateAutoRouters(); // Clicking a router opens the same ?model= drill-in the All Models table uses, so an auto @@ -33,7 +38,10 @@ export function AutoRoutersPanel({ accessToken, userRole, canModify }: AutoRoute const [deletingRouter, setDeletingRouter] = useState(null); const [isDeleting, setIsDeleting] = useState(false); - const routers = useMemo(() => toAutoRouterRows(deployments ?? []), [deployments]); + const routers = useMemo( + () => toAutoRouterRows(deployments ?? [], { userRole, userID }, teams), + [deployments, userRole, userID, teams], + ); const handleCreated = () => { setIsCreating(false); @@ -65,7 +73,7 @@ export function AutoRoutersPanel({ accessToken, userRole, canModify }: AutoRoute so clients keep using a single model name.

- {canModify && ( + {canCreate && (