From 993766be0e1a1ea0f173dc9445f85e4eab3665bc Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 18:03:22 -0700 Subject: [PATCH] feat(proxy): serve the auto-router preset catalog at runtime (#39412) The dashboard's template picker imported autorouter_presets.json at build time, so every catalog change needed a dashboard rebuild and artifacts refresh. The catalog now lives in litellm/proxy/public_endpoints/ and GET /public/autorouter_presets serves it, fetching litellm.autorouter_presets_url (GitHub raw on main, 1h in-process cache, bundled fallback) so a merged catalog change propagates to running proxies like the model cost map does. The dashboard fetches it at runtime via useAutoRouterPresets and keeps no local copy. Resolves LIT-6764 --- litellm/__init__.py | 4 + .../public_endpoints}/autorouter_presets.json | 0 .../public_endpoints/public_endpoints.py | 85 ++++++- .../public_endpoints/public_endpoints.py | 42 ++- .../public_endpoints/test_public_endpoints.py | 240 ++++++++++++++++++ .../hooks/autoRouter/useAutoRouterPresets.ts | 16 ++ .../add_model/add_auto_router_tab.test.tsx | 59 ++++- .../add_model/add_auto_router_tab.tsx | 28 +- .../src/components/networking.tsx | 10 + .../src/lib/autorouter_presets.test.ts | 13 +- .../src/lib/autorouter_presets.ts | 13 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 90 +++++++ .../tests/mocks/autoRouterPresets.ts | 16 ++ 13 files changed, 592 insertions(+), 24 deletions(-) rename {ui/litellm-dashboard/src => litellm/proxy/public_endpoints}/autorouter_presets.json (100%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts create mode 100644 ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts diff --git a/litellm/__init__.py b/litellm/__init__.py index 44f2e7c1f02..41a3789ab0d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -424,6 +424,10 @@ anthropic_beta_headers_url: str = os.getenv( "LITELLM_ANTHROPIC_BETA_HEADERS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json", ) +autorouter_presets_url: str = os.getenv( + "LITELLM_AUTOROUTER_PRESETS_URL", + "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json", +) suppress_debug_info: bool = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json similarity index 100% rename from ui/litellm-dashboard/src/autorouter_presets.json rename to litellm/proxy/public_endpoints/autorouter_presets.json diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 4d58a974bb8..94a59828451 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,11 +1,13 @@ +import asyncio import json import os import re -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from importlib.resources import files from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, HTTPException, Request +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm @@ -28,6 +30,7 @@ from litellm.types.proxy.management_endpoints.model_management_endpoints import ) from litellm.types.proxy.public_endpoints.public_endpoints import ( AgentCreateInfo, + AutoRouterPresetRecord, ComplexityScorerDefaults, ProviderCreateInfo, PublicModelHubInfo, @@ -464,6 +467,86 @@ async def get_litellm_blog_posts(): return BlogPostsResponse(posts=posts) +_AUTOROUTER_PRESETS_ADAPTER: Final = TypeAdapter(dict[str, AutoRouterPresetRecord]) + + +def _load_bundled_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: + raw: Final = json.loads( + files("litellm.proxy.public_endpoints").joinpath("autorouter_presets.json").read_text(encoding="utf-8") + ) + return _AUTOROUTER_PRESETS_ADAPTER.validate_python(raw) + + +async def _fetch_remote_autorouter_presets(url: str) -> Mapping[str, AutoRouterPresetRecord]: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.UI) + response: Final = await client.get(url, timeout=5.0) + response.raise_for_status() + presets: Final = _AUTOROUTER_PRESETS_ADAPTER.validate_python(response.json()) + if not presets: + raise ValueError("remote auto-router preset catalog is empty") + return presets + + +async def _resolve_autorouter_presets( + url: str, + fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]], +) -> Mapping[str, AutoRouterPresetRecord]: + if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": + return _load_bundled_autorouter_presets() + try: + return await fetch(url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: failed to fetch auto-router presets from %s: %s. Serving the bundled catalog for the life of this process.", + url, + str(e), + ) + return _load_bundled_autorouter_presets() + + +class _AutoRouterPresetsCache: + presets: Mapping[str, AutoRouterPresetRecord] | None = None + lock: asyncio.Lock | None = None + + +async def get_autorouter_presets( + url: str, + fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]] = _fetch_remote_autorouter_presets, +) -> Mapping[str, AutoRouterPresetRecord]: + cached: Final = _AutoRouterPresetsCache.presets + if cached is not None: + return cached + if _AutoRouterPresetsCache.lock is None: + _AutoRouterPresetsCache.lock = asyncio.Lock() + async with _AutoRouterPresetsCache.lock: + held: Final = _AutoRouterPresetsCache.presets + if held is not None: + return held + resolved: Final = await _resolve_autorouter_presets(url=url, fetch=fetch) + _AutoRouterPresetsCache.presets = resolved + return resolved + + +@router.get( + "/public/autorouter_presets", + tags=["public", "auto router"], # mutable-ok: FastAPI route tags take a list + response_model=dict[str, AutoRouterPresetRecord], +) +async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: + """ + Return the auto-router preset catalog the dashboard's template picker renders. + + Resolved once per process, like the model cost map: fetched from ``litellm.autorouter_presets_url`` + (override with ``LITELLM_AUTOROUTER_PRESETS_URL``) on the first request, falling back to the + catalog bundled with the package on any failure. Set ``LITELLM_LOCAL_AUTOROUTER_PRESETS=True`` + to serve the bundled catalog only. A restart picks up a newly published catalog. + """ + return await get_autorouter_presets(url=litellm.autorouter_presets_url) + + @router.get( "/public/endpoints", tags=["public"], diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index c7f80a61e0f..fa73926305b 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -1,7 +1,7 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class PublicModelHubInfo(BaseModel): @@ -73,6 +73,44 @@ class SupportedEndpointsResponse(BaseModel): endpoints: list[SupportedEndpoint] +class AutoRouterPresetTiers(BaseModel): + """Exactly the four built-in tiers the dashboard's preset prefill can apply. + + extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the + picker, so such a catalog is rejected wholesale and the bundled one serves instead. + """ + + model_config = ConfigDict(extra="forbid") + + SIMPLE: Sequence[str] + MEDIUM: Sequence[str] + COMPLEX: Sequence[str] + REASONING: Sequence[str] + + +class AutoRouterPresetConfig(BaseModel): + """The complexity_router_config a preset prefills. + + Only tiers is validated, because every dashboard consumer dereferences it; everything else + passes through verbatim with unknown fields kept (extra="allow"), so a catalog published after + this proxy shipped still serves its new fields intact. + """ + + model_config = ConfigDict(extra="allow") + + tiers: AutoRouterPresetTiers + + +class AutoRouterPresetRecord(BaseModel): + """One auto-router preset as served to the dashboard's template picker.""" + + model_config = ConfigDict(extra="allow") + + label: str + description: str + complexity_router_config: AutoRouterPresetConfig + + class ComplexityScorerDefaults(BaseModel): """The complexity router's shipped heuristic scorer defaults. diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 8006f64ba41..551e27a18f5 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1077,3 +1077,243 @@ def test_public_mcp_hub_does_not_expose_upstream_url(): assert all("url" not in item for item in data) assert secret_url not in response.text app.dependency_overrides.clear() + + + +@pytest.fixture +def reset_autorouter_presets_cache(): + from litellm.proxy.public_endpoints.public_endpoints import _AutoRouterPresetsCache + + _AutoRouterPresetsCache.presets = None + _AutoRouterPresetsCache.lock = None + yield + _AutoRouterPresetsCache.presets = None + _AutoRouterPresetsCache.lock = None + + +def test_get_autorouter_presets_local_mode_serves_bundled_catalog( + monkeypatch, reset_autorouter_presets_cache +): + monkeypatch.setenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "True") + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/autorouter_presets") + + assert response.status_code == 200 + payload = response.json() + assert "anthropic_family" in payload + for preset in payload.values(): + assert isinstance(preset["label"], str) + assert isinstance(preset["description"], str) + assert "tiers" in preset["complexity_router_config"] + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_fetches_once_per_process( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + remote = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + ) + calls = [] + + async def fake_fetch(url): + calls.append(url) + return remote + + first = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + second = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + + assert first == remote + assert second == remote + assert calls == ["https://example.test/presets.json"] + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start( + monkeypatch, reset_autorouter_presets_cache +): + import asyncio + + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + remote = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + ) + calls = [] + + async def slow_fetch(url): + calls.append(url) + await asyncio.sleep(0.05) + return remote + + results = await asyncio.gather( + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + ) + + assert all(result == remote for result in results) + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_caches_bundled_fallback_on_remote_failure( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import get_autorouter_presets + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + calls = [] + + async def broken_fetch(url): + calls.append(url) + raise ValueError("remote catalog unavailable") + + first = await get_autorouter_presets(url="https://example.test/presets.json", fetch=broken_fetch) + second = await get_autorouter_presets(url="https://example.test/presets.json", fetch=broken_fetch) + + assert "anthropic_family" in first + assert second == first + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_autorouter_presets_adapter_rejects_wrong_shapes(): + from pydantic import ValidationError + + from litellm.proxy.public_endpoints.public_endpoints import _AUTOROUTER_PRESETS_ADAPTER + + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python({"bad": {"label": "no description or config"}}) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python(["not", "a", "mapping"]) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + {"no_tiers": {"label": "L", "description": "D", "complexity_router_config": {}}} + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "missing_builtin_tier": { + "label": "L", + "description": "D", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"]}}, + } + } + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "unknown_tier_name": { + "label": "L", + "description": "D", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["m1"], + "MEDIUM": ["m2"], + "COMPLEX": ["m3"], + "REASONING": ["m4"], + "ULTRA": ["m5"], + } + }, + } + } + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "bad_tiers": { + "label": "L", + "description": "D", + "complexity_router_config": {"tiers": "not-a-mapping"}, + } + } + ) + + +def test_get_autorouter_presets_passes_unknown_catalog_fields_through( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + _AutoRouterPresetsCache, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + _AutoRouterPresetsCache.presets = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "future_preset": { + "label": "Future", + "description": "carries fields this proxy version does not know", + "complexity_router_config": { + "tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}, + "future_config_knob": 3, + }, + "icon": "sparkles", + } + } + ) + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/autorouter_presets") + + assert response.status_code == 200 + served = response.json()["future_preset"] + assert served["icon"] == "sparkles" + assert served["complexity_router_config"]["future_config_knob"] == 3 + assert served["complexity_router_config"]["tiers"]["SIMPLE"] == ["m1"] + + +@pytest.mark.asyncio +async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypatch): + import litellm.llms.custom_httpx.http_handler as http_handler_module + from litellm.proxy.public_endpoints.public_endpoints import _fetch_remote_autorouter_presets + + catalog = { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + response = MagicMock() + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=catalog) + client = MagicMock() + client.get = AsyncMock(return_value=response) + monkeypatch.setattr(http_handler_module, "get_async_httpx_client", lambda llm_provider: client) + + presets = await _fetch_remote_autorouter_presets("https://example.test/presets.json") + assert presets["remote_only"].label == "Remote Only" + response.raise_for_status.assert_called_once() + + response.json = MagicMock(return_value={}) + with pytest.raises(ValueError, match="empty"): + await _fetch_remote_autorouter_presets("https://example.test/presets.json") diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts new file mode 100644 index 00000000000..82f9cc2da99 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts @@ -0,0 +1,16 @@ +import { AutoRouterPreset, hydratePresets } from "@/lib/autorouter_presets"; +import { getAutoRouterPresets } from "@/components/networking"; +import { useQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +const presetKeys = createQueryKeys("autoRouterPresets"); + +export const useAutoRouterPresets = () => { + const options = { + queryKey: presetKeys.list({}), + queryFn: async () => hydratePresets(await getAutoRouterPresets()), + staleTime: 24 * 60 * 60 * 1000, + gcTime: 24 * 60 * 60 * 1000, + }; + return useQuery(options); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index ba380662403..7769042a0c7 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -9,11 +9,19 @@ import { getSubmitBlockedReason } from "./add_auto_router_tab"; import { buildModelAvailability } from "@/lib/autorouter_presets"; import { testAutoRouterRouting } from "../networking"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { getAllPresets, getPresetByKey, getRequiredModelsInPreset } from "@/lib/autorouter_presets"; +import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets"; +import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets"; vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), ); +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets", + async () => await import("../../../tests/mocks/autoRouterPresets"), +); + +const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS; +const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key); const ANTHROPIC_PRESET = getPresetByKey("anthropic_family")!; const ANTHROPIC_TIERS = ANTHROPIC_PRESET.complexity_router_config.tiers; @@ -1142,3 +1150,52 @@ describe("getSubmitBlockedReason", () => { ); }); }); + +describe("preset catalog fetch states", () => { + afterEach(() => vi.mocked(useAutoRouterPresets).mockReturnValue(LOADED_PRESETS_QUERY)); + + it("keeps showing cached presets without the error banner when only a refetch fails", () => { + vi.mocked(useAutoRouterPresets).mockReturnValue({ + ...LOADED_PRESETS_QUERY, + isError: true, + } as never); + renderWithProviders(); + + expect(screen.queryByText(/Could not load templates/)).not.toBeInTheDocument(); + + openTemplateDropdown(); + expect(screen.queryAllByRole("option").length).toBeGreaterThan(1); + }); + + it("shows a loading hint while the catalog fetch is pending", () => { + vi.mocked(useAutoRouterPresets).mockReturnValue({ + ...LOADED_PRESETS_QUERY, + data: undefined, + isPending: true, + } as never); + renderWithProviders(); + + expect(screen.getByText("Loading templates...")).toBeInTheDocument(); + }); + + it("degrades to Custom Configuration with a retry hint that refetches the catalog", async () => { + const refetch = vi.fn(); + vi.mocked(useAutoRouterPresets).mockReturnValue({ + ...LOADED_PRESETS_QUERY, + data: undefined, + isError: true, + refetch, + } as never); + renderWithProviders(); + + expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument(); + + openTemplateDropdown(); + const options = screen.queryAllByRole("option"); + expect(options).toHaveLength(1); + expect(options[0]).toHaveTextContent("Custom Configuration"); + + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(refetch).toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 81acb7b7e50..ab5ea4cbfc9 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -50,8 +50,6 @@ import AutoRouterConnectionTest from "./auto_router_connection_test"; import AutoRouterRoutingTest from "./AutoRouterRoutingTest"; import { toast } from "@/lib/toast"; import { - getAllPresets, - getPresetByKey, getMissingModelsInPreset, getReferencedModelsError, buildEmptyPrefill, @@ -62,6 +60,7 @@ import { PresetPrefill, AutoRouterPreset, } from "@/lib/autorouter_presets"; +import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; interface AddAutoRouterTabProps { @@ -102,9 +101,7 @@ const presetDisabledHint = (availability: PresetAvailability): string | null => // caller-specific missing-model reason gets the alarming red treatment. const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; -// getAllPresets() already returns a stable, module-level array (see autorouter_presets.ts), so -// this is resolved once at import time rather than re-called from inside the component every render. -const presets = getAllPresets(); +const NO_PRESETS: AutoRouterPreset[] = []; // A one-line summary of what's configured, shown when the detailed section is collapsed so a // caller can see the shape of the config without opening it. @@ -229,6 +226,14 @@ const AddAutoRouterTab: React.FC = ({ }); const modelsLoading = groupsLoading || deploymentsLoading; const modelInfo = React.useMemo(() => data ?? [], [data]); + const { + data: presetsData, + isPending: presetsPending, + isError: presetsError, + refetch: refetchPresets, + } = useAutoRouterPresets(); + const presets = presetsData ?? NO_PRESETS; + const presetsUnavailable = presetsError && presetsData === undefined; // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the // former leaves us with nothing trustworthy to verify a preset's models against. @@ -277,7 +282,7 @@ const AddAutoRouterTab: React.FC = ({ presets .map((preset) => ({ preset, availability: presetAvailability(preset) })) .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), - [presetAvailability], + [presets, presetAvailability], ); const templateItems = React.useMemo( @@ -307,7 +312,7 @@ const AddAutoRouterTab: React.FC = ({ return; } - const preset = getPresetByKey(presetKey); + const preset = presets.find((p) => p.key === presetKey); // Refuse to apply a preset whose models are not verified available. The dropdown disables // these options, so this is a guard against a stale click resolving after the list changed. if (!preset) return; @@ -538,6 +543,15 @@ const AddAutoRouterTab: React.FC = ({ )} + {presetsPending &&
Loading templates...
} + {presetsUnavailable && ( +
+ Could not load templates, so only Custom Configuration is shown.{" "} + +
+ )} {requiresTeamScope && ( diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 63232afc46b..eaffa2b4802 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -90,6 +90,7 @@ import type { } from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types"; import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity_router_config"; +import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab"; import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard"; import { @@ -410,6 +411,15 @@ export const getComplexityScorerDefaults = async (): Promise => { + /** + * Fetch the auto-router preset catalog from the proxy's public endpoint. The template picker + * renders from this rather than from a copy in the dashboard, so a catalog update propagates + * without a dashboard release. + */ + return await apiClient.get(`/public/autorouter_presets`); +}; + export const getAgentCreateMetadata = async (): Promise => { /** * Fetch agent type metadata from the proxy's public endpoint. diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index b5802ffa2dc..e3132067f4c 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect } from "vitest"; +import bundledPresets from "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"; import { - getAllPresets, - getPresetByKey, + hydratePresets, + AutoRouterPreset, + AutoRouterPresetsResponse, getRequiredModelsInPreset, getMissingModelsInPreset, getRequiredModels, @@ -18,8 +20,13 @@ import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKe const groupsOnly = (models: Iterable) => buildModelAvailability(models, []); +// Hydrated from the real bundled catalog so a catalog edit flows into these expectations. +const PRESETS = hydratePresets(bundledPresets as AutoRouterPresetsResponse); +const getAllPresets = (): AutoRouterPreset[] => PRESETS; +const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key); + describe("autorouter_presets", () => { - it("loads exactly the bundled presets", () => { + it("hydrates exactly the bundled presets", () => { const presets = getAllPresets(); expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family"]); // Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here. diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 721bd6f2b2a..30f8ed99e74 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -20,7 +20,6 @@ import { } from "@/components/add_model/complexity_router_tiers"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; -import presetsRaw from "@/autorouter_presets.json"; // `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and // never an identity. @@ -31,16 +30,10 @@ export interface AutoRouterPreset { complexity_router_config: ComplexityRouterConfigPayload; } -// The bundled JSON is a developer-authored, build-time asset, so it is trusted at the import -// boundary rather than re-validated at runtime (resolveJsonModule widens its string literals, -// hence this one cast). autorouter_presets.test.ts pins the parsed shape, so a JSON typo fails CI. -const RAW = presetsRaw as Record>; +export type AutoRouterPresetsResponse = Record>; -const PRESETS: AutoRouterPreset[] = Object.entries(RAW).map(([key, preset]) => ({ key, ...preset })); - -export const getAllPresets = (): AutoRouterPreset[] => PRESETS; - -export const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key); +export const hydratePresets = (raw: AutoRouterPresetsResponse): AutoRouterPreset[] => + Object.entries(raw).map(([key, preset]) => ({ key, ...preset })); // Generalized over ComplexityRouterConfigPayload so the same accessors check either a preset's own // bundled config or a caller's actually-built config - the two need to agree, since a preset only diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7583b6f6df1..90c4f03bf08 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -12129,6 +12129,31 @@ export interface paths { patch?: never; trace?: never; }; + "/public/autorouter_presets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Public Autorouter Presets + * @description Return the auto-router preset catalog the dashboard's template picker renders. + * + * Resolved once per process, like the model cost map: fetched from ``litellm.autorouter_presets_url`` + * (override with ``LITELLM_AUTOROUTER_PRESETS_URL``) on the first request, falling back to the + * catalog bundled with the package on any failure. Set ``LITELLM_LOCAL_AUTOROUTER_PRESETS=True`` + * to serve the bundled catalog only. A restart picks up a newly published catalog. + */ + get: operations["get_public_autorouter_presets_public_autorouter_presets_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/public/complexity_router/scorer_defaults": { parameters: { query?: never; @@ -23391,6 +23416,49 @@ export interface components { /** Tier Definitions */ tier_definitions: components["schemas"]["TierDefinition"][]; }; + /** + * AutoRouterPresetConfig + * @description The complexity_router_config a preset prefills. + * + * Only tiers is validated, because every dashboard consumer dereferences it; everything else + * passes through verbatim with unknown fields kept (extra="allow"), so a catalog published after + * this proxy shipped still serves its new fields intact. + */ + AutoRouterPresetConfig: { + tiers: components["schemas"]["AutoRouterPresetTiers"]; + } & { + [key: string]: unknown; + }; + /** + * AutoRouterPresetRecord + * @description One auto-router preset as served to the dashboard's template picker. + */ + AutoRouterPresetRecord: { + complexity_router_config: components["schemas"]["AutoRouterPresetConfig"]; + /** Description */ + description: string; + /** Label */ + label: string; + } & { + [key: string]: unknown; + }; + /** + * AutoRouterPresetTiers + * @description Exactly the four built-in tiers the dashboard's preset prefill can apply. + * + * extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the + * picker, so such a catalog is rejected wholesale and the bundled one serves instead. + */ + AutoRouterPresetTiers: { + /** Complex */ + COMPLEX: string[]; + /** Medium */ + MEDIUM: string[]; + /** Reasoning */ + REASONING: string[]; + /** Simple */ + SIMPLE: string[]; + }; /** * AutoRouterRoutingTestRequest * @description A single request to classify against a complexity-router config that need not be saved yet. @@ -54679,6 +54747,28 @@ export interface operations { }; }; }; + get_public_autorouter_presets_public_autorouter_presets_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: components["schemas"]["AutoRouterPresetRecord"]; + }; + }; + }; + }; + }; get_complexity_scorer_defaults_public_complexity_router_scorer_defaults_get: { parameters: { query?: never; diff --git a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts new file mode 100644 index 00000000000..f73faaa70fe --- /dev/null +++ b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts @@ -0,0 +1,16 @@ +import { vi } from "vitest"; +import bundledPresets from "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"; +import { hydratePresets, type AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; + +// Derived from the real bundled catalog so a preset edit there flows into test expectations +// instead of redding on a stale copy. Exported as vi.fn so a test can override the query state. +export const BUNDLED_PRESETS = hydratePresets(bundledPresets as AutoRouterPresetsResponse); + +export const LOADED_PRESETS_QUERY = { + data: BUNDLED_PRESETS, + isPending: false, + isError: false, + refetch: vi.fn(), +}; + +export const useAutoRouterPresets = vi.fn(() => LOADED_PRESETS_QUERY);