diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 12b4d4b2412..2974813b9ee 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -952,6 +952,7 @@ class LiteLLMRoutes(enum.Enum): "/auto_router/availability", # Per-session auto-router read - the endpoint scopes the row to the caller's own key hash "/auto_router/session", + "/auto_router/usage", "/cost/predict-cache", # Agent registry - reads are role-scoped and writes are proxy-admin-gated # inside agent_endpoints/endpoints.py diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 9708161397a..244284bf2c1 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -39,6 +39,7 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, refresh_proxy_server_request_body_snapshot, ) +from litellm.proxy.management_endpoints.auto_router_usage import router as usage_router from litellm.proxy.management_endpoints.common_utils import ( _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # shared owner of team-admin membership ) @@ -93,6 +94,7 @@ else: pass router: Final = APIRouter() +router.include_router(usage_router) class _TeamTable(Protocol): diff --git a/litellm/proxy/management_endpoints/auto_router_usage.py b/litellm/proxy/management_endpoints/auto_router_usage.py new file mode 100644 index 00000000000..1928ee2baff --- /dev/null +++ b/litellm/proxy/management_endpoints/auto_router_usage.py @@ -0,0 +1,109 @@ +from collections.abc import Mapping, Sequence +from datetime import date, datetime, time, timedelta +from typing import Annotated, Final, Protocol + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, TypeAdapter + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.proxy._types import UserAPIKeyAuth, user_api_key_has_admin_view +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_endpoints.common_utils import require_caller_user_id_for_non_admin + +router: Final = APIRouter() + + +class AutoRouterUsage(BaseModel): + model: str + router_name: str | None + router_type: str | None + tier: str | None + requests: int + spend: float + + +class RoutingUsageDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + +def routing_usage_database() -> RoutingUsageDatabase: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=503, detail="Database is not connected") + return prisma_client.db # pyright: ignore[reportReturnType] # wrapper forwards query_raw through __getattr__ + + +ROUTING_USAGE_SQL: Final = f""" +WITH requests AS ( + SELECT model, spend, + CASE WHEN jsonb_typeof(metadata->'routing_decision') = 'object' + AND metadata->'routing_decision' <> '{{}}'::jsonb + THEN COALESCE(NULLIF(metadata#>>'{{routing_decision,router_model_name}}', ''), NULLIF(model_group, '')) + END AS router_name, + NULLIF(metadata#>>'{{routing_decision,router_type}}', '') AS router_type, + NULLIF(metadata#>>'{{routing_decision,tier}}', '') AS tier + FROM "LiteLLM_SpendLogs" + WHERE "startTime" >= $1::timestamp AND "startTime" < $2::timestamp + AND ($3::text IS NULL OR "user" = $3) + AND ($4::text IS NULL OR api_key = $4) + AND NULLIF(metadata->>'{INTERNAL_CALL_ORIGIN_METADATA_KEY}', '') IS NULL + AND ($7::text IS NULL OR model = $7) +) +SELECT model, router_name, router_type, tier, COUNT(*)::int AS requests, + COALESCE(SUM(spend), 0)::float8 AS spend +FROM requests +WHERE ($5::text IS NULL OR router_name = $5) + AND ($6::text IS NULL OR router_type = $6) +GROUP BY model, router_name, router_type, tier +ORDER BY spend DESC, model, router_name, tier +""" + +_USAGE_ROWS: Final = TypeAdapter(tuple[AutoRouterUsage, ...]) + + +@router.get( + "/auto_router/usage", + tags=["auto router"], # mutable-ok: FastAPI's decorator requires a list + response_model=tuple[AutoRouterUsage, ...], +) +async def get_auto_router_usage( + start_date: date, + end_date: date, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + db: Annotated[RoutingUsageDatabase, Depends(routing_usage_database)], + destination_model: Annotated[str | None, Query(min_length=1)] = None, + router_name: Annotated[str | None, Query(min_length=1)] = None, + router_type: Annotated[str | None, Query(min_length=1)] = None, + user_id: Annotated[str | None, Query(min_length=1)] = None, + api_key: Annotated[str | None, Query(min_length=1)] = None, +) -> tuple[AutoRouterUsage, ...]: + """Requests and destination-model spend from retained logs in inclusive UTC days. + + Select one model or one router. Internal classifier and shadow-evaluation calls + are excluded. Non-admins can only see requests attributed to their own user. + """ + if (destination_model is None) == (router_name is None): + raise HTTPException(status_code=400, detail="Select exactly one model or router") + if end_date < start_date or end_date == date.max: + raise HTTPException(status_code=400, detail="Invalid date range") + if router_type is not None and router_name is None: + raise HTTPException(status_code=400, detail="router_type requires router_name") + scoped_user: Final = ( + user_id + if user_api_key_has_admin_view(user_api_key_dict) + else require_caller_user_id_for_non_admin(user_api_key_dict) + ) + if user_id is not None and user_id != scoped_user: + raise HTTPException(status_code=403, detail="Cannot view another user's routing usage") + rows: Final = await db.query_raw( + ROUTING_USAGE_SQL, + datetime.combine(start_date, time.min).isoformat(), + datetime.combine(end_date + timedelta(days=1), time.min).isoformat(), + scoped_user, + api_key, + router_name, + router_type, + destination_model, + ) + return _USAGE_ROWS.validate_python(rows) diff --git a/tests/proxy_behavior/spend/test_auto_router_usage.py b/tests/proxy_behavior/spend/test_auto_router_usage.py new file mode 100644 index 00000000000..3fd8f935eff --- /dev/null +++ b/tests/proxy_behavior/spend/test_auto_router_usage.py @@ -0,0 +1,90 @@ +import json +from datetime import date +from typing import Final +from uuid import uuid4 + +import pytest + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.management_endpoints.auto_router_usage import get_auto_router_usage + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +async def test_actual_model_spend_sources_tiers_and_scope(db): + key: Final = f"routing-usage-{uuid4()}" + records: Final = ( + ("fast", 1.0, "a", "SIMPLE", None, "owner", "2026-01-01T00:00:00"), + ("strong", 5.0, "a", "SIMPLE", None, "owner", "2026-01-01T12:00:00"), + ("fast", 0.5, "a", None, None, "owner", "2026-01-01T12:00:00"), + ("fast", 2.0, "b", "COMPLEX", None, "owner", "2026-01-01T12:00:00"), + ("fast", 3.0, None, None, None, "owner", "2026-01-01T12:00:00"), + ("fast", 90.0, "a", "SIMPLE", "shadow_eval_router", "owner", "2026-01-01T12:00:00"), + ("fast", 80.0, None, None, "autorouter_classifier", "owner", "2026-01-01T12:00:00"), + ("fast", 70.0, "a", "SIMPLE", None, "other-user", "2026-01-01T12:00:00"), + ("fast", 60.0, "a", "SIMPLE", None, "owner", "2026-01-02T00:00:00"), + ) + try: + for index, (model, spend, router, tier, origin, user, at) in enumerate(records): + metadata: Final = json.dumps( + { + "routing_decision": {"router_model_name": router, "router_type": "complexity", "tier": tier} + if router + else None, + "internal_call_origin": origin, + } + ) + await db.execute_raw( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, model, model_group, spend, ' + '"user", "startTime", "endTime", metadata) ' + "VALUES ($1, 'completion', $2, $3, 'renamed-alias', $4, $5, $6::timestamp, $6::timestamp, $7::jsonb)", + f"{key}-{index}", + key, + model, + spend, + user, + at, + metadata, + ) + caller: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="owner") + model_usage: Final = await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 1), + caller, + db, + destination_model="fast", + api_key=key, + ) + assert {(row.router_name, row.tier, row.requests, row.spend) for row in model_usage} == { + (None, None, 1, 3.0), + ("b", "COMPLEX", 1, 2.0), + ("a", "SIMPLE", 1, 1.0), + ("a", None, 1, 0.5), + } + router_usage: Final = await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 1), + caller, + db, + router_name="a", + router_type="complexity", + api_key=key, + ) + assert {(row.model, row.tier, row.requests, row.spend) for row in router_usage} == { + ("fast", "SIMPLE", 1, 1.0), + ("strong", "SIMPLE", 1, 5.0), + ("fast", None, 1, 0.5), + } + assert ( + await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 1), + caller, + db, + destination_model="fast", + api_key=f"{key}-other", + ) + == () + ) + finally: + await db.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE api_key = $1', key) diff --git a/tests/unit/proxy/management_endpoints/test_auto_router_usage.py b/tests/unit/proxy/management_endpoints/test_auto_router_usage.py new file mode 100644 index 00000000000..2ab5fe5093a --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_auto_router_usage.py @@ -0,0 +1,93 @@ +from datetime import date +from unittest.mock import AsyncMock + +import pytest +from fastapi import HTTPException + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.management_endpoints.auto_router_usage import get_auto_router_usage + + +class Database: + def __init__(self) -> None: + self.query_raw = AsyncMock(return_value=[]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) +async def test_admin_filters_reach_query_as_parameters(role: LitellmUserRoles) -> None: + db = Database() + await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 2), + UserAPIKeyAuth(user_role=role), + db, + router_name="router'quoted", + router_type="complexity", + user_id="owner", + api_key="key-hash", + ) + assert db.query_raw.call_args.args[1:] == ( + "2026-01-01T00:00:00", + "2026-01-03T00:00:00", + "owner", + "key-hash", + "router'quoted", + "complexity", + None, + ) + + +@pytest.mark.asyncio +async def test_non_admin_is_scoped_to_own_user_when_filter_omitted() -> None: + db = Database() + await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 1), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="own-user"), + db, + destination_model="model-a", + ) + assert db.query_raw.call_args.args[1:] == ( + "2026-01-01T00:00:00", + "2026-01-02T00:00:00", + "own-user", + None, + None, + None, + "model-a", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("caller", [None, "own-user"]) +async def test_non_admin_cannot_read_other_users_or_unbound_service_account(caller: str | None) -> None: + db = Database() + with pytest.raises(HTTPException) as error: + await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 1), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id=caller), + db, + destination_model="model-a", + user_id="another-user", + ) + assert error.value.status_code == 403 + db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model,router_name", [(None, None), ("model-a", "router-a")]) +async def test_query_requires_one_specific_model_or_router(model: str | None, router_name: str | None) -> None: + db = Database() + with pytest.raises(HTTPException) as error: + await get_auto_router_usage( + date(2026, 1, 1), + date(2026, 1, 1), + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + db, + destination_model=model, + router_name=router_name, + ) + assert error.value.status_code == 400 + db.query_raw.assert_not_called() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index a144630cdd0..5f3bc394c1e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -1,3 +1,6 @@ +vi.mock("@/components/UsagePage/useRoutingUsage", () => ({ + useRoutingUsage: () => ({ data: undefined, isError: false }), +})); import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, render, screen, within } from "@testing-library/react"; import React from "react"; @@ -6,7 +9,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import { ApiError } from "@/lib/http/client"; -vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() })); +vi.mock("./useAutoRouterBenchmarks", async (importOriginal) => ({ + ...(await importOriginal()), + useAutoRouterBenchmarks: vi.fn(), +})); vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAutoRouters: vi.fn() })); vi.mock("./ShadowEvalSection", () => ({ default: () =>
})); vi.mock("@/components/shared/advanced_date_picker", () => ({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index 063598bd46e..6bc74ffb8a2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -1,6 +1,8 @@ "use client"; import React, { useState } from "react"; +import { useRoutingUsage } from "@/components/UsagePage/useRoutingUsage"; +import type { RoutingUsage } from "@/components/UsagePage/routingUsage"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels"; @@ -25,6 +27,7 @@ import { groupLabel, pctLabel, viewFor, + viewGroup, type AutoRouterBenchmarksResponse, type AutoRouterCacheStats, type BenchmarkView, @@ -33,7 +36,7 @@ import { import { classificationRatePer1kTurns, formatRangeLabel, usd } from "./costOptimizationUtils"; import ShadowEvalSection from "./ShadowEvalSection"; import TierTurnsChart from "./TierTurnsChart"; -import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; +import { benchmarksWindow, useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; import { DailyActivityRange } from "./useDailyActivityRange"; const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => ( @@ -273,6 +276,8 @@ const CachingCard: React.FC<{ cache: AutoRouterCacheStats }> = ({ cache }) => { }; interface BenchmarksBodyProps { + usage?: readonly RoutingUsage[]; + usageUnavailable?: boolean; isPending: boolean; error: unknown; data: AutoRouterBenchmarksResponse | undefined; @@ -280,7 +285,15 @@ interface BenchmarksBodyProps { autoRouters: readonly AutoRouterDeployment[]; } -const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey, autoRouters }) => { +const BenchmarksBody: React.FC = ({ + isPending, + error, + data, + selectedKey, + autoRouters, + usage, + usageUnavailable, +}) => { if (isPending) return Loading auto-router usage...; if (error instanceof ApiError && error.status === 403) { return Auto-router usage is visible to proxy admin roles only; @@ -293,7 +306,7 @@ const BenchmarksBody: React.FC = ({ isPending, error, data, <> - +
= ({ const [selectedKey, setSelectedKey] = useState(ALL_ROUTERS); const { data: autoRouters } = useAutoRouters(); + const selected = data ? viewGroup(viewFor(data, selectedKey)) : null; + const window = benchmarksWindow(dateValue, new Date()); + const usageQuery = { + start_date: window.start_date ?? "", + end_date: window.end_date ?? "", + router_name: selected?.router_name, + router_type: selected?.router_type, + api_key: apiKey, + user_id: userId, + }; + const hasUsageWindow = Boolean(window.start_date && window.end_date); + const usage = useRoutingUsage(usageQuery, Boolean(accessToken && selected && hasUsageWindow)); const groups = data?.groups ?? []; const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers"; const rangeLabel = formatRangeLabel(dateValue.from, dateValue.to); @@ -388,6 +413,8 @@ export const AutoRouterUsageView: React.FC = ({ error={error} data={data} selectedKey={selectedKey} + usage={usage.data} + usageUnavailable={usage.isError} autoRouters={autoRouters ?? []} />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx index e4417d77463..9282ea11d2c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx @@ -1,4 +1,4 @@ -import { render, screen } from "@testing-library/react"; +import { fireEvent, render, screen } from "@testing-library/react"; import React from "react"; import { describe, expect, it, vi } from "vitest"; @@ -177,3 +177,31 @@ describe("TierTurnsChart", () => { expect(container).toBeEmptyDOMElement(); }); }); + +it("switches the existing donut to actual spend and shows each destination model", () => { + render( + , + ); + expect(screen.getByText("Simple 80% · $6.00")).toBeInTheDocument(); + expect(screen.getByText("strong · 1 request (20%) · $5.00")).toBeInTheDocument(); + expect(screen.getByText("Default / no tier 20% · $4.00")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Spend" })); + expect(screen.getByTestId("donut")).toHaveTextContent("$10.00"); + expect(screen.getByText("Simple 60% · $6.00")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Spend" })).toHaveAttribute("aria-pressed", "true"); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx index 44cca6331b8..c8b6ce19cb1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx @@ -1,6 +1,7 @@ "use client"; -import React from "react"; +import React, { useState } from "react"; +import { routingTiers, routingSpend, trafficShare, type RoutingUsage } from "@/components/UsagePage/routingUsage"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import { hydrateTierLabels } from "@/components/add_model/build_complexity_router_config"; @@ -79,31 +80,67 @@ const tierModelsFor = ( }; interface TierTurnsChartProps { + usage?: readonly RoutingUsage[]; + usageUnavailable?: boolean; view: BenchmarkView; autoRouters: readonly AutoRouterDeployment[]; } -const TierTurnsChart: React.FC = ({ view, autoRouters }) => { +const TierTurnsChart: React.FC = ({ view, autoRouters, usage, usageUnavailable }) => { + const [mode, setMode] = useState<"turns" | "spend">("turns"); const group = viewGroup(view); const entries = Object.entries(group?.tier_turns ?? {}).filter(([, turns]) => turns > 0); - if (!group || entries.length === 0) return null; + if (!group || (entries.length === 0 && !usage?.length)) return null; const tierLabels = tierLabelsFor(group.router_name, group.router_type, autoRouters); - const total = entries.reduce((sum, [, turns]) => sum + turns, 0); - const slices = entries.map(([tier, turns]) => ({ - tier: tierDisplayLabel(tier, tierLabels), - turns, - models: tierModelsFor(tier, group.router_name, group.router_type, autoRouters), - })); + const measured = usage !== undefined && usage.length > 0; + const spendUnavailable = !measured && (usageUnavailable || usage !== undefined); + const slices = measured + ? routingTiers(usage).map((row) => ({ + tier: tierDisplayLabel(row.tier ?? "Default / no tier", tierLabels), + turns: row.requests, + spend: row.spend, + models: row.models.map((model) => model.model), + modelUsage: row.models, + })) + : entries.map(([tier, turns]) => ({ + tier: tierDisplayLabel(tier, tierLabels), + turns, + spend: 0, + models: tierModelsFor(tier, group.router_name, group.router_type, autoRouters), + modelUsage: [], + })); + const category = measured ? mode : "turns"; + const total = slices.reduce((sum, slice) => sum + slice[category], 0); + const requests = slices.reduce((sum, slice) => sum + slice.turns, 0); const colors = slices.map((_, idx) => DEFAULT_COLOR_CYCLE[idx % DEFAULT_COLOR_CYCLE.length]); return ( - Routing by tier +
+ Routing by tier + {measured && ( +
+ {(["turns", "spend"] as const).map((value) => ( + + ))} +
+ )} +

- Turns each tier served. Turns the classifier sent to the default model belong to no tier and are not counted - here, so this can total less than the router's turns. + {measured + ? "Requests and model spend from retained logs in this period. Excludes classifier and shadow-evaluation calls." + : "Turns each tier served. Turns the classifier sent to the default model belong to no tier and are not counted here, so this can total less than the router's turns."} + {spendUnavailable && " Model spend is unavailable for this range."}

@@ -112,11 +149,15 @@ const TierTurnsChart: React.FC = ({ view, autoRouters }) => className="h-80" data={slices} index="tier" - category="turns" + category={category} colors={colors} - valueFormatter={(value) => value.toLocaleString()} + valueFormatter={category === "spend" ? routingSpend : (value) => value.toLocaleString()} showLabel - label={`${total.toLocaleString()} total turns`} + label={ + category === "spend" + ? routingSpend(total) + : `${total.toLocaleString()} total ${measured ? "requests" : "turns"}` + } />
    {slices.map((slice, idx) => ( @@ -127,11 +168,19 @@ const TierTurnsChart: React.FC = ({ view, autoRouters }) => />

    - {slice.tier} {Math.round((100 * slice.turns) / total).toLocaleString()}% + {slice.tier} {trafficShare(slice[category], total)} + {measured && ` · ${routingSpend(slice.spend)}`}

    - {slice.models.length > 0 && ( -

    {slice.models.join(", ")}

    - )} + {measured + ? slice.modelUsage.map((model) => ( +

    + {model.model} · {model.requests.toLocaleString()} {model.requests === 1 ? "request" : "requests"} ( + {trafficShare(model.requests, requests)}) · {routingSpend(model.spend)} +

    + )) + : slice.models.length > 0 && ( +

    {slice.models.join(", ")}

    + )}
    ))} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 4c5db0def62..fb967a948bd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -34,6 +34,7 @@ import { getApiKeyTruncation, getExportBlockedReason } from "@/components/Entity import KeyActivityPanel from "@/components/UsagePage/components/KeyActivityPanel"; import { Team } from "@/components/key_team_helpers/key_list"; import { + formatDate, gatewayDailyActivityCall, Organization, tagListCall, @@ -873,7 +874,18 @@ const UsagePage: React.FC = ({ teams, organizations }) => {
    - + vi.restoreAllMocks()); + +describe("model traffic sources", () => { + it("loads the selected model and user, then opens direct and router request/spend figures", async () => { + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(rows), { + headers: { "Content-Type": "application/json" }, + }), + ); + const client = new QueryClient(); + render( + + + , + ); + fireEvent.click(await screen.findByRole("button", { name: "40% via auto-router" })); + expect(await screen.findByText("Direct")).toBeVisible(); + expect(screen.getByText("router-a")).toBeVisible(); + expect(screen.getByText("$1.20")).toBeVisible(); + expect(screen.getByText("$0.80")).toBeVisible(); + expect(screen.getByText("60%")).toBeVisible(); + expect(screen.getByText(/Based on 10 retained requests/)).toBeVisible(); + const request = fetch.mock.calls[0][0] as Request; + const query = new URL(request.url).searchParams; + expect(Object.fromEntries(query)).toEqual({ ...scope, destination_model: "fast" }); + }); + + it("does not present unavailable or missing logs as zero-percent routed traffic", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("[]", { headers: { "Content-Type": "application/json" } }), + ); + render( + + + , + ); + expect(await screen.findByText("No retained requests for traffic sources")).toBeVisible(); + expect(screen.queryByRole("button", { name: /via auto-router/ })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/ModelRoutingShare.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/ModelRoutingShare.tsx new file mode 100644 index 00000000000..ca864cbe1d9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/components/ModelRoutingShare.tsx @@ -0,0 +1,48 @@ +import { useRoutingUsage } from "../useRoutingUsage"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { routingSources, routingSpend, trafficShare, type RoutingUsageScope } from "../routingUsage"; + +export default function ModelRoutingShare({ model, scope }: { model: string; scope: RoutingUsageScope }) { + const { data, isPending, isError } = useRoutingUsage({ ...scope, destination_model: model }); + if (isPending) return

    Loading traffic sources...

    ; + if (isError) return

    Traffic sources unavailable

    ; + const sources = routingSources(data ?? []); + const total = sources.reduce((sum, source) => sum + source.requests, 0); + if (total === 0) return

    No retained requests for traffic sources

    ; + const routed = sources.filter((source) => source.name !== null).reduce((sum, source) => sum + source.requests, 0); + return ( + + + {trafficShare(routed, total)} via auto-router + + + Traffic sources + + + + Source + Requests + Share + Spend + + + + {sources.map((source) => ( + + {source.name ?? "Direct"} + {source.requests.toLocaleString()} + {trafficShare(source.requests, total)} + {routingSpend(source.spend)} + + ))} + +
    +

    + Based on {total.toLocaleString()} retained requests in this period. Model spend excludes internal classifier + and shadow-evaluation calls. Usage totals can include requests whose logs have expired. +

    +     +     + ); +} diff --git a/ui/litellm-dashboard/src/components/UsagePage/routingUsage.test.ts b/ui/litellm-dashboard/src/components/UsagePage/routingUsage.test.ts new file mode 100644 index 00000000000..d800e8b5be0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/routingUsage.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { routingSources, routingTiers, trafficShare, type RoutingUsage } from "./routingUsage"; + +const rows: RoutingUsage[] = [ + { model: "fast", router_name: "router-a", router_type: "complexity", tier: "SIMPLE", requests: 2, spend: 0.4 }, + { model: "strong", router_name: "router-a", router_type: "complexity", tier: "SIMPLE", requests: 1, spend: 1.2 }, + { model: "fast", router_name: "router-a", router_type: "complexity", tier: null, requests: 1, spend: 0.2 }, + { model: "fast", router_name: "router-b", router_type: "quality", tier: "2", requests: 1, spend: 0.2 }, + { model: "fast", router_name: null, router_type: null, tier: null, requests: 6, spend: 1.2 }, +]; + +describe("routing usage", () => { + it("combines a router's tiers and separates direct traffic and other routers for one model", () => { + const sources = routingSources(rows.filter((row) => row.model === "fast")); + expect(sources).toEqual([ + { name: null, requests: 6, spend: 1.2 }, + { name: "router-a", requests: 3, spend: expect.closeTo(0.6) }, + { name: "router-b", requests: 1, spend: 0.2 }, + ]); + expect(trafficShare(4, 10)).toBe("40%"); + }); + + it("keeps both models within a tier and includes fallback calls without inventing their tier", () => { + expect(routingTiers(rows.filter((row) => row.router_name === "router-a"))).toEqual([ + { + tier: "SIMPLE", + requests: 3, + spend: expect.closeTo(1.6), + models: [ + { model: "fast", requests: 2, spend: 0.4 }, + { model: "strong", requests: 1, spend: 1.2 }, + ], + }, + { tier: null, requests: 1, spend: 0.2, models: [{ model: "fast", requests: 1, spend: 0.2 }] }, + ]); + }); + + it("handles zero-cost traffic and an empty range", () => { + expect(trafficShare(0, 0)).toBe("0%"); + expect(routingTiers([])).toEqual([]); + expect(routingSources([{ ...rows[0], spend: 0 }])).toEqual([ + { name: null, requests: 0, spend: 0 }, + { name: "router-a", requests: 2, spend: 0 }, + ]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/UsagePage/routingUsage.ts b/ui/litellm-dashboard/src/components/UsagePage/routingUsage.ts new file mode 100644 index 00000000000..ef79a8c5079 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/routingUsage.ts @@ -0,0 +1,39 @@ +import type { components, paths } from "@/lib/http/schema"; + +export type RoutingUsage = components["schemas"]["AutoRouterUsage"]; +export type RoutingUsageQuery = paths["/auto_router/usage"]["get"]["parameters"]["query"]; +export type RoutingUsageScope = Pick; + +export const routingSources = (rows: readonly RoutingUsage[]) => { + const names = [null, ...new Set(rows.flatMap((row) => (row.router_name === null ? [] : [row.router_name])))]; + return names.map((name) => ({ + name, + requests: rows.filter((row) => row.router_name === name).reduce((sum, row) => sum + row.requests, 0), + spend: rows.filter((row) => row.router_name === name).reduce((sum, row) => sum + row.spend, 0), + })); +}; + +export const routingTiers = (rows: readonly RoutingUsage[]) => + [...new Set(rows.map((row) => row.tier))].map((tier) => { + const tierRows = rows.filter((row) => row.tier === tier); + return { + tier, + requests: tierRows.reduce((sum, row) => sum + row.requests, 0), + spend: tierRows.reduce((sum, row) => sum + row.spend, 0), + models: [...new Set(tierRows.map((row) => row.model))].map((model) => ({ + model, + requests: tierRows.filter((row) => row.model === model).reduce((sum, row) => sum + row.requests, 0), + spend: tierRows.filter((row) => row.model === model).reduce((sum, row) => sum + row.spend, 0), + })), + }; + }); + +export const trafficShare = (requests: number, total: number) => + `${(total > 0 ? (100 * requests) / total : 0).toLocaleString(undefined, { maximumFractionDigits: 1 })}%`; + +export const routingSpend = (spend: number) => + spend.toLocaleString("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: spend > 0 && spend < 0.01 ? 6 : 2, + }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/useRoutingUsage.ts b/ui/litellm-dashboard/src/components/UsagePage/useRoutingUsage.ts new file mode 100644 index 00000000000..93a05804625 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/useRoutingUsage.ts @@ -0,0 +1,5 @@ +import { $api } from "@/lib/http/api"; +import type { RoutingUsageQuery } from "./routingUsage"; + +export const useRoutingUsage = (query: RoutingUsageQuery, enabled = true) => + $api.useQuery("get", "/auto_router/usage", { params: { query } }, { enabled, retry: false, staleTime: 60_000 }); diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index 95315f8ec27..d7fae0c7500 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -15,12 +15,15 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component import { ChevronDown } from "lucide-react"; import React, { useState } from "react"; import { Team } from "./key_team_helpers/key_list"; +import ModelRoutingShare from "./UsagePage/components/ModelRoutingShare"; +import type { RoutingUsageScope } from "./UsagePage/routingUsage"; import KeyModelUsageView from "./UsagePage/components/KeyModelUsageView"; import { keyActivityLabel } from "./UsagePage/keyActivityLabel"; import { DailyData, KeyMetricWithMetadata, ModelActivityData, TopApiKeyData, TopModelData } from "./UsagePage/types"; import { averageResponseTimeMs, formatResponseTime, valueFormatter } from "./UsagePage/utils/value_formatters"; interface ActivityMetricsProps { + routingScope?: RoutingUsageScope; modelMetrics: Record; hidePromptCachingMetrics?: boolean; } @@ -41,8 +44,10 @@ const ModelSection = ({ modelName, metrics, hidePromptCachingMetrics = false, + routingScope, }: { modelName: string; + routingScope?: RoutingUsageScope; metrics: ModelActivityData; hidePromptCachingMetrics?: boolean; }) => { @@ -54,6 +59,7 @@ const ModelSection = ({

    Total Requests

    {metrics.total_requests.toLocaleString()}

    + {routingScope && }     @@ -300,7 +306,11 @@ const ModelCollapsible = ({ ); }; -export const ActivityMetrics: React.FC = ({ modelMetrics, hidePromptCachingMetrics = false }) => { +export const ActivityMetrics: React.FC = ({ + modelMetrics, + hidePromptCachingMetrics = false, + routingScope, +}) => { const modelNames = Object.keys(modelMetrics).sort((a, b) => { if (a === "") return 1; if (b === "") return -1; @@ -479,6 +489,7 @@ export const ActivityMetrics: React.FC = ({ modelMetrics, > diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 52718ce2951..11774c91b5b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1493,6 +1493,29 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/usage": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Auto Router Usage + * @description Requests and destination-model spend from retained logs in inclusive UTC days. + * + * Select one model or one router. Internal classifier and shadow-evaluation calls + * are excluded. Non-admins can only see requests attributed to their own user. + */ + get: operations["get_auto_router_usage_auto_router_usage_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/validate_complexity_router_config": { parameters: { query?: never; @@ -25307,6 +25330,21 @@ export interface components { */ turns: number; }; + /** AutoRouterUsage */ + AutoRouterUsage: { + /** Model */ + model: string; + /** Requests */ + requests: number; + /** Router Name */ + router_name: string | null; + /** Router Type */ + router_type: string | null; + /** Spend */ + spend: number; + /** Tier */ + tier: string | null; + }; /** AwsSessionTag */ AwsSessionTag: { /** Key */ @@ -49006,6 +49044,43 @@ export interface operations { }; }; }; + get_auto_router_usage_auto_router_usage_get: { + parameters: { + query: { + start_date: string; + end_date: string; + destination_model?: string | null; + router_name?: string | null; + router_type?: string | null; + user_id?: string | null; + api_key?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AutoRouterUsage"][]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; validate_complexity_router_config_auto_router_validate_complexity_router_config_post: { parameters: { query?: never;