From f6fc6d299a7daa6fd405c9ddd0f4fabdc4b90432 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 22 Jul 2026 17:49:14 -0700 Subject: [PATCH 01/49] feat(ui): add standalone /connect route for MCP OAuth The MCP connect surface only existed as the Integrations tab inside the enable_chat_ui-gated /chat shell, so a keyless SSO user was bounced to the dashboard and could never reach it unless an admin enabled Chat UI first. Add a sibling /connect route with its own thin, auth-only layout that renders the same MCPAppsPanel without the chat-ui gate or chat shell. The user OAuth flow already returns to whatever URL started it, so no backend changes are needed. The chat playground and its gate are left unchanged. --- .../src/app/connect/layout.test.tsx | 65 +++++++++++++++++++ .../src/app/connect/layout.tsx | 20 ++++++ .../src/app/connect/page.test.tsx | 55 ++++++++++++++++ ui/litellm-dashboard/src/app/connect/page.tsx | 36 ++++++++++ 4 files changed, 176 insertions(+) create mode 100644 ui/litellm-dashboard/src/app/connect/layout.test.tsx create mode 100644 ui/litellm-dashboard/src/app/connect/layout.tsx create mode 100644 ui/litellm-dashboard/src/app/connect/page.test.tsx create mode 100644 ui/litellm-dashboard/src/app/connect/page.tsx diff --git a/ui/litellm-dashboard/src/app/connect/layout.test.tsx b/ui/litellm-dashboard/src/app/connect/layout.test.tsx new file mode 100644 index 00000000000..795a79d77e3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/connect/layout.test.tsx @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import ConnectLayout from "./layout"; + +const { mockUseAuthorized, state } = vi.hoisted(() => { + const state = { + accessToken: "token-123" as string | null, + isAuthorized: true, + isLoading: false, + }; + return { + state, + mockUseAuthorized: vi.fn(() => ({ + accessToken: state.accessToken, + isAuthorized: state.isAuthorized, + isLoading: state.isLoading, + })), + }; +}); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: mockUseAuthorized })); +vi.mock("@/components/navbar", () => ({ default: () =>
})); +vi.mock("@/contexts/ThemeContext", () => ({ + ThemeProvider: ({ children }: { children: React.ReactNode }) => <>{children}, +})); + +describe("ConnectLayout", () => { + afterEach(() => { + state.accessToken = "token-123"; + state.isAuthorized = true; + state.isLoading = false; + }); + + it("renders the connect surface for an authorized user without any chat-ui flag", () => { + render( + +
+ , + ); + expect(screen.getByTestId("navbar")).toBeInTheDocument(); + expect(screen.getByTestId("page-content")).toBeInTheDocument(); + }); + + it("renders nothing when the user is not authorized", () => { + state.isAuthorized = false; + render( + +
+ , + ); + expect(screen.queryByTestId("page-content")).not.toBeInTheDocument(); + expect(screen.queryByTestId("navbar")).not.toBeInTheDocument(); + }); + + it("renders nothing while authorization is still loading", () => { + state.isLoading = true; + render( + +
+ , + ); + expect(screen.queryByTestId("page-content")).not.toBeInTheDocument(); + expect(screen.queryByTestId("navbar")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/connect/layout.tsx b/ui/litellm-dashboard/src/app/connect/layout.tsx new file mode 100644 index 00000000000..63b1c484094 --- /dev/null +++ b/ui/litellm-dashboard/src/app/connect/layout.tsx @@ -0,0 +1,20 @@ +"use client"; + +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import Navbar from "@/components/navbar"; +import { ThemeProvider } from "@/contexts/ThemeContext"; + +export default function ConnectLayout({ children }: { children: React.ReactNode }) { + const { accessToken, isAuthorized, isLoading } = useAuthorized(); + + if (isLoading || !isAuthorized) return null; + + return ( + +
+ +
{children}
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/app/connect/page.test.tsx b/ui/litellm-dashboard/src/app/connect/page.test.tsx new file mode 100644 index 00000000000..07b0e7a305a --- /dev/null +++ b/ui/litellm-dashboard/src/app/connect/page.test.tsx @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import ConnectPage from "./page"; + +interface PanelProps { + accessToken: string; + selectedServers: string[]; + onChange: (servers: string[]) => void; +} + +const { mockReplace, mockPanel, state } = vi.hoisted(() => { + const state = { + oauthReturn: null as string | null, + }; + return { + state, + mockReplace: vi.fn(), + mockPanel: vi.fn((_props: PanelProps) =>
), + }; +}); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: mockReplace }), + useSearchParams: () => ({ get: (key: string) => (key === "mcpOauthReturn" ? state.oauthReturn : null) }), +})); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "token-123" }), +})); +vi.mock("@/components/chat/MCPAppsPanel", () => ({ default: mockPanel })); + +describe("ConnectPage", () => { + afterEach(() => { + state.oauthReturn = null; + mockReplace.mockClear(); + mockPanel.mockClear(); + }); + + it("renders the MCP connect panel with the user's access token", () => { + render(); + expect(screen.getByTestId("mcp-apps-panel")).toBeInTheDocument(); + expect(mockPanel.mock.calls[0][0]).toMatchObject({ accessToken: "token-123", selectedServers: [] }); + }); + + it("strips the mcpOauthReturn param from the URL after an OAuth return", () => { + state.oauthReturn = "apps"; + window.history.replaceState({}, "", "/connect?mcpOauthReturn=apps"); + render(); + expect(mockReplace).toHaveBeenCalledWith("/connect"); + }); + + it("does not rewrite the URL when there is no OAuth return param", () => { + render(); + expect(mockReplace).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/connect/page.tsx b/ui/litellm-dashboard/src/app/connect/page.tsx new file mode 100644 index 00000000000..84770915e46 --- /dev/null +++ b/ui/litellm-dashboard/src/app/connect/page.tsx @@ -0,0 +1,36 @@ +"use client"; + +import { Suspense, useEffect, useState } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import MCPAppsPanel from "@/components/chat/MCPAppsPanel"; + +function ConnectPageContent() { + const { accessToken } = useAuthorized(); + const [selectedServers, setSelectedServers] = useState([]); + const router = useRouter(); + const searchParams = useSearchParams(); + const oauthReturn = searchParams.get("mcpOauthReturn"); + + useEffect(() => { + if (oauthReturn) { + const url = new URL(window.location.href); + url.searchParams.delete("mcpOauthReturn"); + router.replace(url.pathname + url.search); + } + }, [oauthReturn, router]); + + return ( +
+ +
+ ); +} + +export default function ConnectPage() { + return ( + + + + ); +} From 9e7b1c0b3617da0d26706d794609dd8aec0e4ffa Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 22 Jul 2026 18:15:37 -0700 Subject: [PATCH 02/49] feat(ui): land keyless users on the connect page after login A standalone /ui/connect route is only reachable if something points a user at it. Post-login the dashboard always rendered the API-keys view, so a keyless SSO user saw an empty dashboard and no path to connect. Redirect to /ui/connect from the dashboard landing when the URL carries ?login=success, the user is not an admin, and their key list is empty. Gating on the post-login marker keeps the dashboard reachable afterwards, and an explicit stored return URL still wins. useKeys takes an optional enabled flag so the lookup only runs on that landing. --- .../src/app/(dashboard)/hooks/keys/useKeys.ts | 3 +- .../src/app/(dashboard)/page.test.tsx | 111 ++++++++++++++++++ .../src/app/(dashboard)/page.tsx | 24 +++- 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.ts index 198058803eb..0df809bc582 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.ts @@ -101,13 +101,14 @@ export const useKeys = ( page: number, pageSize: number, options: KeyListCallOptions = {}, + enabled: boolean = true, ): UseQueryResult => { const { accessToken } = useAuthorized(); return useQuery({ queryKey: keyKeys.list({ page, limit: pageSize, ...options }), queryFn: async () => await keyListCall(accessToken!, page, pageSize, options), - enabled: Boolean(accessToken), + enabled: Boolean(accessToken) && enabled, staleTime: 30000, // 30 seconds placeholderData: keepPreviousData, }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx new file mode 100644 index 00000000000..6fb64333089 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx @@ -0,0 +1,111 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import CreateKeyPage from "./page"; + +interface KeyRow { + token: string; +} + +const { mockReplace, mockUseKeys, mockMigratedHref, state } = vi.hoisted(() => { + const state = { + login: "success" as string | null, + userRole: "Internal User", + keys: [] as KeyRow[], + keysLoading: false, + returnUrl: null as string | null, + }; + return { + state, + mockReplace: vi.fn(), + mockMigratedHref: vi.fn((segment: string) => `/mocked-ui/${segment}`), + mockUseKeys: vi.fn((_page: number, _size: number, _opts: unknown, _enabled: boolean) => ({ + data: state.keysLoading ? undefined : { keys: state.keys, total_count: state.keys.length }, + isLoading: state.keysLoading, + })), + }; +}); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: mockReplace }), + useSearchParams: () => ({ get: (key: string) => (key === "login" ? state.login : null) }), +})); +vi.mock("@/contexts/AuthContext", () => ({ + useAuth: () => ({ + authLoading: false, + token: "tok", + userRole: state.userRole, + userID: "user-1", + }), +})); +vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useKeys: mockUseKeys })); +vi.mock("@/app/(dashboard)/api-keys/ApiKeysDashboard", () => ({ + default: () =>
, +})); +vi.mock("@/components/common_components/LoadingScreen", () => ({ + default: () =>
, +})); +vi.mock("@/components/networking", () => ({ proxyBaseUrl: "" })); +vi.mock("@/utils/migratedPages", () => ({ MIGRATED_PAGES: {}, migratedHref: mockMigratedHref })); +vi.mock("@/utils/returnUrlUtils", () => ({ + buildLoginUrlWithReturn: (u: string) => u, + consumeReturnUrl: () => state.returnUrl, + getLoginUrl: () => "/login", + isValidReturnUrl: () => true, + normalizeUrlForCompare: (u: string) => u, + storeReturnUrl: () => undefined, +})); + +describe("dashboard landing keyless redirect", () => { + afterEach(() => { + state.login = "success"; + state.userRole = "Internal User"; + state.keys = []; + state.keysLoading = false; + state.returnUrl = null; + mockReplace.mockClear(); + mockUseKeys.mockClear(); + mockMigratedHref.mockClear(); + }); + + it("sends a keyless non-admin to the connect page after login", () => { + render(); + expect(mockReplace).toHaveBeenCalledWith("/mocked-ui/connect"); + expect(screen.queryByTestId("api-keys-dashboard")).not.toBeInTheDocument(); + }); + + it("leaves an admin with no keys on the dashboard", () => { + state.userRole = "Admin"; + render(); + expect(mockReplace).not.toHaveBeenCalled(); + expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + }); + + it("leaves a user who already has a key on the dashboard", () => { + state.keys = [{ token: "sk-abc" }]; + render(); + expect(mockReplace).not.toHaveBeenCalled(); + expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + }); + + it("does not redirect outside the post-login landing, and skips the key lookup entirely", () => { + state.login = null; + render(); + expect(mockReplace).not.toHaveBeenCalled(); + expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + expect(mockUseKeys.mock.calls[0][3]).toBe(false); + }); + + it("holds the loading screen while the key lookup is in flight", () => { + state.keysLoading = true; + render(); + expect(screen.getByTestId("loading-screen")).toBeInTheDocument(); + expect(screen.queryByTestId("api-keys-dashboard")).not.toBeInTheDocument(); + expect(mockReplace).not.toHaveBeenCalled(); + }); + + it("yields to an explicit return URL instead of the connect redirect", () => { + state.returnUrl = "/ui/models-and-endpoints"; + render(); + expect(mockReplace).not.toHaveBeenCalledWith("/mocked-ui/connect"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx index 02b2ccf5357..883aff8b36b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx @@ -3,6 +3,8 @@ import ApiKeysDashboard from "@/app/(dashboard)/api-keys/ApiKeysDashboard"; import LoadingScreen from "@/components/common_components/LoadingScreen"; import { proxyBaseUrl } from "@/components/networking"; +import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { isAdminRole } from "@/utils/roles"; import { useAuth } from "@/contexts/AuthContext"; import { buildLoginUrlWithReturn, @@ -17,7 +19,7 @@ import { useRouter, useSearchParams } from "next/navigation"; import { Suspense, useEffect, useRef } from "react"; function CreateKeyPageContent() { - const { authLoading, token } = useAuth(); + const { authLoading, token, userRole, userID } = useAuth(); const router = useRouter(); const searchParams = useSearchParams()!; @@ -26,6 +28,7 @@ function CreateKeyPageContent() { // Track if we've already attempted a return URL redirect to prevent race conditions const hasAttemptedReturnRedirectRef = useRef(false); + const didReturnRedirectRef = useRef(false); const redirectToLogin = authLoading === false && token === null; @@ -75,6 +78,7 @@ function CreateKeyPageContent() { // Only redirect if the return URL is different from the current URL // This prevents infinite redirect loops if (normalizedReturnUrl !== normalizedCurrentUrl) { + didReturnRedirectRef.current = true; window.location.replace(safeUrl.href); } } @@ -83,10 +87,26 @@ function CreateKeyPageContent() { useEffect(() => { if (!token) { hasAttemptedReturnRedirectRef.current = false; + didReturnRedirectRef.current = false; } }, [token]); - if (authLoading || redirectToLogin || isLegacyRedirect) { + const isPostLoginLanding = searchParams.get("login") === "success"; + const isSignedIn = !authLoading && Boolean(token); + const shouldCheckForKeys = isPostLoginLanding && isSignedIn && !isAdminRole(userRole); + const { data: keysData, isLoading: keysLoading } = useKeys(1, 1, { userID }, shouldCheckForKeys); + const isKeylessLanding = shouldCheckForKeys && !keysLoading && keysData?.keys?.length === 0; + const isResolvingKeylessLanding = (shouldCheckForKeys && keysLoading) || isKeylessLanding; + + useEffect(() => { + if (isKeylessLanding && !didReturnRedirectRef.current) { + router.replace(migratedHref("connect")); + } + }, [isKeylessLanding, router]); + + const isRedirecting = redirectToLogin || isLegacyRedirect || isResolvingKeylessLanding; + + if (authLoading || isRedirecting) { return ; } From 9dda5d882c206e6003cf8c0bf54c7155f1736c50 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 22 Jul 2026 22:23:45 -0700 Subject: [PATCH 03/49] test(ui): characterise the memory page's drawer and header before migrating Adds role/text-based coverage for MemoryDetailDrawer (which had none) and extends MemoryView's test past the mocked table to the header, the create modal trigger and the detail drawer round trip. Both are green against the current antd components, so they act as an unedited regression net for the shadcn migration that follows. --- .../_components/MemoryDetailDrawer.test.tsx | 88 +++++++++++++++++++ .../memory/_components/MemoryView.test.tsx | 47 +++++++++- 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.test.tsx new file mode 100644 index 00000000000..df65d5b534f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.test.tsx @@ -0,0 +1,88 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { MemoryRow } from "@/components/networking"; + +import { MemoryDetailDrawer } from "./MemoryDetailDrawer"; + +const makeMemory = (overrides: Partial = {}): MemoryRow => ({ + memory_id: "mem-1", + key: "user:profile", + value: "The user prefers concise answers.", + metadata: null, + user_id: "user-42", + team_id: "team-7", + created_at: "2024-05-01T12:00:00Z", + updated_at: "2024-05-02T12:00:00Z", + created_by: "alice", + updated_by: "bob", + ...overrides, +}); + +describe("MemoryDetailDrawer", () => { + it("renders nothing until a row is selected", () => { + render(); + + expect(screen.queryByText("Memory ID")).not.toBeInTheDocument(); + expect(screen.queryByText("Value")).not.toBeInTheDocument(); + }); + + it("shows the selected row's key, identifiers and value", () => { + render(); + + expect(screen.getByText("user:profile")).toBeInTheDocument(); + expect(screen.getByText("Memory ID")).toBeInTheDocument(); + expect(screen.getByText("mem-1")).toBeInTheDocument(); + expect(screen.getByText("User ID")).toBeInTheDocument(); + expect(screen.getByText("user-42")).toBeInTheDocument(); + expect(screen.getByText("Team ID")).toBeInTheDocument(); + expect(screen.getByText("team-7")).toBeInTheDocument(); + expect(screen.getByText("Value")).toBeInTheDocument(); + expect(screen.getByText("The user prefers concise answers.")).toBeInTheDocument(); + }); + + it("falls back to a dash for a memory with no owning user or team", () => { + render(); + + expect(screen.getAllByText("-")).toHaveLength(2); + expect(screen.queryByText("user-42")).not.toBeInTheDocument(); + }); + + it("omits the metadata block when the row carries no metadata", () => { + render(); + + expect(screen.queryByText("Metadata")).not.toBeInTheDocument(); + }); + + it("pretty-prints metadata as JSON when present", () => { + render(); + + expect(screen.getByText("Metadata")).toBeInTheDocument(); + expect(screen.getByText('{ "tags": [ "example" ] }')).toBeInTheDocument(); + }); + + it("attributes the created and updated timestamps to their actors", () => { + render(); + + expect(screen.getByText(/^Created .* by alice$/)).toBeInTheDocument(); + expect(screen.getByText(/^Updated .* by bob$/)).toBeInTheDocument(); + }); + + it("renders an em dash for a timestamp the backend did not send", () => { + render(); + + expect(screen.getByText("Created —")).toBeInTheDocument(); + }); + + it("closes through the close control", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /close/i })); + + expect(onClose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx index f415c99225a..9ccef5357b9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx @@ -1,5 +1,6 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { render } from "@testing-library/react"; +import { act, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import React from "react"; import { describe, expect, it, vi } from "vitest"; @@ -12,6 +13,7 @@ interface CapturedTableProps { rowCount: number; data: MemoryRow[]; hasActiveSearch: boolean; + onViewClick: (row: MemoryRow) => void; } const captured = vi.hoisted(() => ({ current: null as CapturedTableProps | null })); @@ -42,4 +44,47 @@ describe("MemoryView", () => { expect(captured.current?.rowCount).toBe(0); expect(captured.current?.hasActiveSearch).toBe(false); }); + + it("heads the page with the Memory title and the /v1/memory scope note", () => { + renderView(null); + + expect(screen.getByRole("heading", { name: "Memory" })).toBeInTheDocument(); + expect(screen.getByText("/v1/memory")).toBeInTheDocument(); + expect(screen.getByText(/Scoped to memories visible to your user \/ team \(admins see all\)/)).toBeInTheDocument(); + }); + + it("opens the create modal from the New memory button", async () => { + const user = userEvent.setup(); + renderView(null); + + expect(screen.queryByText("Create memory")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /new memory/i })); + + expect(await screen.findByText("Create memory")).toBeInTheDocument(); + }); + + it("opens the detail drawer for the row the table hands back, and closes it again", async () => { + const user = userEvent.setup(); + renderView(null); + + expect(screen.queryByText("Memory ID")).not.toBeInTheDocument(); + + const row: MemoryRow = { + memory_id: "mem-drawer", + key: "user:profile", + value: "remembered", + metadata: null, + user_id: null, + team_id: null, + }; + act(() => captured.current?.onViewClick(row)); + + expect(await screen.findByText("Memory ID")).toBeInTheDocument(); + expect(screen.getByText("mem-drawer")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /close/i })); + + expect(screen.queryByText("mem-drawer")).not.toBeInTheDocument(); + }); }); From d2f872d04fd67435452beffe26ca3b6f5c23550b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 22 Jul 2026 22:52:15 -0700 Subject: [PATCH 04/49] refactor(ui): migrate memory page to shadcn Replaces the antd Drawer with ui/sheet and the antd Button, Typography and Space usage with ui/button plus token utilities, and swaps the @ant-design PlusOutlined icon for lucide's Plus. Toasts now go through the shared MessageManager so the route no longer imports antd directly. The route's tests were written against the antd components in the previous commit and are unchanged here, so they pass on both implementations. MemoryEditModal is left alone because it is built on antd Form; the table already sits on the shared DataTable. --- ui/litellm-dashboard/eslint-suppressions.json | 18 --- .../memory/_components/MemoryDetailDrawer.tsx | 133 +++++++----------- .../memory/_components/MemoryView.tsx | 49 +++---- 3 files changed, 78 insertions(+), 122 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index e31cdcae596..01258f55ba6 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1128,29 +1128,11 @@ "count": 1 } }, - "src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/memory/_components/MemoryEditModal.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/memory/_components/MemoryView.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx": { - "no-restricted-imports": { - "count": 3 - }, - "react-hooks/preserve-manual-memoization": { - "count": 4 - } - }, "src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx": { "max-params": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx index 970e088ec00..63e6644d7e0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx @@ -1,17 +1,19 @@ "use client"; -import { Drawer, Space, Typography } from "antd"; import React from "react"; import { MemoryRow } from "@/components/networking"; - -const { Text, Paragraph } = Typography; +import { Sheet, SheetContent, SheetHeader, SheetTitle } from "@/components/ui/sheet"; interface MemoryDetailDrawerProps { row: MemoryRow | null; onClose: () => void; } +const CODE_CLASS = "rounded-sm border border-border bg-muted px-1 py-0.5 font-mono text-xs text-foreground"; +const BLOCK_CLASS = "mt-1 rounded-md bg-muted p-3 font-mono whitespace-pre-wrap text-foreground"; +const LABEL_CLASS = "text-sm font-semibold text-foreground"; + function formatTimestamp(ts?: string): string { if (!ts) return "—"; try { @@ -24,90 +26,61 @@ function formatTimestamp(ts?: string): string { export function MemoryDetailDrawer({ row, onClose }: MemoryDetailDrawerProps) { return ( - - {row.key} - - ) : ( - "Memory" - ) - } - width={720} - destroyOnClose + onOpenChange={(open) => { + if (!open) onClose(); + }} > - {row && ( - - -
- - Memory ID - - - {row.memory_id} - + + + {row ? {row.key} : "Memory"} + + {row && ( +
+
+
+ Memory ID + {row.memory_id} +
+
+ User ID + + {row.user_id ?? "-"} + +
+
+ Team ID + + {row.team_id ?? "-"} + +
- - User ID - - {row.user_id ?? "-"} + Value +

{row.value}

-
- - Team ID - - {row.team_id ?? "-"} + {row.metadata !== undefined && row.metadata !== null && ( +
+ Metadata +

{JSON.stringify(row.metadata, null, 2)}

+
+ )} +
+ + Created {formatTimestamp(row.created_at)} + {row.created_by ? ` by ${row.created_by}` : ""} + + + + Updated {formatTimestamp(row.updated_at)} + {row.updated_by ? ` by ${row.updated_by}` : ""} +
- -
- Value - - {row.value} -
- {row.metadata !== undefined && row.metadata !== null && ( -
- Metadata - - {JSON.stringify(row.metadata, null, 2)} - -
- )} - ·} wrap size="small" style={{ color: "rgba(0,0,0,0.45)" }}> - - Created {formatTimestamp(row.created_at)} - {row.created_by ? ` by ${row.created_by}` : ""} - - - Updated {formatTimestamp(row.updated_at)} - {row.updated_by ? ` by ${row.updated_by}` : ""} - - - - )} - + )} + + ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx index fcb15978f47..a91110cf780 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx @@ -3,20 +3,19 @@ import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import type { PaginationState } from "@tanstack/react-table"; -import { PlusOutlined } from "@ant-design/icons"; -import { Button, Space, Typography, message } from "antd"; +import { Plus } from "lucide-react"; import React, { useCallback, useMemo, useState } from "react"; import { MemoryRow, createMemory, deleteMemory, fetchMemoryList, updateMemory } from "@/components/networking"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; +import MessageManager from "@/components/molecules/message_manager"; +import { Button } from "@/components/ui/button"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { MemoryDetailDrawer } from "./MemoryDetailDrawer"; import { MemoryEditModal } from "./MemoryEditModal"; import { MemoryTable } from "./MemoryTable"; -const { Text, Paragraph, Title } = Typography; - interface MemoryViewProps { accessToken: string | null; userID: string | null; @@ -62,7 +61,7 @@ export const MemoryView: React.FC = ({ accessToken }) => { // All three write endpoints share the same success/error plumbing: // - on success: invalidate the list query so every cached page // refetches from scratch (pagination + filter-aware). - // - on error: surface the message via antd `message.error`. + // - on error: surface the message via `MessageManager.error`. const invalidateList = useCallback( () => queryClient.invalidateQueries({ queryKey: [MEMORY_LIST_KEY] }), @@ -75,11 +74,11 @@ export const MemoryView: React.FC = ({ accessToken }) => { return createMemory(accessToken, args); }, onSuccess: (row) => { - message.success(`Created ${row.key}`); + MessageManager.success(`Created ${row.key}`); invalidateList(); }, onError: (err: Error) => { - message.error(`Save failed: ${err.message}`); + MessageManager.error(`Save failed: ${err.message}`); }, }); @@ -90,11 +89,11 @@ export const MemoryView: React.FC = ({ accessToken }) => { return updateMemory(accessToken, key, payload); }, onSuccess: (row) => { - message.success(`Updated ${row.key}`); + MessageManager.success(`Updated ${row.key}`); invalidateList(); }, onError: (err: Error) => { - message.error(`Save failed: ${err.message}`); + MessageManager.error(`Save failed: ${err.message}`); }, }); @@ -104,11 +103,11 @@ export const MemoryView: React.FC = ({ accessToken }) => { return deleteMemory(accessToken, key).then(() => key); }, onSuccess: (key) => { - message.success(`Deleted ${key}`); + MessageManager.success(`Deleted ${key}`); invalidateList(); }, onError: (err: Error) => { - message.error(`Delete failed: ${err.message}`); + MessageManager.error(`Delete failed: ${err.message}`); }, }); @@ -150,7 +149,7 @@ export const MemoryView: React.FC = ({ accessToken }) => { try { metadataPayload = JSON.parse(metadataText); } catch { - message.error("Metadata must be valid JSON (or leave empty)."); + MessageManager.error("Metadata must be valid JSON (or leave empty)."); return false; } } @@ -177,19 +176,21 @@ export const MemoryView: React.FC = ({ accessToken }) => { }; return ( -
- -
+
+
+
- - Memory - - - Inspect what your agents have stored under /v1/memory. Scoped to memories visible to - your user / team (admins see all). - +

Memory

+

+ Inspect what your agents have stored under{" "} + + /v1/memory + + . Scoped to memories visible to your user / team (admins see all). +

-
@@ -209,7 +210,7 @@ export const MemoryView: React.FC = ({ accessToken }) => { onEditClick={handleEdit} onDeleteClick={handleDelete} /> - +
{/* Detail drawer */} setDetailRow(null)} /> From a928db1fab12ff190afc91266c0be8c58b6f11aa Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 22 Jul 2026 23:40:19 -0700 Subject: [PATCH 05/49] fix(ui): keep the memory detail sheet inside the viewport on narrow screens The migrated sheet asked for a flat 720px width while its only max-width came from the primitive's sm:-scoped rule, so below the sm breakpoint no cap applied at all: on a 375px viewport the sheet rendered 720px wide with its left edge at -305px, and because it is position:fixed there was no scroll to reach the hidden content. The primitive's own w-3/4 default did not have this problem; the fixed pixel width is what removed the guard. Caps the width to the viewport at every breakpoint and only asks for 720px from sm up. Verified in a browser at 375px, 700px and 1280px: the sheet is now 375, 700 and 720 wide respectively, always at left 0. --- .../app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx index 63e6644d7e0..5c2fac2d5a9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryDetailDrawer.tsx @@ -32,7 +32,7 @@ export function MemoryDetailDrawer({ row, onClose }: MemoryDetailDrawerProps) { if (!open) onClose(); }} > - + {row ? {row.key} : "Memory"} From 8929e09f4927a65672a161337f365e8546adfaf4 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 09:56:01 -0700 Subject: [PATCH 06/49] fix(ui): gate the keyless connect redirect on internal-user roles isAdminRole compares against a list that mixes raw and formatted role strings: it holds raw org_admin but not the "Org Admin" that formatUserRole produces, and AuthContext stores the formatted form. A keyless org admin therefore read as a non-admin and was redirected to the connect page. Gate positively on internalUserRoles instead, which carries both representations, so the redirect targets the persona it is meant for and any role that is not unambiguously an internal user is left on the dashboard. The shared admin list is left alone: completing it would change org-admin access across every isAdminRole caller, which is a roles-policy decision of its own. --- ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx | 7 ++++--- ui/litellm-dashboard/src/app/(dashboard)/page.tsx | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx index 6fb64333089..affcca401f2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx @@ -67,14 +67,15 @@ describe("dashboard landing keyless redirect", () => { mockMigratedHref.mockClear(); }); - it("sends a keyless non-admin to the connect page after login", () => { + it.each(["Internal User", "Internal Viewer"])("sends a keyless %s to the connect page after login", (role) => { + state.userRole = role; render(); expect(mockReplace).toHaveBeenCalledWith("/mocked-ui/connect"); expect(screen.queryByTestId("api-keys-dashboard")).not.toBeInTheDocument(); }); - it("leaves an admin with no keys on the dashboard", () => { - state.userRole = "Admin"; + it.each(["Admin", "Admin Viewer", "Org Admin"])("leaves a keyless %s on the dashboard", (role) => { + state.userRole = role; render(); expect(mockReplace).not.toHaveBeenCalled(); expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx index 883aff8b36b..682ba0bbcb2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx @@ -4,7 +4,7 @@ import ApiKeysDashboard from "@/app/(dashboard)/api-keys/ApiKeysDashboard"; import LoadingScreen from "@/components/common_components/LoadingScreen"; import { proxyBaseUrl } from "@/components/networking"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; -import { isAdminRole } from "@/utils/roles"; +import { internalUserRoles } from "@/utils/roles"; import { useAuth } from "@/contexts/AuthContext"; import { buildLoginUrlWithReturn, @@ -93,7 +93,7 @@ function CreateKeyPageContent() { const isPostLoginLanding = searchParams.get("login") === "success"; const isSignedIn = !authLoading && Boolean(token); - const shouldCheckForKeys = isPostLoginLanding && isSignedIn && !isAdminRole(userRole); + const shouldCheckForKeys = isPostLoginLanding && isSignedIn && internalUserRoles.includes(userRole); const { data: keysData, isLoading: keysLoading } = useKeys(1, 1, { userID }, shouldCheckForKeys); const isKeylessLanding = shouldCheckForKeys && !keysLoading && keysData?.keys?.length === 0; const isResolvingKeylessLanding = (shouldCheckForKeys && keysLoading) || isKeylessLanding; From 6ca40e7dcd5e0e5f337067526ac82aef305e2ca9 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 10:41:15 -0700 Subject: [PATCH 07/49] refactor(mcp): delete unreachable v1 OBO handler and gate REST OAuth on v2 resolver The v2 credential resolver owns oauth2_token_exchange end to end: any server with a token-exchange config maps to a non-None TokenExchangeConfig spec, and that config is in _create_mcp_client's override-exclusion set, so a caller x-mcp-* override cannot force it back to v1 either. The v1 handler resolve_mcp_auth reached at spec is None was therefore dead for OBO, including its warn-then-proceed-unauthenticated fall-through. Delete auth/token_exchange.py and the exchange branch, dropping the subject_token parameter that only fed it. Separately, the REST listing and call paths still ran the v1 per-user OAuth lookup for servers the v2 resolver owns. Unlike the two protocol-path call sites they gated on auth_type == oauth2 only, with no to_server_spec check, so a migrated authorization_code server did a DB round-trip whose Authorization header _resolve_v2_auth then discards. Add the same guard via _is_v1_resolved_oauth2_server, shared by the per-server lookup and the prefetch preflight. Also collapses MCPOAuth2TokenCache.async_get_token's now single-caller require_client_credentials_flow kwarg and removes the dead _get_bulk_user_oauth_headers helper (zero callers). --- .../mcp_server/auth/token_exchange.py | 192 ------- .../mcp_server/mcp_server_manager.py | 4 +- .../mcp_server/oauth2_token_cache.py | 34 +- .../mcp_server/rest_endpoints.py | 65 +-- .../types/mcp_server/mcp_server_manager.py | 9 - .../mcp_server/auth/test_token_exchange.py | 539 ------------------ .../mcp_server/test_mcp_server_manager.py | 53 ++ .../mcp_server/test_mcp_tool_search.py | 4 +- .../mcp_server/test_rest_endpoints.py | 65 +++ 9 files changed, 151 insertions(+), 814 deletions(-) delete mode 100644 litellm/proxy/_experimental/mcp_server/auth/token_exchange.py delete mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py diff --git a/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py b/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py deleted file mode 100644 index cd41dd648ee..00000000000 --- a/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py +++ /dev/null @@ -1,192 +0,0 @@ -""" -OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers. - -Exchanges a user's incoming JWT (subject_token) for a scoped access token -at an IDP's token exchange endpoint. The exchanged token is then used to -authenticate requests to the upstream MCP server. - -See: https://datatracker.ietf.org/doc/html/rfc8693 -""" - -import asyncio -import hashlib -import weakref -from typing import TYPE_CHECKING, Dict, Tuple - -import httpx - -from litellm._logging import verbose_logger -from litellm.caching.in_memory_cache import InMemoryCache -from litellm.constants import ( - MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, - MCP_OAUTH2_TOKEN_CACHE_MIN_TTL, - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, - MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, -) -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( - build_token_endpoint_client_auth, -) -from litellm.types.llms.custom_http import httpxSpecialProvider -from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE - -if TYPE_CHECKING: - from litellm.types.mcp_server.mcp_server_manager import MCPServer - -# RFC 8693 grant type constant -TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" - - -class TokenExchangeHandler: - """Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers. - - Caches exchanged tokens keyed by ``hash(subject_token + server_id)`` so - repeated calls with the same user token skip the IDP round-trip. - """ - - def __init__(self) -> None: - self._cache = InMemoryCache( - max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, - default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, - ) - # WeakValueDictionary so locks are GC'd once no coroutine holds a reference, - # preventing unbounded growth with many rotating user tokens. - self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary() - - def _get_lock(self, cache_key: str) -> asyncio.Lock: - lock = self._locks.get(cache_key) - if lock is None: - lock = asyncio.Lock() - self._locks[cache_key] = lock - return lock - - @staticmethod - def _cache_key(subject_token: str, server_id: str) -> str: - raw = f"{subject_token}:{server_id}" - return hashlib.sha256(raw.encode()).hexdigest() - - async def exchange_token( - self, - subject_token: str, - server: "MCPServer", - ) -> str: - """Exchange *subject_token* for a scoped access token. - - Returns the exchanged ``access_token`` string (suitable for a - ``Bearer`` header). - - Raises ``ValueError`` on configuration or IDP errors. - """ - cache_key = self._cache_key(subject_token, server.server_id) - - # Fast path - cached = self._cache.get_cache(cache_key) - if cached is not None: - return cached - - # Slow path — one exchange at a time per (user, server) pair - async with self._get_lock(cache_key): - cached = self._cache.get_cache(cache_key) - if cached is not None: - return cached - - token, ttl = await self._do_exchange(subject_token, server) - self._cache.set_cache(cache_key, token, ttl=ttl) - return token - - async def _do_exchange( - self, - subject_token: str, - server: "MCPServer", - ) -> Tuple[str, int]: - """POST to the token exchange endpoint with RFC 8693 parameters. - - Returns ``(access_token, ttl_seconds)``. - """ - endpoint = server.token_exchange_endpoint or server.token_url - if not endpoint: - raise ValueError( - f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange " - f"but no token_exchange_endpoint or token_url configured" - ) - if not server.client_id or not server.client_secret: - raise ValueError( - f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange " - f"but missing client_id or client_secret" - ) - - client_auth = build_token_endpoint_client_auth( - auth_method=server.token_endpoint_auth_method, - client_id=server.client_id, - client_secret=server.client_secret, - ) - data: Dict[str, str] = { - "grant_type": TOKEN_EXCHANGE_GRANT_TYPE, - "subject_token": subject_token, - "subject_token_type": server.subject_token_type or DEFAULT_SUBJECT_TOKEN_TYPE, - **client_auth.body, - } - if server.audience: - data["audience"] = server.audience - if server.scopes: - data["scope"] = " ".join(server.scopes) - - verbose_logger.debug( - "Exchanging token for MCP server %s at %s (audience=%s)", - server.server_id, - endpoint, - server.audience, - ) - - client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) - post_kwargs = {"data": data, **({"headers": client_auth.headers} if client_auth.headers else {})} - try: - response = await client.post(endpoint, **post_kwargs) - response.raise_for_status() - except httpx.HTTPStatusError as exc: - verbose_logger.debug( - "Token exchange IDP error for MCP server %s (status %d)", - server.server_id, - exc.response.status_code, - ) - raise ValueError( - f"Token exchange for MCP server '{server.server_id}' failed with status {exc.response.status_code}" - ) from exc - - body = response.json() - if not isinstance(body, dict): - raise ValueError( - f"Token exchange response for MCP server '{server.server_id}' " - f"returned non-object JSON (got {type(body).__name__})" - ) - - access_token = body.get("access_token") - if not access_token: - raise ValueError(f"Token exchange response for MCP server '{server.server_id}' missing 'access_token'") - - raw_expires_in = body.get("expires_in") - try: - expires_in = int(raw_expires_in) if raw_expires_in is not None else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL - except (TypeError, ValueError): - expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL - - ttl = max( - expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, - MCP_OAUTH2_TOKEN_CACHE_MIN_TTL, - ) - - verbose_logger.info( - "Token exchange succeeded for MCP server %s (expires in %ds)", - server.server_id, - expires_in, - ) - return access_token, ttl - - def invalidate(self, subject_token: str, server_id: str) -> None: - """Remove a cached exchanged token (e.g. after a 401).""" - cache_key = self._cache_key(subject_token, server_id) - self._cache.delete_cache(cache_key) - - -# Module-level singleton -mcp_token_exchange_handler = TokenExchangeHandler() diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index b442ea5de70..0ee74960293 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -3086,9 +3086,7 @@ class MCPServerManager: ) ): spec = None - auth_value = ( - await resolve_mcp_auth(server, mcp_auth_header, subject_token=subject_token) if spec is None else None - ) + auth_value = await resolve_mcp_auth(server, mcp_auth_header) if spec is None else None # Create sampling and elicitation callbacks for this client sampling_cb = _create_sampling_callback(user_api_key_auth=user_api_key_auth) if server.allow_sampling else None diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index 43fe3999291..a6acaf8e1d6 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -26,7 +26,6 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) -from litellm.proxy._experimental.mcp_server.auth import token_exchange from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( build_token_endpoint_client_auth, ) @@ -58,17 +57,12 @@ class MCPOAuth2TokenCache(InMemoryCache): def _has_client_credentials_config(server: "MCPServer") -> bool: return bool(server.client_id and server.client_secret and server.token_url) - async def async_get_token( - self, - server: "MCPServer", - *, - require_client_credentials_flow: bool = True, - ) -> Optional[str]: + async def async_get_token(self, server: "MCPServer") -> Optional[str]: """Return a valid access token, fetching or refreshing as needed. Returns ``None`` when the server lacks client credentials config. """ - if require_client_credentials_flow and not server.has_client_credentials: + if not server.has_client_credentials: return None if not self._has_client_credentials_config(server): return None @@ -278,36 +272,16 @@ mcp_per_user_token_cache = MCPPerUserTokenCache() async def resolve_mcp_auth( server: "MCPServer", mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, - subject_token: Optional[str] = None, ) -> Optional[Union[str, Dict[str, str]]]: """Resolve the auth value for an MCP server. Priority: 1. ``mcp_auth_header`` — per-request/per-user override - 2. OAuth2 Token Exchange (OBO / RFC 8693) — exchange user token for scoped token - 3. OAuth2 client_credentials token — auto-fetched and cached - 4. ``server.authentication_token`` — static token from config/DB + 2. OAuth2 client_credentials token — auto-fetched and cached + 3. ``server.authentication_token`` — static token from config/DB """ if mcp_auth_header: return mcp_auth_header - if server.has_token_exchange_config: - if subject_token: - return await token_exchange.mcp_token_exchange_handler.exchange_token(subject_token, server) - # No subject_token — fall back to client_credentials using the same client - # credentials and token_url so M2M scenarios still work. - if server.client_id and server.client_secret and server.token_url: - return await mcp_oauth2_token_cache.async_get_token( - server, - require_client_credentials_flow=False, - ) - # OBO configured but no subject_token and missing client credentials — warn - # rather than silently proceeding unauthenticated. - verbose_logger.warning( - "MCP server '%s' is configured for token exchange (OBO) but no subject_token " - "was provided and client credentials (client_id/client_secret/token_url) are " - "incomplete. The request will proceed without authentication.", - server.server_id, - ) if server.has_client_credentials: return await mcp_oauth2_token_cache.async_get_token(server) return server.authentication_token diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 94271c54f4b..26e4176e09b 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -230,16 +230,33 @@ if MCP_AVAILABLE: return server_auth return mcp_auth_header - def _get_oauth2_server_ids(allowed_server_ids: List[str]) -> Set[str]: - """Return the subset of *allowed_server_ids* whose servers use OAuth2 auth. + def _is_v1_resolved_oauth2_server(server: Optional[MCPServer]) -> bool: + """Whether this server's per-user OAuth2 token is still resolved by v1. - Used as a cheap pre-flight check to skip bulk credential fetching when no - OAuth2 servers are involved in the current request. + A server the v2 resolver owns reads its stored token from the resolver at connect + time and drops any Authorization built for it here, so the v1 lookup would be a DB + round-trip whose result is discarded. Mirrors the same guard on the protocol listing + path and in ``_resolve_oauth2_headers_for_tool_call``. + """ + from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + to_server_spec, + ) + + if getattr(server, "auth_type", None) != MCPAuth.oauth2: + return False + return to_server_spec(server) is None + + def _v1_resolved_oauth2_server_ids(allowed_server_ids: List[str]) -> Set[str]: + """Return the subset of *allowed_server_ids* whose per-user OAuth2 token is still + resolved by v1. + + Used as a cheap pre-flight check to skip bulk credential fetching when no such + server is involved in the current request. """ return { sid for sid in allowed_server_ids - if getattr(global_mcp_server_manager.get_mcp_server_by_id(sid), "auth_type", None) == MCPAuth.oauth2 + if _is_v1_resolved_oauth2_server(global_mcp_server_manager.get_mcp_server_by_id(sid)) } async def _get_user_oauth_extra_headers( @@ -253,11 +270,13 @@ if MCP_AVAILABLE: the MCP server the same way the admin "Add MCP / Authorize and Fetch" flow does. Returns None for non-OAuth2 servers or when no credential is stored. + A server the v2 resolver owns is skipped; see ``_is_v1_resolved_oauth2_server``. + Args: prefetched_creds: Optional dict keyed by server_id with credential payloads. When provided, avoids a per-server DB round-trip. """ - if getattr(server, "auth_type", None) != MCPAuth.oauth2: + if not _is_v1_resolved_oauth2_server(server): return None user_id = getattr(user_api_key_dict, "user_id", None) server_id = getattr(server, "server_id", None) @@ -320,38 +339,6 @@ if MCP_AVAILABLE: verbose_logger.warning(f"_prefetch_user_oauth_creds: failed to prefetch for user={user_id}: {e}") return {} - async def _get_bulk_user_oauth_headers( - user_api_key_dict: UserAPIKeyAuth, - ) -> Dict[str, Dict[str, str]]: - """ - Fetch ALL OAuth2 credentials for the current user in a single DB query and - return a mapping of server_id → {"Authorization": "Bearer "}. - - This is the batch alternative to calling _get_user_oauth_extra_headers - per-server inside a loop (N+1 DB queries). - """ - user_id = getattr(user_api_key_dict, "user_id", None) - if not user_id: - return {} - try: - from litellm.proxy._experimental.mcp_server.db import ( - list_user_oauth_credentials, - ) - from litellm.proxy.utils import get_prisma_client_or_throw - - prisma_client = get_prisma_client_or_throw( - "Database not connected. Connect a database to use OAuth2 MCP tools." - ) - creds = await list_user_oauth_credentials(prisma_client, user_id) - return { - c["server_id"]: {"Authorization": f"Bearer {c['access_token']}"} - for c in creds - if c.get("access_token") and c.get("server_id") - } - except Exception: - verbose_logger.debug("Failed to bulk-fetch OAuth credentials", exc_info=True) - return {} - def _create_tool_response_objects(tools, server: MCPServer): """Helper function to create tool response objects. @@ -825,7 +812,7 @@ if MCP_AVAILABLE: # to avoid an unnecessary DB round-trip on requests with no OAuth2 MCP servers. prefetched_oauth_creds = ( await _prefetch_user_oauth_creds(user_api_key_dict) - if _get_oauth2_server_ids(allowed_server_ids) + if _v1_resolved_oauth2_server_ids(allowed_server_ids) else {} ) diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 8ae974b19a6..b0af22e7c3f 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -261,12 +261,3 @@ class MCPServer(BaseModel): if self.oauth_passthrough is not True: return False return any(h.lower() == "authorization" for h in self.extra_headers) - - @property - def has_token_exchange_config(self) -> bool: - """True if this server is configured for OAuth2 token exchange (OBO / RFC 8693).""" - return ( - self.auth_type == MCPAuth.oauth2_token_exchange - and bool(self.client_id and self.client_secret) - and bool(self.token_exchange_endpoint or self.token_url) - ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py deleted file mode 100644 index d2aa58e29ea..00000000000 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py +++ /dev/null @@ -1,539 +0,0 @@ -""" -Tests for OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers. - -Covers: exchange flow, caching, error handling, resolve_mcp_auth integration, -bearer token extraction, and config loading. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest - -from litellm.proxy._experimental.mcp_server.auth.token_exchange import ( - TOKEN_EXCHANGE_GRANT_TYPE, - TokenExchangeHandler, -) -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - MCPServerManager, -) -from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( - resolve_mcp_auth, -) -from litellm.proxy._types import LiteLLM_MCPServerTable, MCPTransport -from litellm.types.mcp import MCPAuth -from litellm.types.mcp_server.mcp_server_manager import MCPServer - - -def _obo_server(**overrides) -> MCPServer: - defaults = dict( - server_id="srv-obo-1", - name="test-obo", - url="https://mcp.example.com/mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2_token_exchange, - client_id="litellm-client-id", - client_secret="litellm-client-secret", - token_exchange_endpoint="https://idp.example.com/oauth2/token", - audience="api://mcp-server", - scopes=["mcp.tools.read", "mcp.tools.execute"], - ) - defaults.update(overrides) - return MCPServer(**defaults) - - -def _exchange_response(token="exchanged-tok-abc", expires_in=3600): - resp = MagicMock() - resp.json.return_value = { - "access_token": token, - "token_type": "Bearer", - "expires_in": expires_in, - } - resp.raise_for_status = MagicMock() - resp.text = "" - return resp - - -# ── Exchange Flow ── - - -@pytest.mark.asyncio -async def test_exchange_token_success(): - """Token exchange sends correct RFC 8693 parameters and returns access_token.""" - handler = TokenExchangeHandler() - server = _obo_server() - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response("scoped-token-1") - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ): - result = await handler.exchange_token("user-jwt-xyz", server) - - assert result == "scoped-token-1" - mock_client.post.assert_called_once() - - _, kwargs = mock_client.post.call_args - data = kwargs["data"] - assert data["grant_type"] == TOKEN_EXCHANGE_GRANT_TYPE - assert data["subject_token"] == "user-jwt-xyz" - assert data["subject_token_type"] == "urn:ietf:params:oauth:token-type:access_token" - assert data["audience"] == "api://mcp-server" - assert data["scope"] == "mcp.tools.read mcp.tools.execute" - assert data["client_id"] == "litellm-client-id" - assert data["client_secret"] == "litellm-client-secret" - - -@pytest.mark.asyncio -async def test_exchange_token_no_audience(): - """When audience is None, it is omitted from the request.""" - handler = TokenExchangeHandler() - server = _obo_server(audience=None) - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response() - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ): - await handler.exchange_token("user-jwt", server) - - _, kwargs = mock_client.post.call_args - assert "audience" not in kwargs["data"] - - -@pytest.mark.asyncio -async def test_exchange_token_no_scopes(): - """When scopes is None, scope param is omitted from the request.""" - handler = TokenExchangeHandler() - server = _obo_server(scopes=None) - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response() - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ): - await handler.exchange_token("user-jwt", server) - - _, kwargs = mock_client.post.call_args - assert "scope" not in kwargs["data"] - - -# ── Caching ── - - -@pytest.mark.asyncio -async def test_exchange_token_cached(): - """Second call with same user token uses cache — only 1 HTTP POST.""" - handler = TokenExchangeHandler() - server = _obo_server() - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response("cached-exchange-tok") - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ): - t1 = await handler.exchange_token("same-jwt", server) - t2 = await handler.exchange_token("same-jwt", server) - - assert t1 == t2 == "cached-exchange-tok" - assert mock_client.post.call_count == 1 - - -@pytest.mark.asyncio -async def test_different_user_tokens_not_shared(): - """Different user JWTs get different exchanged tokens.""" - handler = TokenExchangeHandler() - server = _obo_server() - call_count = 0 - - async def mock_post(url, data=None): - nonlocal call_count - call_count += 1 - resp = MagicMock() - resp.json.return_value = { - "access_token": f"exchanged-{call_count}", - "expires_in": 3600, - } - resp.raise_for_status = MagicMock() - return resp - - mock_client = AsyncMock() - mock_client.post = mock_post - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ): - t1 = await handler.exchange_token("user-a-jwt", server) - t2 = await handler.exchange_token("user-b-jwt", server) - - assert t1 == "exchanged-1" - assert t2 == "exchanged-2" - assert call_count == 2 - - -# ── Error Handling ── - - -@pytest.mark.asyncio -async def test_exchange_token_http_error(): - """HTTP errors from the IDP are wrapped in a ValueError.""" - handler = TokenExchangeHandler() - server = _obo_server() - mock_response = MagicMock() - mock_response.status_code = 400 - mock_response.text = "invalid_grant" - mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( - "Bad Request", - request=MagicMock(), - response=mock_response, - ) - mock_client = AsyncMock() - mock_client.post.return_value = mock_response - - with ( - patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ), - pytest.raises(ValueError, match="failed with status 400"), - ): - await handler.exchange_token("bad-jwt", server) - - -@pytest.mark.asyncio -async def test_exchange_token_http_error_does_not_log_response_body(): - """Raw IDP error bodies are not logged because they can contain credentials.""" - handler = TokenExchangeHandler() - server = _obo_server() - raw_response_body = "client_secret=do-not-log" - mock_response = MagicMock() - mock_response.status_code = 401 - mock_response.text = raw_response_body - mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( - "Unauthorized", - request=MagicMock(), - response=mock_response, - ) - mock_client = AsyncMock() - mock_client.post.return_value = mock_response - - with ( - patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ), - patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.verbose_logger.debug" - ) as mock_debug, - pytest.raises(ValueError, match="failed with status 401"), - ): - await handler.exchange_token("bad-jwt", server) - - logged_values = " ".join( - str(value) - for call in mock_debug.call_args_list - for value in [*call.args, *call.kwargs.values()] - ) - assert raw_response_body not in logged_values - - -@pytest.mark.asyncio -async def test_exchange_token_missing_access_token(): - """Response without access_token raises ValueError.""" - handler = TokenExchangeHandler() - server = _obo_server() - resp = MagicMock() - resp.json.return_value = {"token_type": "Bearer"} - resp.raise_for_status = MagicMock() - mock_client = AsyncMock() - mock_client.post.return_value = resp - - with ( - patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ), - pytest.raises(ValueError, match="missing 'access_token'"), - ): - await handler.exchange_token("jwt", server) - - -@pytest.mark.asyncio -async def test_exchange_token_missing_endpoint(): - """Missing token_exchange_endpoint and token_url raises ValueError.""" - handler = TokenExchangeHandler() - server = _obo_server(token_exchange_endpoint=None, token_url=None) - - with pytest.raises(ValueError, match="no token_exchange_endpoint or token_url"): - await handler.exchange_token("jwt", server) - - -@pytest.mark.asyncio -async def test_exchange_token_missing_credentials(): - """Missing client_id or client_secret raises ValueError.""" - handler = TokenExchangeHandler() - server = _obo_server(client_id=None, client_secret=None) - # has_token_exchange_config will be False, so we call _do_exchange directly - with pytest.raises(ValueError, match="missing client_id or client_secret"): - await handler._do_exchange("jwt", server) - - -# ── resolve_mcp_auth Integration ── - - -@pytest.mark.asyncio -async def test_resolve_mcp_auth_with_token_exchange(): - """resolve_mcp_auth delegates to token exchange when server has OBO config and subject_token provided.""" - server = _obo_server() - mock_handler = AsyncMock() - mock_handler.exchange_token.return_value = "obo-scoped-token" - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.mcp_token_exchange_handler", - mock_handler, - ): - result = await resolve_mcp_auth(server, subject_token="user-jwt") - - assert result == "obo-scoped-token" - mock_handler.exchange_token.assert_called_once_with("user-jwt", server) - - -@pytest.mark.asyncio -async def test_resolve_mcp_auth_obo_without_subject_token_falls_through(): - """Without a subject_token, resolve_mcp_auth falls through to client_credentials.""" - server = _obo_server( - token_url="https://auth.example.com/token", - ) - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response("cc-token") - - with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", - return_value=mock_client, - ): - result = await resolve_mcp_auth(server, subject_token=None) - - # Falls through to client_credentials since subject_token is None - # The server has client_id/client_secret/token_url so has_client_credentials is True - assert result == "cc-token" - - -@pytest.mark.asyncio -async def test_resolve_mcp_auth_obo_without_subject_token_uses_cached_client_credentials(): - """The M2M fallback for OBO servers reuses the client_credentials cache.""" - server = _obo_server( - server_id="srv-obo-m2m-cache", - token_url="https://auth.example.com/token", - ) - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response("cached-cc-token") - - with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", - return_value=mock_client, - ): - first = await resolve_mcp_auth(server, subject_token=None) - second = await resolve_mcp_auth(server, subject_token=None) - - assert first == second == "cached-cc-token" - mock_client.post.assert_called_once() - - -@pytest.mark.asyncio -async def test_resolve_mcp_auth_header_beats_obo(): - """An explicit mcp_auth_header takes priority over OBO token exchange.""" - server = _obo_server() - result = await resolve_mcp_auth( - server, mcp_auth_header="Bearer override", subject_token="user-jwt" - ) - assert result == "Bearer override" - - -# ── Bearer Token Extraction ── - - -def test_extract_bearer_token_from_oauth2_headers(): - """Extracts token from oauth2_headers Authorization header.""" - result = MCPServerManager._extract_bearer_token( - oauth2_headers={"Authorization": "Bearer my-jwt-token"}, - raw_headers=None, - ) - assert result == "my-jwt-token" - - -def test_extract_bearer_token_from_raw_headers(): - """Falls back to raw_headers when oauth2_headers missing.""" - result = MCPServerManager._extract_bearer_token( - oauth2_headers=None, - raw_headers={"authorization": "Bearer raw-jwt"}, - ) - assert result == "raw-jwt" - - -def test_extract_bearer_token_no_bearer_prefix(): - """Returns token as-is when no Bearer prefix.""" - result = MCPServerManager._extract_bearer_token( - oauth2_headers={"Authorization": "some-opaque-token"}, - raw_headers=None, - ) - assert result == "some-opaque-token" - - -def test_extract_bearer_token_none(): - """Returns None when no auth headers present.""" - result = MCPServerManager._extract_bearer_token( - oauth2_headers=None, - raw_headers=None, - ) - assert result is None - - -# ── MCPServer Properties ── - - -def test_has_token_exchange_config_true(): - """has_token_exchange_config is True for a fully configured OBO server.""" - server = _obo_server() - assert server.has_token_exchange_config is True - - -def test_has_token_exchange_config_false_wrong_auth_type(): - """has_token_exchange_config is False when auth_type is not oauth2_token_exchange.""" - server = _obo_server(auth_type=MCPAuth.oauth2) - assert server.has_token_exchange_config is False - - -def test_has_token_exchange_config_false_missing_creds(): - """has_token_exchange_config is False when client_id/client_secret missing.""" - server = _obo_server(client_id=None) - assert server.has_token_exchange_config is False - - -def test_has_token_exchange_config_uses_token_url_fallback(): - """has_token_exchange_config is True when token_url is set instead of token_exchange_endpoint.""" - server = _obo_server( - token_exchange_endpoint=None, - token_url="https://idp.example.com/token", - ) - assert server.has_token_exchange_config is True - - -# ── Config Loading ── - - -@pytest.mark.asyncio -async def test_config_loading_token_exchange_fields(): - """load_servers_from_config correctly maps OBO config fields to MCPServer.""" - manager = MCPServerManager() - config = { - "my_obo_server": { - "url": "https://mcp.example.com/mcp", - "transport": "http", - "auth_type": "oauth2_token_exchange", - "client_id": "my-client", - "client_secret": "my-secret", - "token_exchange_endpoint": "https://idp.example.com/oauth2/token", - "audience": "api://my-mcp", - "scopes": ["read", "write"], - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - } - } - await manager.load_servers_from_config(config) - - servers = list(manager.config_mcp_servers.values()) - assert len(servers) == 1 - - server = servers[0] - assert server.auth_type == MCPAuth.oauth2_token_exchange - assert server.token_exchange_endpoint == "https://idp.example.com/oauth2/token" - assert server.audience == "api://my-mcp" - assert server.subject_token_type == "urn:ietf:params:oauth:token-type:jwt" - assert server.client_id == "my-client" - assert server.client_secret == "my-secret" - assert server.scopes == ["read", "write"] - assert server.has_token_exchange_config is True - - -@pytest.mark.asyncio -async def test_config_loading_default_subject_token_type(): - """subject_token_type defaults to access_token when not specified in config.""" - manager = MCPServerManager() - config = { - "obo_defaults": { - "url": "https://mcp.example.com/mcp", - "transport": "http", - "auth_type": "oauth2_token_exchange", - "client_id": "cid", - "client_secret": "csec", - "token_exchange_endpoint": "https://idp.example.com/token", - } - } - await manager.load_servers_from_config(config) - - server = list(manager.config_mcp_servers.values())[0] - assert server.subject_token_type == "urn:ietf:params:oauth:token-type:access_token" - - -@pytest.mark.asyncio -async def test_database_loading_token_exchange_scopes_from_credentials(): - """DB-loaded OBO server credentials retain configured scopes.""" - manager = MCPServerManager() - db_server = LiteLLM_MCPServerTable( - server_id="srv-obo-db", - server_name="obo_db_server", - url="https://mcp.example.com/mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2_token_exchange, - credentials={ - "client_id": "db-client", - "client_secret": "db-secret", - "token_exchange_endpoint": "https://idp.example.com/oauth2/token", - "audience": "api://db-mcp", - "scopes": ["db.read", "db.write"], - }, - ) - - server = await manager.build_mcp_server_from_table( - db_server, - credentials_are_encrypted=False, - ) - - assert server.auth_type == MCPAuth.oauth2_token_exchange - assert server.client_id == "db-client" - assert server.client_secret == "db-secret" - assert server.token_exchange_endpoint == "https://idp.example.com/oauth2/token" - assert server.audience == "api://db-mcp" - assert server.scopes == ["db.read", "db.write"] - - -@pytest.mark.asyncio -async def test_exchange_token_uses_client_secret_basic_when_configured(): - """LIT-4091: token exchange with token_endpoint_auth_method=client_secret_basic sends the - client credentials as HTTP Basic and omits client_secret from the body.""" - import base64 - - handler = TokenExchangeHandler() - server = _obo_server( - server_id="srv-obo-basic", token_endpoint_auth_method="client_secret_basic" - ) - mock_client = AsyncMock() - mock_client.post.return_value = _exchange_response("scoped-basic") - - with patch( - "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", - return_value=mock_client, - ): - result = await handler.exchange_token("user-jwt-basic", server) - - assert result == "scoped-basic" - _, kwargs = mock_client.post.call_args - expected = "Basic " + base64.b64encode(b"litellm-client-id:litellm-client-secret").decode() - assert kwargs["headers"]["Authorization"] == expected - assert "client_secret" not in kwargs["data"] - assert "client_id" not in kwargs["data"] - assert kwargs["data"]["grant_type"] == TOKEN_EXCHANGE_GRANT_TYPE diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 77f072b81d5..42b6cbee1c4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -2333,6 +2333,59 @@ class TestMCPServerManager: assert emitted.headers["Authorization"] == "Bearer upstream-token" assert not kwargs["extra_headers"] or "authorization" not in {k.lower() for k in kwargs["extra_headers"]} + @pytest.mark.asyncio + async def test_create_mcp_client_token_exchange_never_falls_back_to_v1(self): + """A configured OBO server is owned end to end by the v2 token_exchange arm, even when the + caller supplies an x-mcp-* override. This is what makes the v1 OBO handler unreachable, so if + it ever defers to v1 again the deleted handler is silently needed back.""" + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( + OAuthToken, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import ( + UpstreamCredentialProvider, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + + class _StubExchanger: + def __init__(self): + self.subject_tokens = [] + + async def exchange(self, subject_token, server, config, *, tenant_id=""): + self.subject_tokens.append(subject_token) + return Ok(OAuthToken(access_token="exchanged-token")) + + async def invalidate(self, subject_token, server, config, *, tenant_id=""): + return None + + exchanger = _StubExchanger() + manager = MCPServerManager() + server = MCPServer( + server_id="obo-egress", + name="obo", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + client_id="gateway-client", + client_secret="gateway-secret", + token_exchange_endpoint="https://idp.example.com/oauth2/token", + ) + with ( + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.resolve_mcp_auth", + new_callable=AsyncMock, + ) as mock_resolve, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as mock_client_cls, + ): + await manager._create_mcp_client( + server=server, + mcp_auth_header="Bearer caller-override", + subject_token="eyJ-subject-token", + cred_provider=UpstreamCredentialProvider(token_exchanger=exchanger), + ) + mock_resolve.assert_not_awaited() + assert exchanger.subject_tokens == ["eyJ-subject-token"] + assert self._emitted_authorization(mock_client_cls) == "Bearer exchanged-token" + @staticmethod def _emitted_authorization(mock_client_cls) -> str: kwargs = mock_client_cls.call_args.kwargs diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index 1da44029b5c..0e442102e53 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -237,7 +237,7 @@ class TestListToolRestApiWithToolSearch: return_value={}, ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_oauth2_server_ids", + "litellm.proxy._experimental.mcp_server.rest_endpoints._v1_resolved_oauth2_server_ids", return_value=[], ), patch( @@ -316,7 +316,7 @@ class TestListToolRestApiWithToolSearch: return_value={}, ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_oauth2_server_ids", + "litellm.proxy._experimental.mcp_server.rest_endpoints._v1_resolved_oauth2_server_ids", return_value=[], ), patch( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index d4ba66c4381..5c9612a055e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -2783,3 +2783,68 @@ class TestRestListToolsetFiltering: ) assert [tool.name for tool in result] == ["lookup_status"] + + +class TestV1ResolvedOauth2Gate: + """The REST surface must stop resolving per-user OAuth2 tokens for servers the v2 resolver owns. + + ``_resolve_v2_auth`` drops any Authorization built here for an ``authorization_code`` server and + injects the resolver's own token, so the v1 lookup was a DB round-trip whose result was discarded. + A server that still defers to v1 (upstream-delegated oauth2) must keep resolving, which is what + makes these assertions non-vacuous. + """ + + @staticmethod + def _oauth2_server(*, delegate_auth_to_upstream: bool) -> Any: + from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.types.mcp import MCPTransport + + return MCPServer( + server_id="oauth2-srv", + name="oauth2-srv", + url="https://upstream.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=delegate_auth_to_upstream, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "delegate_auth_to_upstream, expected_headers, expected_lookups", + [ + (False, None, 0), + (True, {"Authorization": "Bearer stored-token"}, 1), + ], + ) + async def test_user_oauth_headers_skip_v2_owned_servers( + self, delegate_auth_to_upstream, expected_headers, expected_lookups, monkeypatch + ): + from litellm.proxy._experimental.mcp_server import db as mcp_db + + server = self._oauth2_server(delegate_auth_to_upstream=delegate_auth_to_upstream) + resolve_token = AsyncMock(return_value={"access_token": "stored-token"}) + monkeypatch.setattr(mcp_db, "resolve_valid_user_oauth_token", resolve_token) + + headers = await rest_endpoints._get_user_oauth_extra_headers( + server, + UserAPIKeyAuth(user_id="alice", api_key="sk-1234"), + prefetched_creds={"oauth2-srv": {"access_token": "stored-token"}}, + ) + + assert headers == expected_headers + assert resolve_token.await_count == expected_lookups + + def test_prefetch_preflight_only_counts_v1_resolved_servers(self, monkeypatch): + v2_owned = self._oauth2_server(delegate_auth_to_upstream=False) + v1_resolved = self._oauth2_server(delegate_auth_to_upstream=True) + v1_resolved.server_id = "delegate-srv" + registry = {"oauth2-srv": v2_owned, "delegate-srv": v1_resolved} + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "get_mcp_server_by_id", + lambda server_id: registry.get(server_id), + ) + + assert rest_endpoints._v1_resolved_oauth2_server_ids(["oauth2-srv"]) == set() + assert rest_endpoints._v1_resolved_oauth2_server_ids(["oauth2-srv", "delegate-srv"]) == {"delegate-srv"} From bd73ca8c64338fcd58f4e73c1987623786d3ee01 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 10:37:21 -0700 Subject: [PATCH 08/49] feat(cost-optimization): add spend-by-tool and cache leakage views Adds GET /v1/tool/spend returning per-tool and daily tool spend with a deduplicated request total, and a cache leakage breakdown on the Prompt Caching tab of the Cost Optimization page. Tool-spend rows are validated at the boundary with pydantic, the endpoint is scoped to proxy admins, date params are cast to timestamptz for real-Postgres query_raw, and the leakage math treats litellm-normalized prompt_tokens as cache-inclusive (uncached = max(0, prompt - cache_read - cache_creation)). --- litellm/proxy/_lazy_openapi_snapshot.json | 189 ++++++++++++++++++ .../tool_management_endpoints.py | 152 +++++++++++++- litellm/types/tool_management.py | 35 ++++ .../test_tool_management_endpoints.py | 125 +++++++++++- .../_components/CacheLeakageCard.test.tsx | 83 ++++++++ .../_components/CacheLeakageCard.tsx | 112 +++++++++++ .../CostOptimizationView.activity.test.tsx | 53 +++++ .../_components/CostOptimizationView.tsx | 7 +- .../_components/PromptCachingTab.test.tsx | 43 ++++ .../_components/PromptCachingTab.tsx | 8 +- .../_components/UsageTab.test.tsx | 50 ++++- .../_components/UsageTab.tsx | 126 +++++++++--- .../_components/costOptimizationUtils.test.ts | 156 +++++++++++++++ .../_components/costOptimizationUtils.ts | 123 ++++++++++++ .../useDailyActivityRange.test.tsx | 39 ++++ .../_components/useDailyActivityRange.ts | 49 +++++ .../src/components/networking.tsx | 32 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 121 +++++++++++ 18 files changed, 1456 insertions(+), 47 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 67d935e34e3..7e8d08e7cad 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26762,6 +26762,113 @@ "title": "ToolPolicyUpdateResponse", "type": "object" }, + "ToolSpendDailyEntry": { + "description": "Spend attributed to one tool on one UTC day.", + "properties": { + "call_count": { + "default": 0, + "title": "Call Count", + "type": "integer" + }, + "date": { + "title": "Date", + "type": "string" + }, + "spend": { + "default": 0.0, + "title": "Spend", + "type": "number" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "date", + "tool_name" + ], + "title": "ToolSpendDailyEntry", + "type": "object" + }, + "ToolSpendEntry": { + "description": "Total spend attributed to one tool over the requested window.", + "properties": { + "call_count": { + "default": 0, + "title": "Call Count", + "type": "integer" + }, + "spend": { + "default": 0.0, + "description": "Attributed spend: a request that used several tools counts its full spend toward each of them", + "title": "Spend", + "type": "number" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + }, + "total_tokens": { + "default": 0, + "title": "Total Tokens", + "type": "integer" + } + }, + "required": [ + "tool_name" + ], + "title": "ToolSpendEntry", + "type": "object" + }, + "ToolSpendResponse": { + "properties": { + "by_tool": { + "items": { + "$ref": "#/components/schemas/ToolSpendEntry" + }, + "title": "By Tool", + "type": "array" + }, + "daily": { + "items": { + "$ref": "#/components/schemas/ToolSpendDailyEntry" + }, + "title": "Daily", + "type": "array" + }, + "end_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "End Date" + }, + "start_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start Date" + }, + "total_spend": { + "default": 0.0, + "description": "Deduplicated spend of every request that called at least one tool in the window; less than the sum of per-tool attributed spend whenever multi-tool requests exist", + "title": "Total Spend", + "type": "number" + } + }, + "title": "ToolSpendResponse", + "type": "object" + }, "ToolUsageLogEntry": { "description": "One spend log row for a tool call (for UI \"recent logs\" table).", "properties": { @@ -26858,6 +26965,13 @@ }, "ValidationError": { "properties": { + "ctx": { + "title": "Context", + "type": "object" + }, + "input": { + "title": "Input" + }, "loc": { "items": { "anyOf": [ @@ -27301,6 +27415,81 @@ ] } }, + "/v1/tool/spend": { + "get": { + "description": "Spend attributed to each tool over a date range, for the Cost Optimization dashboard.\n\nJoins ``LiteLLM_SpendLogToolIndex`` (which tool names ran on which request) to\n``LiteLLM_SpendLogs`` (what the request cost). A request that used multiple tools\ncounts its full spend toward each of those tools, so per-tool numbers are\nattributions. ``total_spend`` is the deduplicated spend of every request that\ncalled at least one tool in the window, so it never double counts.", + "operationId": "get_tool_spend_v1_tool_spend_get", + "parameters": [ + { + "description": "YYYY-MM-DD (defaults to 30 days ago)", + "in": "query", + "name": "start_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD (defaults to 30 days ago)", + "title": "Start Date" + } + }, + { + "description": "YYYY-MM-DD (defaults to today)", + "in": "query", + "name": "end_date", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD (defaults to today)", + "title": "End Date" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolSpendResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Tool Spend", + "tags": [ + "tools" + ] + } + }, "/v1/tool/{tool_name}": { "get": { "description": "Get details for a single tool.", diff --git a/litellm/proxy/management_endpoints/tool_management_endpoints.py b/litellm/proxy/management_endpoints/tool_management_endpoints.py index 9d71761f115..ca606e07cee 100644 --- a/litellm/proxy/management_endpoints/tool_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tool_management_endpoints.py @@ -10,16 +10,18 @@ POST /v1/tool/policy - Update the input_policy / output_policy for a """ import uuid -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, List, Optional +from datetime import datetime, timedelta, timezone +from itertools import groupby +from typing import TYPE_CHECKING, Annotated, Any, List, Optional from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, TypeAdapter if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.table_repositories import ( @@ -39,6 +41,9 @@ from litellm.types.tool_management import ( ToolPolicyOptionsResponse, ToolPolicyUpdateRequest, ToolPolicyUpdateResponse, + ToolSpendDailyEntry, + ToolSpendEntry, + ToolSpendResponse, ToolUsageLogEntry, ToolUsageLogsResponse, ) @@ -124,6 +129,147 @@ async def list_tools( raise HTTPException(status_code=500, detail=str(e)) +def _parse_day_start(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.strptime(value.strip(), "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Invalid date format: {value}. Expected: 'YYYY-MM-DD'", + ) + + +class _ToolSpendRow(BaseModel): + date: str + tool_name: str + call_count: int + spend: float + total_tokens: int + + +class _RequestTotalRow(BaseModel): + total_spend: float + + +_TOOL_SPEND_ROWS = TypeAdapter(list[_ToolSpendRow]) +_REQUEST_TOTAL_ROWS = TypeAdapter(list[_RequestTotalRow]) + + +def _summarize_tool(name: str, grp: tuple[_ToolSpendRow, ...]) -> ToolSpendEntry: + return ToolSpendEntry( + tool_name=name, + spend=sum(r.spend for r in grp), + call_count=sum(r.call_count for r in grp), + total_tokens=sum(r.total_tokens for r in grp), + ) + + +def _build_tool_spend_response( + rows: list[_ToolSpendRow], + total_spend: float, + start_date: str, + end_date: str, +) -> ToolSpendResponse: + daily = [ + ToolSpendDailyEntry(date=r.date, tool_name=r.tool_name, spend=r.spend, call_count=r.call_count) for r in rows + ] + grouped = groupby(sorted(rows, key=lambda r: r.tool_name), key=lambda r: r.tool_name) + by_tool = sorted( + (_summarize_tool(name, tuple(grp)) for name, grp in grouped), + key=lambda e: e.spend, + reverse=True, + ) + return ToolSpendResponse( + by_tool=by_tool, + daily=daily, + total_spend=total_spend, + start_date=start_date, + end_date=end_date, + ) + + +@router.get( + "/v1/tool/spend", + tags=["tool management"], + dependencies=[Depends(user_api_key_auth)], + response_model=ToolSpendResponse, +) +async def get_tool_spend( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[str | None, Query(description="YYYY-MM-DD (defaults to 30 days ago)")] = None, + end_date: Annotated[str | None, Query(description="YYYY-MM-DD (defaults to today)")] = None, +): + """ + Spend attributed to each tool over a date range, for the Cost Optimization dashboard. + + Joins ``LiteLLM_SpendLogToolIndex`` (which tool names ran on which request) to + ``LiteLLM_SpendLogs`` (what the request cost). A request that used multiple tools + counts its full spend toward each of those tools, so per-tool numbers are + attributions. ``total_spend`` is the deduplicated spend of every request that + called at least one tool in the window, so it never double counts. + """ + from litellm.proxy.proxy_server import prisma_client + + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise HTTPException( + status_code=403, + detail="Only proxy admin roles can view tool spend across the deployment", + ) + + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + + now = datetime.now(timezone.utc) + end_day = _parse_day_start(end_date) + start_dt = _parse_day_start(start_date) or ((end_day or now) - timedelta(days=30)) + end_exclusive = (end_day + timedelta(days=1)) if end_day else now + + rows = await prisma_client.db.query_raw( + """ + SELECT to_char(ti.start_time, 'YYYY-MM-DD') AS date, + ti.tool_name AS tool_name, + COUNT(*)::int AS call_count, + COALESCE(SUM(sl.spend), 0)::double precision AS spend, + COALESCE(SUM(sl.total_tokens), 0)::bigint AS total_tokens + FROM "LiteLLM_SpendLogToolIndex" ti + JOIN "LiteLLM_SpendLogs" sl ON sl.request_id = ti.request_id + WHERE ti.start_time >= ($1::timestamptz AT TIME ZONE 'UTC') + AND ti.start_time < ($2::timestamptz AT TIME ZONE 'UTC') + GROUP BY date, ti.tool_name + ORDER BY date ASC, spend DESC + """, + start_dt.isoformat(), + end_exclusive.isoformat(), + ) + totals = await prisma_client.db.query_raw( + """ + SELECT COALESCE(SUM(sl.spend), 0)::double precision AS total_spend + FROM "LiteLLM_SpendLogs" sl + WHERE EXISTS ( + SELECT 1 + FROM "LiteLLM_SpendLogToolIndex" ti + WHERE ti.request_id = sl.request_id + AND ti.start_time >= ($1::timestamptz AT TIME ZONE 'UTC') + AND ti.start_time < ($2::timestamptz AT TIME ZONE 'UTC') + ) + """, + start_dt.isoformat(), + end_exclusive.isoformat(), + ) + total_rows = _REQUEST_TOTAL_ROWS.validate_python(totals or []) + return _build_tool_spend_response( + rows=_TOOL_SPEND_ROWS.validate_python(rows or []), + total_spend=total_rows[0].total_spend if total_rows else 0.0, + start_date=start_dt.strftime("%Y-%m-%d"), + end_date=(end_day or now).strftime("%Y-%m-%d"), + ) + + @router.get( "/v1/tool/{tool_name:path}/detail", tags=["tool management"], diff --git a/litellm/types/tool_management.py b/litellm/types/tool_management.py index 1c5e1df9e9a..71ec412e8ef 100644 --- a/litellm/types/tool_management.py +++ b/litellm/types/tool_management.py @@ -98,3 +98,38 @@ class ToolUsageLogsResponse(BaseModel): total: int page: int page_size: int + + +class ToolSpendEntry(BaseModel): + """Total spend attributed to one tool over the requested window.""" + + tool_name: str + spend: float = Field( + 0.0, + description="Attributed spend: a request that used several tools counts its full spend toward each of them", + ) + call_count: int = 0 + total_tokens: int = 0 + + +class ToolSpendDailyEntry(BaseModel): + """Spend attributed to one tool on one UTC day.""" + + date: str + tool_name: str + spend: float = 0.0 + call_count: int = 0 + + +class ToolSpendResponse(BaseModel): + by_tool: List[ToolSpendEntry] = Field(default_factory=list) + daily: List[ToolSpendDailyEntry] = Field(default_factory=list) + total_spend: float = Field( + 0.0, + description=( + "Deduplicated spend of every request that called at least one tool in the window; " + "less than the sum of per-tool attributed spend whenever multi-tool requests exist" + ), + ) + start_date: str | None = None + end_date: str | None = None diff --git a/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py index cf80ee5dee5..351f125052d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py @@ -13,12 +13,17 @@ from datetime import datetime, timezone from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch +import pytest from fastapi import FastAPI from fastapi.testclient import TestClient sys.path.insert(0, os.path.abspath("../../..")) -from litellm.proxy.management_endpoints.tool_management_endpoints import router +from litellm.proxy.management_endpoints.tool_management_endpoints import ( + _build_tool_spend_response, + _ToolSpendRow, + router, +) from litellm.types.tool_management import LiteLLM_ToolTableRow # --- helpers --- @@ -50,9 +55,9 @@ def _make_app() -> FastAPI: # Stub the auth dependency so we don't need a real proxy running. def _override_auth(): - from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - return UserAPIKeyAuth(api_key="sk-test", user_id="admin") + return UserAPIKeyAuth(api_key="sk-test", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) # A real (non-None) prisma stub for truthiness checks. @@ -147,3 +152,117 @@ class TestToolManagementEndpoints: json={"tool_name": "my_tool", "input_policy": "invalid_value"}, ) assert resp.status_code == 422 + + def test_tool_spend_route_not_shadowed_by_get_tool(self): + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + resp = self.client.get("/v1/tool/spend") + assert resp.status_code == 200 + assert resp.json()["by_tool"] == [] + + def test_tool_spend_aggregates_and_sorts(self): + rows = [ + {"date": "2026-07-01", "tool_name": "search", "call_count": 2, "spend": 1.0, "total_tokens": 100}, + {"date": "2026-07-02", "tool_name": "search", "call_count": 1, "spend": 4.0, "total_tokens": 50}, + {"date": "2026-07-01", "tool_name": "read_file", "call_count": 3, "spend": 2.0, "total_tokens": 300}, + ] + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(side_effect=[rows, [{"total_spend": 5.5}]]) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + resp = self.client.get("/v1/tool/spend?start_date=2026-07-01&end_date=2026-07-02") + assert resp.status_code == 200 + body = resp.json() + assert [t["tool_name"] for t in body["by_tool"]] == ["search", "read_file"] + search = body["by_tool"][0] + assert search["spend"] == 5.0 + assert search["call_count"] == 3 + assert search["total_tokens"] == 150 + assert len(body["daily"]) == 3 + assert body["start_date"] == "2026-07-01" + assert body["end_date"] == "2026-07-02" + assert body["total_spend"] == 5.5 + + @patch("litellm.proxy.proxy_server.prisma_client", None) + def test_tool_spend_no_db_returns_500(self): + resp = self.client.get("/v1/tool/spend") + assert resp.status_code == 500 + + def test_tool_spend_end_date_is_inclusive_via_exclusive_next_day_bound(self): + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + resp = self.client.get("/v1/tool/spend?start_date=2026-07-01&end_date=2026-07-02") + assert resp.status_code == 200 + expected_binds = ( + datetime(2026, 7, 1, tzinfo=timezone.utc).isoformat(), + datetime(2026, 7, 3, tzinfo=timezone.utc).isoformat(), + ) + assert prisma.db.query_raw.await_count == 2 + for call in prisma.db.query_raw.await_args_list: + assert tuple(call.args[1:]) == expected_binds + assert resp.json()["end_date"] == "2026-07-02" + + @pytest.mark.parametrize( + "query", + [ + "start_date=not-a-date", + "start_date=2026-02-30", + "start_date=07/01/2026", + "end_date=2026-13-01", + "end_date=20260701", + ], + ) + def test_tool_spend_malformed_date_returns_400(self, query: str): + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + resp = self.client.get(f"/v1/tool/spend?{query}") + assert resp.status_code == 400 + assert "Invalid date format" in resp.json()["detail"] + prisma.db.query_raw.assert_not_awaited() + + def test_tool_spend_non_admin_returns_403(self): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + app = _make_app() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="sk-user", user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER + ) + client = TestClient(app, raise_server_exceptions=True) + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + resp = client.get("/v1/tool/spend") + assert resp.status_code == 403 + prisma.db.query_raw.assert_not_awaited() + + +def _spend_row(date: str, tool_name: str, spend: float, call_count: int = 1, total_tokens: int = 10) -> _ToolSpendRow: + return _ToolSpendRow(date=date, tool_name=tool_name, call_count=call_count, spend=spend, total_tokens=total_tokens) + + +class TestBuildToolSpendResponse: + def test_multi_tool_attribution_double_counts_per_tool_but_not_total(self): + rows = [ + _spend_row("2026-07-01", "a", spend=3.0), + _spend_row("2026-07-01", "b", spend=3.0), + ] + resp = _build_tool_spend_response(rows, total_spend=3.0, start_date="2026-07-01", end_date="2026-07-01") + by_tool = {t.tool_name: t.spend for t in resp.by_tool} + assert by_tool == {"a": 3.0, "b": 3.0} + assert resp.total_spend == 3.0 + + def test_groups_across_days_and_sorts_by_spend(self): + rows = [ + _spend_row("2026-07-01", "b", spend=1.0, call_count=2, total_tokens=100), + _spend_row("2026-07-02", "b", spend=4.0, call_count=1, total_tokens=50), + _spend_row("2026-07-01", "a", spend=2.0, call_count=3, total_tokens=300), + ] + resp = _build_tool_spend_response(rows, total_spend=7.0, start_date="2026-07-01", end_date="2026-07-02") + assert [(t.tool_name, t.spend, t.call_count, t.total_tokens) for t in resp.by_tool] == [ + ("b", 5.0, 3, 150), + ("a", 2.0, 3, 300), + ] + assert len(resp.daily) == 3 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx new file mode 100644 index 00000000000..1d82fbb48ea --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx @@ -0,0 +1,83 @@ +import { render } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { DailyData, KeyMetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; + +vi.mock("@/components/shared/advanced_date_picker", () => ({ + __esModule: true, + default: () =>
, +})); + +import CacheLeakageCard from "./CacheLeakageCard"; + +const baseMetrics = (overrides: Partial): SpendMetrics => ({ + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + ...overrides, +}); + +const key = (alias: string, metrics: Partial): KeyMetricWithMetadata => ({ + metrics: baseMetrics(metrics), + metadata: { key_alias: alias, team_id: null }, +}); + +const dayWithKeys = (date: string, apiKeys: Record): DailyData => ({ + date, + metrics: baseMetrics({}), + breakdown: { + models: {}, + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: apiKeys, + entities: {}, + }, +}); + +const renderWith = (results: DailyData[]) => + render( + , + ); + +describe("CacheLeakageCard", () => { + it("ranks leaking keys by uncached prompt tokens and shows cache hit ratio", () => { + const { getByText, getByLabelText } = renderWith([ + dayWithKeys("2026-07-12", { + "hash-caching": key("caching-key", { prompt_tokens: 1000, cache_read_input_tokens: 900 }), + "hash-leaky": key("leaky-key", { prompt_tokens: 10000, cache_read_input_tokens: 0 }), + }), + ]); + + expect(getByText("leaky-key")).toBeInTheDocument(); + expect(getByText("0.0%")).toBeInTheDocument(); + expect(getByText("90.0%")).toBeInTheDocument(); + [ + "Input tokens in the selected range that were neither read from nor written to the prompt cache", + "Share of this key's total input tokens that were served from the prompt cache", + "Dollars this key actually saved because cached input was billed at the discounted cache-read rate", + "Approximate dollars this key could still save if its uncached input had hit the cache at the portfolio's realized discount", + ].forEach((info) => expect(getByLabelText(info)).toBeInTheDocument()); + }); + + it("shows an empty state when no key used tokens in the range", () => { + const { getByText, queryByRole } = renderWith([dayWithKeys("2026-07-12", {})]); + + expect(getByText("No key usage in this range.")).toBeInTheDocument(); + expect(queryByRole("table")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx new file mode 100644 index 00000000000..359ce502b07 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -0,0 +1,112 @@ +"use client"; + +import React, { useMemo } from "react"; +import { Info } from "lucide-react"; + +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; +import { DailyActivityRange } from "./useDailyActivityRange"; + +interface CacheLeakageCardProps { + activity: DailyActivityRange; +} + +const HeadWithInfo = ({ label, info }: { label: string; info: string }) => ( + + {label} + + + + + + + {info} + + +); + +const CacheLeakageCard: React.FC = ({ activity }) => { + const { dateValue, onDateChange, results, loading, isFetchingMore } = activity; + const leakage = useMemo(() => computeCacheLeakage(results), [results]); + + return ( + + + +
+
+ Cache leakage by virtual key +

+ Keys sending large volumes of uncached prompt tokens with a low cache-hit ratio are likely missing + prompt caching. Estimated savings left is approximate: uncached prompt tokens priced at the + portfolio's realized cache-read discount. +

+
+ +
+
+ + {leakage.rows.length === 0 ? ( +

+ {loading || isFetchingMore ? "Loading..." : "No key usage in this range."} +

+ ) : ( + + + + Key + + + + + + + + + + + + + + + + {leakage.rows.map((row) => ( + + + {row.keyAlias || `${row.apiKey.slice(0, 8)}...`} + {row.teamId && ({row.teamId})} + + {formatNumberWithCommas(row.uncachedPromptTokens)} + {pct(row.cacheHitRatio)} + {usd(row.realizedCachingSavings)} + + {row.estSavingsLeft == null ? "—" : usd(row.estSavingsLeft)} + + + ))} + +
+ )} +
+
+
+ ); +}; + +export default CacheLeakageCard; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx new file mode 100644 index 00000000000..97289a7ca46 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -0,0 +1,53 @@ +import { fireEvent, render, waitFor } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +const mockUserDailyActivityCall = vi.fn(); + +vi.mock("@/components/networking", () => ({ + userDailyActivityCall: (...args: unknown[]) => mockUserDailyActivityCall(...args), + getToolSpend: vi.fn().mockResolvedValue({ by_tool: [], daily: [], total_spend: 0, start_date: null, end_date: null }), + getGeneralSettingsCall: vi.fn().mockResolvedValue([]), +})); + +vi.mock("@/components/shared/advanced_date_picker", () => ({ + __esModule: true, + default: () =>
, +})); + +vi.mock("@/components/shared/charts", () => ({ + AreaChart: () =>
, + DonutChart: () =>
, + BarChart: () =>
, + DEFAULT_COLOR_CYCLE: ["emerald"], +})); + +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ + PromptCachingPanel: () =>
, +})); + +vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () =>
})); + +import CostOptimizationView from "./CostOptimizationView"; + +const singlePage = { + results: [], + metadata: { total_pages: 1, has_more: false, page: 1 }, +}; + +describe("CostOptimizationView daily activity", () => { + it("fetches daily activity once for the page and shares it with every tab that needs it", async () => { + mockUserDailyActivityCall.mockResolvedValue(singlePage); + + const { getByRole, getByTestId } = render( + , + ); + + await waitFor(() => expect(mockUserDailyActivityCall).toHaveBeenCalledTimes(1)); + + fireEvent.click(getByRole("tab", { name: "Prompt Caching" })); + await waitFor(() => expect(getByTestId("caching-settings")).toBeInTheDocument()); + + expect(mockUserDailyActivityCall).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index 6e6830b8451..3bab6afee57 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -8,6 +8,7 @@ import UsageTab from "./UsageTab"; import PromptCompressionTab from "./PromptCompressionTab"; import AutorouterTab from "./AutorouterTab"; import PromptCachingTab from "./PromptCachingTab"; +import { useDailyActivityRange } from "./useDailyActivityRange"; interface CostOptimizationViewProps { accessToken: string | null; @@ -16,11 +17,13 @@ interface CostOptimizationViewProps { } const CostOptimizationView: React.FC = ({ accessToken, userId, userRole }) => { + const activity = useDailyActivityRange(accessToken, userId, userRole); + const items = [ { key: "usage", label: "Usage", - children: , + children: , }, { key: "compression", @@ -35,7 +38,7 @@ const CostOptimizationView: React.FC = ({ accessToken { key: "caching", label: "Prompt Caching", - children: , + children: , }, ]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx new file mode 100644 index 00000000000..a18109e8133 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx @@ -0,0 +1,43 @@ +import { render, waitFor } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +const mockGetGeneralSettingsCall = vi.fn(); + +vi.mock("@/components/networking", () => ({ + getGeneralSettingsCall: (...args: unknown[]) => mockGetGeneralSettingsCall(...args), +})); + +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ + PromptCachingPanel: () =>
, +})); + +const mockCacheLeakageCard = vi.fn(); + +vi.mock("./CacheLeakageCard", () => ({ + __esModule: true, + default: (props: unknown) => { + mockCacheLeakageCard(props); + return
; + }, +})); + +import PromptCachingTab from "./PromptCachingTab"; + +describe("PromptCachingTab", () => { + it("renders the cache leakage table alongside the caching settings", async () => { + mockGetGeneralSettingsCall.mockResolvedValue([]); + + const activity = { + dateValue: {}, + onDateChange: vi.fn(), + results: [], + loading: false, + isFetchingMore: false, + }; + const { getByTestId } = render(); + + expect(getByTestId("caching-settings")).toBeInTheDocument(); + expect(getByTestId("cache-leakage-card")).toBeInTheDocument(); + await waitFor(() => expect(mockCacheLeakageCard).toHaveBeenCalledWith(expect.objectContaining({ activity }))); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx index e6f73088824..952e9f653ac 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx @@ -8,12 +8,15 @@ import { PromptCachingPanel, generalSettingsItem, } from "@/app/(dashboard)/router-settings/_components/general_settings"; +import CacheLeakageCard from "./CacheLeakageCard"; +import { DailyActivityRange } from "./useDailyActivityRange"; interface PromptCachingTabProps { accessToken: string | null; + activity: DailyActivityRange; } -const PromptCachingTab: React.FC = ({ accessToken }) => { +const PromptCachingTab: React.FC = ({ accessToken, activity }) => { const [settings, setSettings] = useState([]); const loadSettings = useCallback(() => { @@ -43,8 +46,9 @@ const PromptCachingTab: React.FC = ({ accessToken }) => { } return ( -
+
+
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 048d9a34d31..0e2f16c5d93 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -1,16 +1,13 @@ import { render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; +import type { ToolSpendResponse } from "@/components/networking"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; -const mockUsePaginatedDailyActivity = vi.fn(); - -vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ - usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args), -})); +const mockGetToolSpend = vi.fn(); vi.mock("@/components/networking", () => ({ - userDailyActivityCall: vi.fn(), + getToolSpend: (...args: unknown[]) => mockGetToolSpend(...args), })); vi.mock("@/components/shared/advanced_date_picker", () => ({ @@ -25,10 +22,16 @@ vi.mock("@/components/shared/charts", () => ({ DonutChart: ({ data, label }: { data: unknown; label: string }) => (
), + BarChart: ({ data, categories }: { data: unknown; categories: string[] }) => ( +
+ ), + DEFAULT_COLOR_CYCLE: ["emerald", "blue", "violet", "amber"], })); import UsageTab from "./UsageTab"; +const emptyToolSpend: ToolSpendResponse = { by_tool: [], daily: [], total_spend: 0, start_date: null, end_date: null }; + const baseMetrics = (overrides: Partial): SpendMetrics => ({ spend: 0, prompt_tokens: 0, @@ -55,9 +58,20 @@ const day = (date: string, metrics: Partial): DailyData => ({ }, }); -const renderWith = (results: DailyData[]) => { - mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false }); - return render(); +const renderWith = (results: DailyData[], toolSpend = emptyToolSpend) => { + mockGetToolSpend.mockResolvedValue(toolSpend); + return render( + , + ); }; describe("UsageTab", () => { @@ -105,4 +119,22 @@ describe("UsageTab", () => { const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); }); + + it("renders spend-by-tool bars from the tool spend endpoint", async () => { + const toolSpend = { + by_tool: [ + { tool_name: "search", spend: 4.0, call_count: 3, total_tokens: 150 }, + { tool_name: "read_file", spend: 1.0, call_count: 2, total_tokens: 50 }, + ], + daily: [{ date: "2026-07-12", tool_name: "search", spend: 4.0, call_count: 3 }], + total_spend: 5.0, + start_date: "2026-07-12", + end_date: "2026-07-12", + }; + const { findAllByTestId } = renderWith([day("2026-07-12", {})], toolSpend); + + const bars = await findAllByTestId("bar-chart"); + const series = JSON.parse(bars[0].getAttribute("data-series") ?? "[]"); + expect(series[0]).toMatchObject({ tool_name: "search", spend: 4.0 }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index 8e6fc40b5ad..9216dbfaad2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -1,35 +1,35 @@ "use client"; -import React, { useMemo, useState } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { Collapse } from "antd"; -import { AreaChart, DonutChart } from "@/components/shared/charts"; +import { AreaChart, BarChart, DonutChart, DEFAULT_COLOR_CYCLE } from "@/components/shared/charts"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { userDailyActivityCall } from "@/components/networking"; -import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { getToolSpend, ToolSpendResponse } from "@/components/networking"; +import { SpendMetrics } from "@/components/UsagePage/types"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { all_admin_roles } from "@/utils/roles"; -import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; +import { buildDailyToolSeries, topToolsBySpend, usd } from "./costOptimizationUtils"; +import { DailyActivityRange } from "./useDailyActivityRange"; interface UsageTabProps { accessToken: string | null; - userId: string | null; - userRole: string; + activity: DailyActivityRange; } -type DateRange = { from?: Date; to?: Date }; - -const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; - -const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; +const EMPTY_TOOL_SPEND: ToolSpendResponse = { + by_tool: [], + daily: [], + total_spend: 0, + start_date: null, + end_date: null, }; const shortDate = (iso: string): string => new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); +const isoDay = (d: Date): string => d.toISOString().slice(0, 10); + const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; @@ -81,23 +81,33 @@ const SummaryCard = ({ label, value, hint }: { label: string; value: string; hin ); -const UsageTab: React.FC = ({ accessToken, userId, userRole }) => { - const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); - const initialTo = useMemo(() => new Date(), []); - const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); +const UsageTab: React.FC = ({ accessToken, activity }) => { + const { dateValue, onDateChange, results, loading, isFetchingMore } = activity; const startTime = dateValue.from ?? null; const endTime = dateValue.to ?? null; - const isAdmin = all_admin_roles.includes(userRole); - const effectiveUserId = isAdmin ? null : userId; - const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ - fetchFn: userDailyActivityCall, - args: [accessToken, startTime, endTime, effectiveUserId], - enabled: !!accessToken && !!startTime && !!endTime, - }); + const toolSpendEnabled = !!accessToken && !!startTime && !!endTime; + const rangeKey = startTime && endTime ? `${isoDay(startTime)}|${isoDay(endTime)}` : ""; + const [toolSpendState, setToolSpendState] = useState<{ key: string; data: ToolSpendResponse } | null>(null); - const results = data.results as DailyData[]; + useEffect(() => { + if (!accessToken || !startTime || !endTime) return; + let cancelled = false; + getToolSpend(accessToken, isoDay(startTime), isoDay(endTime)) + .then((res) => { + if (!cancelled) setToolSpendState({ key: rangeKey, data: res }); + }) + .catch(() => { + if (!cancelled) setToolSpendState({ key: rangeKey, data: EMPTY_TOOL_SPEND }); + }); + return () => { + cancelled = true; + }; + }, [accessToken, startTime, endTime, rangeKey]); + + const toolSpend = toolSpendState?.key === rangeKey ? toolSpendState.data : null; + const toolSpendLoading = toolSpendEnabled && toolSpend === null; const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); @@ -123,11 +133,27 @@ const UsageTab: React.FC = ({ accessToken, userId, userRole }) => [compressionTotal, cachingTotal], ); + const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]); + const topToolNames = useMemo(() => topTools.map((t) => t.tool_name), [topTools]); + const topToolsChart = useMemo[]>( + () => topTools.map((t) => ({ tool_name: t.tool_name, spend: t.spend })), + [topTools], + ); + const dailyToolSeries = useMemo( + () => + buildDailyToolSeries(toolSpend?.daily ?? [], topToolNames).map((point) => ({ + ...point, + date: shortDate(String(point.date)), + })), + [toolSpend, topToolNames], + ); + const toolColors = useMemo(() => DEFAULT_COLOR_CYCLE.slice(0, Math.max(topToolNames.length, 1)), [topToolNames]); + return (
- setDateValue(v)} /> +
@@ -177,6 +203,50 @@ const UsageTab: React.FC = ({ accessToken, userId, userRole }) =>
+ + + + Spend by tool +

+ Spend on requests that called each tool (MCP and client-side tools). A request that used multiple tools + counts its full spend toward each, so this attributes rather than partitions spend. +

+
+ + {topTools.length === 0 ? ( +

+ {toolSpendLoading ? "Loading..." : "No tool usage in this range."} +

+ ) : ( +
+
+

Total by tool

+ +
+
+

Daily spend by tool

+ +
+
+ )} +
+
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts new file mode 100644 index 00000000000..562552ffb50 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest"; + +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import type { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; +import { buildDailyToolSeries, computeCacheLeakage, topToolsBySpend } from "./costOptimizationUtils"; + +const metrics = (overrides: Partial): SpendMetrics => ({ + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + ...overrides, +}); + +const day = ( + date: string, + keys: Record }>, +): DailyData => ({ + date, + metrics: metrics({}), + breakdown: { + models: {}, + model_groups: {}, + mcp_servers: {}, + providers: {}, + entities: {}, + api_keys: Object.fromEntries( + Object.entries(keys).map(([hash, v]) => [ + hash, + { metrics: metrics(v.metrics), metadata: { key_alias: v.alias, team_id: null } }, + ]), + ), + }, +}); + +describe("computeCacheLeakage", () => { + it("aggregates a key's tokens and savings across multiple days", () => { + const results = [ + day("2026-07-01", { h1: { alias: "svc-a", metrics: { prompt_tokens: 1000, cache_read_input_tokens: 0 } } }), + day("2026-07-02", { h1: { alias: "svc-a", metrics: { prompt_tokens: 500, cache_read_input_tokens: 0 } } }), + ]; + const { rows } = computeCacheLeakage(results); + expect(rows).toHaveLength(1); + expect(rows[0].uncachedPromptTokens).toBe(1500); + }); + + it("subtracts cache reads and writes from prompt tokens instead of double-counting them", () => { + const results = [ + day("2026-07-01", { + h1: { + alias: "svc-a", + metrics: { prompt_tokens: 1000, cache_read_input_tokens: 400, cache_creation_input_tokens: 100 }, + }, + }), + ]; + const { rows } = computeCacheLeakage(results); + expect(rows).toHaveLength(1); + expect(rows[0].uncachedPromptTokens).toBe(500); + expect(rows[0].cacheHitRatio).toBeCloseTo(0.4, 6); + }); + + it("prices leakage at the portfolio's realized cache-read discount and drops fully cached keys", () => { + const results = [ + day("2026-07-01", { + cacher: { + alias: "cacher", + metrics: { prompt_tokens: 1000, cache_read_input_tokens: 1000, prompt_caching_savings_spend: 2.0 }, + }, + leaker: { alias: "leaker", metrics: { prompt_tokens: 500 } }, + }), + ]; + const { rows, discountPerToken } = computeCacheLeakage(results); + expect(discountPerToken).toBeCloseTo(0.002, 6); + expect(rows.map((r) => r.keyAlias)).toEqual(["leaker"]); + expect(rows[0].estSavingsLeft).toBeCloseTo(1.0, 6); + }); + + it("returns null estimate and ranks by uncached tokens when nobody used caching", () => { + const results = [ + day("2026-07-01", { + big: { alias: "big", metrics: { prompt_tokens: 9000 } }, + small: { alias: "small", metrics: { prompt_tokens: 100 } }, + }), + ]; + const { rows, discountPerToken } = computeCacheLeakage(results); + expect(discountPerToken).toBeNull(); + expect(rows.map((r) => r.keyAlias)).toEqual(["big", "small"]); + expect(rows.every((r) => r.estSavingsLeft === null)).toBe(true); + }); + + it("computes cache hit ratio against total prompt tokens and clamps inconsistent data at zero", () => { + const results = [ + day("2026-07-01", { + onlycache: { alias: "onlycache", metrics: { cache_read_input_tokens: 100 } }, + mixed: { alias: "mixed", metrics: { prompt_tokens: 1000, cache_read_input_tokens: 750 } }, + }), + ]; + const { rows } = computeCacheLeakage(results); + expect(rows.map((r) => r.keyAlias)).toEqual(["mixed"]); + expect(rows[0].cacheHitRatio).toBeCloseTo(0.75, 6); + expect(rows[0].uncachedPromptTokens).toBe(250); + }); + + it("respects the row limit", () => { + const keys = Object.fromEntries( + Array.from({ length: 15 }, (_, i) => [`h${i}`, { alias: `k${i}`, metrics: { prompt_tokens: i + 1 } }]), + ); + const { rows } = computeCacheLeakage([day("2026-07-01", keys)], 5); + expect(rows).toHaveLength(5); + }); +}); + +describe("buildDailyToolSeries", () => { + const daily: ToolSpendDailyEntry[] = [ + { date: "2026-07-01", tool_name: "search", spend: 1.0, call_count: 1 }, + { date: "2026-07-01", tool_name: "read", spend: 0.5, call_count: 1 }, + { date: "2026-07-02", tool_name: "search", spend: 2.0, call_count: 1 }, + { date: "2026-07-01", tool_name: "excluded", spend: 9.0, call_count: 1 }, + ]; + + it("pivots to per-date points keyed by the selected tools, dropping others", () => { + const series = buildDailyToolSeries(daily, ["search", "read"]); + expect(series).toEqual([ + { date: "2026-07-01", search: 1.0, read: 0.5 }, + { date: "2026-07-02", search: 2.0, read: 0 }, + ]); + }); + + it("sums repeated (date, tool) rows", () => { + const series = buildDailyToolSeries( + [ + { date: "2026-07-01", tool_name: "search", spend: 1.0, call_count: 1 }, + { date: "2026-07-01", tool_name: "search", spend: 2.5, call_count: 1 }, + ], + ["search"], + ); + expect(series[0].search).toBe(3.5); + }); +}); + +describe("topToolsBySpend", () => { + const byTool: ToolSpendEntry[] = [ + { tool_name: "a", spend: 1, call_count: 1, total_tokens: 1 }, + { tool_name: "b", spend: 5, call_count: 1, total_tokens: 1 }, + { tool_name: "c", spend: 3, call_count: 1, total_tokens: 1 }, + ]; + + it("sorts by spend descending and truncates to the limit", () => { + expect(topToolsBySpend(byTool, 2).map((t) => t.tool_name)).toEqual(["b", "c"]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts new file mode 100644 index 00000000000..2868bdac880 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -0,0 +1,123 @@ +import { DailyData } from "@/components/UsagePage/types"; +import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; + +export const usd = (value: number): string => { + const decimals = value > 0 && value < 1 ? 4 : 2; + return `$${formatNumberWithCommas(value, decimals)}`; +}; + +export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; + +export interface CacheLeakageRow { + apiKey: string; + keyAlias: string | null; + teamId: string | null; + uncachedPromptTokens: number; + cacheReadTokens: number; + cacheHitRatio: number; + realizedCachingSavings: number; + estSavingsLeft: number | null; +} + +export interface CacheLeakageResult { + rows: CacheLeakageRow[]; + discountPerToken: number | null; +} + +interface KeyAccumulator { + keyAlias: string | null; + teamId: string | null; + promptTokens: number; + cacheReadTokens: number; + cacheCreationTokens: number; + realizedCachingSavings: number; +} + +const emptyAccumulator = (): KeyAccumulator => ({ + keyAlias: null, + teamId: null, + promptTokens: 0, + cacheReadTokens: 0, + cacheCreationTokens: 0, + realizedCachingSavings: 0, +}); + +export const computeCacheLeakage = (results: readonly DailyData[], limit = 10): CacheLeakageResult => { + const byKey = new Map(); + for (const day of results) { + const apiKeys = day.breakdown?.api_keys ?? {}; + for (const [apiKey, entry] of Object.entries(apiKeys)) { + const acc = byKey.get(apiKey) ?? emptyAccumulator(); + const m = entry.metrics; + const next: KeyAccumulator = { + keyAlias: acc.keyAlias ?? entry.metadata?.key_alias ?? null, + teamId: acc.teamId ?? entry.metadata?.team_id ?? null, + promptTokens: acc.promptTokens + (m.prompt_tokens ?? 0), + cacheReadTokens: acc.cacheReadTokens + (m.cache_read_input_tokens ?? 0), + cacheCreationTokens: acc.cacheCreationTokens + (m.cache_creation_input_tokens ?? 0), + realizedCachingSavings: acc.realizedCachingSavings + (m.prompt_caching_savings_spend ?? 0), + }; + byKey.set(apiKey, next); + } + } + + const totals = [...byKey.values()].reduce( + (agg, a) => ({ + cacheReadTokens: agg.cacheReadTokens + a.cacheReadTokens, + realizedCachingSavings: agg.realizedCachingSavings + a.realizedCachingSavings, + }), + { cacheReadTokens: 0, realizedCachingSavings: 0 }, + ); + const discountPerToken = totals.cacheReadTokens > 0 ? totals.realizedCachingSavings / totals.cacheReadTokens : null; + + const rows: CacheLeakageRow[] = [...byKey.entries()] + .map(([apiKey, a]) => { + const uncachedPromptTokens = Math.max(0, a.promptTokens - a.cacheReadTokens - a.cacheCreationTokens); + return { + apiKey, + keyAlias: a.keyAlias, + teamId: a.teamId, + uncachedPromptTokens, + cacheReadTokens: a.cacheReadTokens, + cacheHitRatio: a.promptTokens > 0 ? a.cacheReadTokens / a.promptTokens : 0, + realizedCachingSavings: a.realizedCachingSavings, + estSavingsLeft: discountPerToken != null ? uncachedPromptTokens * discountPerToken : null, + }; + }) + .filter((row) => row.uncachedPromptTokens > 0); + + const sorted = rows.sort((x, y) => + discountPerToken != null + ? (y.estSavingsLeft ?? 0) - (x.estSavingsLeft ?? 0) + : y.uncachedPromptTokens - x.uncachedPromptTokens, + ); + + return { rows: sorted.slice(0, limit), discountPerToken }; +}; + +export interface DailyToolSpendPoint { + date: string; + [toolName: string]: string | number; +} + +export const buildDailyToolSeries = ( + daily: readonly ToolSpendDailyEntry[], + topToolNames: readonly string[], +): DailyToolSpendPoint[] => { + const top = new Set(topToolNames); + const byDate = new Map(); + for (const d of daily) { + if (!top.has(d.tool_name)) continue; + const point = byDate.get(d.date) ?? seedPoint(d.date, topToolNames); + point[d.tool_name] = (Number(point[d.tool_name]) || 0) + d.spend; + byDate.set(d.date, point); + } + return [...byDate.values()].sort((a, b) => a.date.localeCompare(b.date)); +}; + +const seedPoint = (date: string, toolNames: readonly string[]): DailyToolSpendPoint => + toolNames.reduce((p, name) => ({ ...p, [name]: 0 }), { date }); + +export const topToolsBySpend = (byTool: readonly ToolSpendEntry[], limit = 8): ToolSpendEntry[] => + [...byTool].sort((a, b) => b.spend - a.spend).slice(0, limit); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx new file mode 100644 index 00000000000..9fd27d80c37 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx @@ -0,0 +1,39 @@ +import { renderHook } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +const mockUsePaginatedDailyActivity = vi.fn(); + +vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ + usePaginatedDailyActivity: (args: unknown) => { + mockUsePaginatedDailyActivity(args); + return { data: { results: [] }, loading: false, isFetchingMore: false }; + }, +})); + +vi.mock("@/components/networking", () => ({ + userDailyActivityCall: vi.fn(), +})); + +import { useDailyActivityRange } from "./useDailyActivityRange"; + +const argsOfLastCall = () => mockUsePaginatedDailyActivity.mock.calls.at(-1)?.[0].args as unknown[]; + +describe("useDailyActivityRange", () => { + it("queries every user's activity for an admin", () => { + renderHook(() => useDailyActivityRange("test-token", "u1", "proxy_admin")); + + expect(argsOfLastCall()).toEqual(["test-token", expect.any(Date), expect.any(Date), null]); + }); + + it("scopes the query to the caller for a non-admin", () => { + renderHook(() => useDailyActivityRange("test-token", "u1", "internal_user")); + + expect(argsOfLastCall()).toEqual(["test-token", expect.any(Date), expect.any(Date), "u1"]); + }); + + it("stays disabled until an access token is available", () => { + renderHook(() => useDailyActivityRange(null, "u1", "proxy_admin")); + + expect(mockUsePaginatedDailyActivity).toHaveBeenLastCalledWith(expect.objectContaining({ enabled: false })); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts new file mode 100644 index 00000000000..1c3f706726e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts @@ -0,0 +1,49 @@ +import { useMemo, useState } from "react"; + +import { userDailyActivityCall } from "@/components/networking"; +import { DailyData } from "@/components/UsagePage/types"; +import { all_admin_roles } from "@/utils/roles"; +import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; + +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; + +export interface DateRange { + from?: Date; + to?: Date; +} + +export interface DailyActivityRange { + dateValue: DateRange; + onDateChange: (value: DateRange) => void; + results: DailyData[]; + loading: boolean; + isFetchingMore: boolean; +} + +export const useDailyActivityRange = ( + accessToken: string | null, + userId: string | null, + userRole: string, +): DailyActivityRange => { + const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); + const initialTo = useMemo(() => new Date(), []); + const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); + + const startTime = dateValue.from ?? null; + const endTime = dateValue.to ?? null; + const effectiveUserId = all_admin_roles.includes(userRole) ? null : userId; + + const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ + fetchFn: userDailyActivityCall, + args: [accessToken, startTime, endTime, effectiveUserId], + enabled: !!accessToken && !!startTime && !!endTime, + }); + + return { + dateValue, + onDateChange: setDateValue, + results: data.results as DailyData[], + loading, + isFetchingMore, + }; +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index f5fe3990ba7..9dcb4321445 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -7582,6 +7582,38 @@ export const fetchToolsList = async (accessToken: string): Promise => return data.tools ?? []; }; +export interface ToolSpendEntry { + tool_name: string; + spend: number; + call_count: number; + total_tokens: number; +} + +export interface ToolSpendDailyEntry { + date: string; + tool_name: string; + spend: number; + call_count: number; +} + +export interface ToolSpendResponse { + by_tool: ToolSpendEntry[]; + daily: ToolSpendDailyEntry[]; + total_spend: number; + start_date: string | null; + end_date: string | null; +} + +export const getToolSpend = async ( + accessToken: string, + startDate?: string, + endDate?: string, +): Promise => + apiClient.get(`/v1/tool/spend`, { + accessToken, + query: { start_date: startDate, end_date: endDate }, + }); + export interface ToolPolicyOverrideRow { override_id: string; tool_name: string; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e0223aa4af6..825d06d6a38 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17900,6 +17900,32 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/tool/spend": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Tool Spend + * @description Spend attributed to each tool over a date range, for the Cost Optimization dashboard. + * + * Joins ``LiteLLM_SpendLogToolIndex`` (which tool names ran on which request) to + * ``LiteLLM_SpendLogs`` (what the request cost). A request that used multiple tools + * counts its full spend toward each of those tools, so per-tool numbers are + * attributions. ``total_spend`` is the deduplicated spend of every request that + * called at least one tool in the window, so it never double counts. + */ + get: operations["get_tool_spend_v1_tool_spend_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/tool/{tool_name}": { parameters: { query?: never; @@ -31978,6 +32004,67 @@ export interface components { /** Updated */ updated: boolean; }; + /** + * ToolSpendDailyEntry + * @description Spend attributed to one tool on one UTC day. + */ + ToolSpendDailyEntry: { + /** + * Call Count + * @default 0 + */ + call_count: number; + /** Date */ + date: string; + /** + * Spend + * @default 0 + */ + spend: number; + /** Tool Name */ + tool_name: string; + }; + /** + * ToolSpendEntry + * @description Total spend attributed to one tool over the requested window. + */ + ToolSpendEntry: { + /** + * Call Count + * @default 0 + */ + call_count: number; + /** + * Spend + * @description Attributed spend: a request that used several tools counts its full spend toward each of them + * @default 0 + */ + spend: number; + /** Tool Name */ + tool_name: string; + /** + * Total Tokens + * @default 0 + */ + total_tokens: number; + }; + /** ToolSpendResponse */ + ToolSpendResponse: { + /** By Tool */ + by_tool?: components["schemas"]["ToolSpendEntry"][]; + /** Daily */ + daily?: components["schemas"]["ToolSpendDailyEntry"][]; + /** End Date */ + end_date?: string | null; + /** Start Date */ + start_date?: string | null; + /** + * Total Spend + * @description Deduplicated spend of every request that called at least one tool in the window; less than the sum of per-tool attributed spend whenever multi-tool requests exist + * @default 0 + */ + total_spend: number; + }; /** * ToolUsageLogEntry * @description One spend log row for a tool call (for UI "recent logs" table). @@ -56176,6 +56263,40 @@ export interface operations { }; }; }; + get_tool_spend_v1_tool_spend_get: { + parameters: { + query?: { + /** @description YYYY-MM-DD (defaults to 30 days ago) */ + start_date?: string | null; + /** @description YYYY-MM-DD (defaults to today) */ + end_date?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ToolSpendResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_tool_v1_tool__tool_name__get: { parameters: { query?: never; From 531854db4eb8ad04e50330ef5c6bdc5389aea9d1 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 10:53:59 -0700 Subject: [PATCH 09/49] fix(ui): hold the landing until the role hydrates before deciding the redirect AuthContext sets token and clears authLoading in one effect, then a second token-keyed effect populates userRole, so there is a render where the user is signed in but userRole is still the initial empty string. The positive internalUserRoles check reads that interim role as non-internal, which let the api-keys dashboard paint for a frame before the role arrived and the keyless redirect ran. Treat "signed in on the post-login landing with an unhydrated role" as a resolving state that holds the loading screen, so the dashboard never flashes. Every login=success token carries a required user_role claim, so the role always hydrates within a tick and this cannot hang; it is scoped to the landing, so ordinary dashboard visits are unaffected. --- .../src/app/(dashboard)/page.test.tsx | 15 +++++++++++++++ ui/litellm-dashboard/src/app/(dashboard)/page.tsx | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx index affcca401f2..89975f231aa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.test.tsx @@ -96,6 +96,21 @@ describe("dashboard landing keyless redirect", () => { expect(mockUseKeys.mock.calls[0][3]).toBe(false); }); + it("holds the loading screen on the landing until the role hydrates, instead of flashing the dashboard", () => { + state.userRole = ""; + render(); + expect(screen.getByTestId("loading-screen")).toBeInTheDocument(); + expect(screen.queryByTestId("api-keys-dashboard")).not.toBeInTheDocument(); + expect(mockReplace).not.toHaveBeenCalled(); + }); + + it("does not hold the dashboard for an unhydrated role outside the post-login landing", () => { + state.login = null; + state.userRole = ""; + render(); + expect(screen.getByTestId("api-keys-dashboard")).toBeInTheDocument(); + }); + it("holds the loading screen while the key lookup is in flight", () => { state.keysLoading = true; render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx index 682ba0bbcb2..c43ba12985d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx @@ -93,10 +93,12 @@ function CreateKeyPageContent() { const isPostLoginLanding = searchParams.get("login") === "success"; const isSignedIn = !authLoading && Boolean(token); + const isAwaitingRole = isPostLoginLanding && isSignedIn && userRole === ""; const shouldCheckForKeys = isPostLoginLanding && isSignedIn && internalUserRoles.includes(userRole); const { data: keysData, isLoading: keysLoading } = useKeys(1, 1, { userID }, shouldCheckForKeys); const isKeylessLanding = shouldCheckForKeys && !keysLoading && keysData?.keys?.length === 0; const isResolvingKeylessLanding = (shouldCheckForKeys && keysLoading) || isKeylessLanding; + const isResolvingLanding = isAwaitingRole || isResolvingKeylessLanding; useEffect(() => { if (isKeylessLanding && !didReturnRedirectRef.current) { @@ -104,7 +106,7 @@ function CreateKeyPageContent() { } }, [isKeylessLanding, router]); - const isRedirecting = redirectToLogin || isLegacyRedirect || isResolvingKeylessLanding; + const isRedirecting = redirectToLogin || isLegacyRedirect || isResolvingLanding; if (authLoading || isRedirecting) { return ; From b5bc3631e1a62f78b82fe225fadc557d8d54125c Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 23 Jul 2026 12:14:22 -0700 Subject: [PATCH 10/49] refactor(e2e): drop require_env, read os.environ where a cred is used (#34413) require_env hard-failed a test (and, for the shared litellm-ops secret, drove piling every provider credential into one blob) whenever an optional cred was absent. Most call sites either read a value the test actually uses or just gated on the runner's env for a key the gateway consumes. Read os.environ directly where the test uses the value; drop the presence-only gates so those cases run against the proxy instead of pre-failing on the runner's environment. Removes the require_env helper from e2e_config. --- tests/e2e/batches/test_batches_e2e.py | 13 +++---------- tests/e2e/e2e_config.py | 16 ---------------- .../e2e/guardrails/test_bedrock_guardrail_e2e.py | 11 +++++------ .../test_block_code_execution_guardrail_e2e.py | 3 +-- .../test_openai_moderation_guardrail_e2e.py | 3 +-- .../guardrails/test_presidio_guardrail_e2e.py | 11 ++++------- .../test_chat_completions_regression_e2e.py | 15 +++------------ .../llm_translation/test_image_generation_e2e.py | 3 +-- tests/e2e/llm_translation/test_messages_e2e.py | 3 +-- tests/e2e/llm_translation/test_rerank_e2e.py | 3 +-- tests/e2e/llm_translation/test_responses_e2e.py | 4 +--- .../test_responses_metadata_e2e.py | 4 ++-- .../ratelimit/test_redis_backed_ratelimit_e2e.py | 4 ++-- .../ratelimit/test_redis_circuit_breaker_e2e.py | 4 ++-- 14 files changed, 27 insertions(+), 70 deletions(-) diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index b0c53becb6b..f9cd2a3f15f 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -23,7 +23,7 @@ from typing import Callable import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from batch_client import ( UPLOAD_FILENAME, @@ -702,14 +702,7 @@ class TestBedrockBatchAssumeRole: def test_unified_batch_create_with_assume_role( self, client: BatchClient, resources: ResourceManager ) -> None: - (role_arn,) = require_env("AWS_ROLE_NAME") - require_env( - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_REGION", - "AWS_BATCH_S3_BUCKET", - "AWS_BATCH_ROLE_ARN", - ) + role_arn = os.environ["AWS_ROLE_NAME"] session_name = f"e2e-batch-sts-{unique_marker()}"[:64] model_name = batch_model_name("bedrock-sts-batch") @@ -819,7 +812,7 @@ class TestHostedVllmBatch: def test_unified_file_and_batch_create( self, client: BatchClient, resources: ResourceManager ) -> None: - (api_base,) = require_env("HOSTED_VLLM_API_BASE") + api_base = os.environ["HOSTED_VLLM_API_BASE"] api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None model_id = ( os.environ.get("HOSTED_VLLM_MODEL") or "meta-llama/Llama-3.2-3B-Instruct" diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 4ecc215a22d..feed680bd4a 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -102,22 +102,6 @@ ANOMALY_SPEND_SETTLE_SECONDS = float( ) -def require_env(*names: str) -> tuple[str, ...]: - """Return the non-empty values for each env name, or hard-fail naming which are missing. - - Live e2e never skips for missing credentials: a missing key is a red run so - ops knows the suite cannot prove the product path. - """ - missing = tuple(name for name in names if not (os.environ.get(name) or "").strip()) - if missing: - joined = ", ".join(missing) - raise AssertionError( - f"missing required env for e2e: {joined}. " - "Add them to tests/e2e/.env locally and to litellm ops for stage/CI." - ) - return tuple((os.environ.get(name) or "").strip() for name in names) - - def datadog_mcp_url(*, toolsets: str = "core") -> str: """Regional Datadog remote MCP endpoint for this process's DD_SITE. diff --git a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py index 9e41b8808e8..a2408f0021e 100644 --- a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py @@ -8,9 +8,11 @@ a 200 means the guardrail never ran. from __future__ import annotations +import os + import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import UnknownApiError from guardrails_client import GuardrailsClient from lifecycle import ResourceManager @@ -33,11 +35,8 @@ class TestBedrockGuardrail: def test_bedrock_pre_call_blocks_harmful_prompt( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: - (identifier, version) = require_env( - "BEDROCK_GUARDRAIL_IDENTIFIER", - "BEDROCK_GUARDRAIL_VERSION", - ) - require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + identifier = os.environ["BEDROCK_GUARDRAIL_IDENTIFIER"] + version = os.environ["BEDROCK_GUARDRAIL_VERSION"] name = f"e2e-bedrock-guard-{unique_marker()}" guardrail_id = client.create_bedrock_guardrail( diff --git a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py index e36fc7c3f9d..de087b190d0 100644 --- a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py @@ -16,7 +16,7 @@ from __future__ import annotations import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import unwrap from guardrails_client import BlockCodeExecutionParamsBody, GuardrailsClient from lifecycle import ResourceManager @@ -46,7 +46,6 @@ class TestBlockCodeExecutionGuardrail: def test_blocks_execution_request_but_allows_explanation( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: - require_env("GEMINI_API_KEY") model = client.create_backend_model(resources, prefix="e2e-blockcode-backend") name = f"e2e-block-code-{unique_marker()}" diff --git a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py index 4e2fcbf8fba..39950259fb5 100644 --- a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py @@ -14,7 +14,7 @@ from __future__ import annotations import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import UnknownApiError, unwrap from guardrails_client import GuardrailsClient, OpenAIModerationParamsBody from lifecycle import ResourceManager @@ -34,7 +34,6 @@ class TestOpenAIModerationGuardrail: def test_moderation_blocks_flagged_input( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: - require_env("OPENAI_API_KEY", "GEMINI_API_KEY") model = client.create_backend_model(resources, prefix="e2e-moderation-backend") name = f"e2e-openai-moderation-{unique_marker()}" diff --git a/tests/e2e/guardrails/test_presidio_guardrail_e2e.py b/tests/e2e/guardrails/test_presidio_guardrail_e2e.py index a911f387382..d103714b1dd 100644 --- a/tests/e2e/guardrails/test_presidio_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_presidio_guardrail_e2e.py @@ -25,11 +25,12 @@ The chat backend is a gemini deployment created for the test. from __future__ import annotations +import os import time import pytest -from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, require_env, unique_marker +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import NoBody, require_successful_call, unwrap from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody from lifecycle import ResourceManager @@ -88,9 +89,8 @@ def _poll_logged_prompt(reader: OtelReader, *, call_id: str, genai_span: str) -> def _presidio_params( mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False ) -> PresidioParamsBody: - analyzer, anonymizer = require_env( - "PRESIDIO_ANALYZER_API_BASE", "PRESIDIO_ANONYMIZER_API_BASE" - ) + analyzer = os.environ["PRESIDIO_ANALYZER_API_BASE"] + anonymizer = os.environ["PRESIDIO_ANONYMIZER_API_BASE"] return PresidioParamsBody( mode=mode, default_on=False, @@ -124,7 +124,6 @@ class TestPresidioGuardrail: def test_pre_call_masks_pii_before_the_model_sees_it( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: - require_env("GEMINI_API_KEY") model = client.create_backend_model(resources, prefix="e2e-presidio-pre") name = f"e2e-presidio-pre-{unique_marker()}" guardrail_id = client.register(name, _presidio_params("pre_call")) @@ -149,7 +148,6 @@ class TestPresidioGuardrail: def test_post_call_masks_pii_in_model_output( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: - require_env("GEMINI_API_KEY") model = client.create_backend_model(resources, prefix="e2e-presidio-post") name = f"e2e-presidio-post-{unique_marker()}" guardrail_id = client.register(name, _presidio_params("post_call", apply_to_output=True)) @@ -173,7 +171,6 @@ class TestPresidioGuardrail: def test_logging_only_masks_the_logged_prompt( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: - require_env("GEMINI_API_KEY") _require_otel_v2_active(client) reader = build_otel_reader() diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 8d3622e441a..af0e782e224 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -21,7 +21,7 @@ import os import pytest from pydantic import BaseModel -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import StreamingResponse, unwrap from lifecycle import ResourceManager from models import ( @@ -250,7 +250,7 @@ class TestCohereChat: def test_cohere_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: - (cohere_key,) = require_env("COHERE_API_KEY") + cohere_key = os.environ["COHERE_API_KEY"] model = f"e2e-cohere-chat-{unique_marker()}" model_id = client.proxy.create_model( model, @@ -343,7 +343,7 @@ class TestHostedVllmChat: def test_hosted_vllm_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: - (api_base,) = require_env("HOSTED_VLLM_API_BASE") + api_base = os.environ["HOSTED_VLLM_API_BASE"] api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None backend = ( os.environ.get("HOSTED_VLLM_MODEL") or "meta-llama/Llama-3.2-3B-Instruct" @@ -395,7 +395,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_streams_real_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-chat-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -423,7 +422,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_logs_cost( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-cost-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -457,7 +455,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_returns_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-tool-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -488,7 +485,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_structured_output_conforms_to_schema( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-schema-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -522,7 +518,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_reasoning_reports_reasoning_tokens( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-reasoning-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -561,7 +556,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_vision_describes_image( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-vision-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_VISION_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -579,7 +573,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_prompt_cache_hits_on_repeat( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-cache-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -610,7 +603,6 @@ class TestOpenAIChatCompletions: def test_openai_chat_streams_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: - require_env("OPENAI_API_KEY") model = f"e2e-openai-tool-stream-{unique_marker()}" model_id = client.proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") @@ -645,7 +637,6 @@ class TestBedrockConverseChatCompletions: """ def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: - require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") model = f"{prefix}-{unique_marker()}" model_id = client.proxy.create_model(model, _bedrock_params()) resources.defer(lambda: client.proxy.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index 1ba78a7e083..45861d1e93a 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, ImagesResult from lifecycle import ResourceManager @@ -50,7 +50,6 @@ class TestImageGeneration: def test_bedrock_image_generation_returns_image( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") model = f"e2e-bedrock-image-{unique_marker()}" model_id = endpoints_client.create_model( model, diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index 44376218c6b..ef6ba5b95d3 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -10,7 +10,7 @@ from __future__ import annotations import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call, unwrap from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager @@ -73,7 +73,6 @@ class TestAnthropicMessages: def test_messages_logs_cost_matching_the_response_header( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - require_env("ANTHROPIC_API_KEY") model = f"e2e-messages-cost-{unique_marker()}" model_id = endpoints_client.create_model( model, diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index 0857ff65a52..c3614251e77 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, RerankResult from lifecycle import ResourceManager @@ -56,7 +56,6 @@ class TestRerank: def test_bedrock_rerank_scores_top_n( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") model = f"e2e-bedrock-rerank-{unique_marker()}" model_id = endpoints_client.create_model( model, diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index d24d2b53b71..0b2ffce5b2a 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -13,7 +13,7 @@ from typing import cast import pytest from pydantic import BaseModel, ValidationError -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import ( EndpointsClient, @@ -255,7 +255,6 @@ class TestResponses: def test_responses_bedrock_returns_completion( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") model = f"e2e-responses-{unique_marker()}" model_id = endpoints_client.create_model(model, _bedrock_params()) resources.defer(lambda: endpoints_client.delete_model(model_id)) @@ -270,7 +269,6 @@ class TestResponses: def test_responses_bedrock_returns_function_call( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") model = f"e2e-responses-{unique_marker()}" model_id = endpoints_client.create_model(model, _bedrock_params()) resources.defer(lambda: endpoints_client.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_responses_metadata_e2e.py b/tests/e2e/llm_translation/test_responses_metadata_e2e.py index 6cf24348095..df854dcfa19 100644 --- a/tests/e2e/llm_translation/test_responses_metadata_e2e.py +++ b/tests/e2e/llm_translation/test_responses_metadata_e2e.py @@ -14,7 +14,7 @@ import time import pytest from pydantic import BaseModel, ConfigDict -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, ResponsesResult from lifecycle import ResourceManager @@ -42,7 +42,7 @@ class RedisKeyInfo(BaseModel): def _redis_scan(marker: str) -> tuple[RedisKeyInfo, ...]: import redis - (host,) = require_env("REDIS_HOST") + host = os.environ["REDIS_HOST"] port = int((os.environ.get("REDIS_PORT") or "6379").strip() or "6379") try: with socket.create_connection((host, port), timeout=3): diff --git a/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py b/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py index ed6f0ce3b2c..a88f0ca546a 100644 --- a/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py @@ -11,7 +11,7 @@ import socket import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager from models import KeyGenerateBody, LiteLLMParamsBody @@ -23,7 +23,7 @@ BACKEND = "anthropic/claude-haiku-4-5-20251001" def _require_redis_reachable() -> None: - (host,) = require_env("REDIS_HOST") + host = os.environ["REDIS_HOST"] port = int((os.environ.get("REDIS_PORT") or "6379").strip() or "6379") try: with socket.create_connection((host, port), timeout=3): diff --git a/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py b/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py index b509ae000f5..3e1bc662470 100644 --- a/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py @@ -13,7 +13,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import pytest -from e2e_config import require_env, unique_marker +from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager from models import KeyGenerateBody, LiteLLMParamsBody @@ -28,7 +28,7 @@ RECOVERY_TIMEOUT = float( def _require_redis() -> None: - (host,) = require_env("REDIS_HOST") + host = os.environ["REDIS_HOST"] port = int((os.environ.get("REDIS_PORT") or "6379").strip() or "6379") try: with socket.create_connection((host, port), timeout=3): From d6d52d95e5a359c70f733bd77e539c7f419cfc1f Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 12:24:26 -0700 Subject: [PATCH 11/49] feat(cost-optimization): add by-model view to cache leakage table with plain-language columns Adds a By virtual key / By model toggle to the cache leakage table. The model view aggregates the daily activity model breakdown and is scoped to Anthropic (Claude) models, which support prompt caching. Renames the columns to plain language: Uncached input, Cache hit rate, and Potential savings (replacing Realized caching savings and Est. savings left), with a tooltip on Potential savings that spells out how it is calculated --- .../_components/CacheLeakageCard.test.tsx | 42 +++++++- .../_components/CacheLeakageCard.tsx | 69 +++++++------ .../_components/costOptimizationUtils.test.ts | 81 +++++++++++++-- .../_components/costOptimizationUtils.ts | 98 ++++++++++++------- 4 files changed, 212 insertions(+), 78 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx index 1d82fbb48ea..bb40c341786 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx @@ -1,4 +1,4 @@ -import { render } from "@testing-library/react"; +import { fireEvent, render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import type { DailyData, KeyMetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; @@ -41,6 +41,24 @@ const dayWithKeys = (date: string, apiKeys: Record>): DailyData => ({ + date, + metrics: baseMetrics({}), + breakdown: { + models: Object.fromEntries( + Object.entries(models).map(([name, m]) => [ + name, + { metrics: baseMetrics(m), metadata: {}, api_key_breakdown: {} }, + ]), + ), + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, +}); + const renderWith = (results: DailyData[]) => render( { expect(getByText("0.0%")).toBeInTheDocument(); expect(getByText("90.0%")).toBeInTheDocument(); [ - "Input tokens in the selected range that were neither read from nor written to the prompt cache", - "Share of this key's total input tokens that were served from the prompt cache", - "Dollars this key actually saved because cached input was billed at the discounted cache-read rate", - "Approximate dollars this key could still save if its uncached input had hit the cache at the portfolio's realized discount", + "Input tokens you sent in this range that weren't served from or written to the cache", + "Share of your input tokens that were served from the cache", + "About how much you'd save if this uncached input used prompt caching. Estimated as uncached input tokens times the per-token discount your cached traffic already gets (realized cache savings ÷ cache-read tokens).", ].forEach((info) => expect(getByLabelText(info)).toBeInTheDocument()); }); + it("switches to the model view and lists only Anthropic models", () => { + const { getByText, queryByText } = renderWith([ + dayWithModels("2026-07-12", { + "claude-sonnet-5": { prompt_tokens: 5000, cache_read_input_tokens: 0 }, + "gpt-4o": { prompt_tokens: 8000, cache_read_input_tokens: 0 }, + }), + ]); + + fireEvent.click(getByText("By model")); + + expect(getByText("Cache leakage by model")).toBeInTheDocument(); + expect(getByText("claude-sonnet-5")).toBeInTheDocument(); + expect(queryByText("gpt-4o")).not.toBeInTheDocument(); + }); + it("shows an empty state when no key used tokens in the range", () => { const { getByText, queryByRole } = renderWith([dayWithKeys("2026-07-12", {})]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index 359ce502b07..ad1ff303dbc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -1,14 +1,15 @@ "use client"; -import React, { useMemo } from "react"; +import React, { useMemo, useState } from "react"; import { Info } from "lucide-react"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; +import { CacheLeakageDimension, computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; import { DailyActivityRange } from "./useDailyActivityRange"; interface CacheLeakageCardProps { @@ -19,19 +20,22 @@ const HeadWithInfo = ({ label, info }: { label: string; info: string }) => ( {label} - - - - + }> + - {info} + {info} ); const CacheLeakageCard: React.FC = ({ activity }) => { const { dateValue, onDateChange, results, loading, isFetchingMore } = activity; - const leakage = useMemo(() => computeCacheLeakage(results), [results]); + const [dimension, setDimension] = useState("key"); + const leakage = useMemo(() => computeCacheLeakage(results, dimension), [results, dimension]); + + const subject = dimension === "model" ? "Models" : "Keys"; + const firstColumn = dimension === "model" ? "Model" : "Key"; + const emptyNoun = dimension === "model" ? "model" : "key"; return ( @@ -39,64 +43,67 @@ const CacheLeakageCard: React.FC = ({ activity }) => {
- Cache leakage by virtual key + Cache leakage by {dimension === "model" ? "model" : "virtual key"}

- Keys sending large volumes of uncached prompt tokens with a low cache-hit ratio are likely missing - prompt caching. Estimated savings left is approximate: uncached prompt tokens priced at the - portfolio's realized cache-read discount. + {subject} sending large volumes of uncached input with a low cache hit rate are likely missing prompt + caching. Potential savings is approximate: uncached input priced at the realized cache-read discount. + {dimension === "model" ? " Limited to Anthropic (Claude) models, which support prompt caching." : ""}

+ setDimension(value === "model" ? "model" : "key")} + className="mt-3" + > + + By virtual key + By model + +
{leakage.rows.length === 0 ? (

- {loading || isFetchingMore ? "Loading..." : "No key usage in this range."} + {loading || isFetchingMore ? "Loading..." : `No ${emptyNoun} usage in this range.`}

) : ( - Key + {firstColumn} - - - {leakage.rows.map((row) => ( - + - {row.keyAlias || `${row.apiKey.slice(0, 8)}...`} - {row.teamId && ({row.teamId})} + {row.label} + {row.sublabel && ({row.sublabel})} {formatNumberWithCommas(row.uncachedPromptTokens)} {pct(row.cacheHitRatio)} - {usd(row.realizedCachingSavings)} - {row.estSavingsLeft == null ? "—" : usd(row.estSavingsLeft)} + {row.potentialSavings == null ? "—" : usd(row.potentialSavings)} ))} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index 562552ffb50..2f1558465d5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import type { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; -import { buildDailyToolSeries, computeCacheLeakage, topToolsBySpend } from "./costOptimizationUtils"; +import { buildDailyToolSeries, computeCacheLeakage, isAnthropicModel, topToolsBySpend } from "./costOptimizationUtils"; const metrics = (overrides: Partial): SpendMetrics => ({ spend: 0, @@ -38,6 +38,21 @@ const day = ( }, }); +const modelDay = (date: string, models: Record>): DailyData => ({ + date, + metrics: metrics({}), + breakdown: { + models: Object.fromEntries( + Object.entries(models).map(([name, m]) => [name, { metrics: metrics(m), metadata: {}, api_key_breakdown: {} }]), + ), + model_groups: {}, + mcp_servers: {}, + providers: {}, + entities: {}, + api_keys: {}, + }, +}); + describe("computeCacheLeakage", () => { it("aggregates a key's tokens and savings across multiple days", () => { const results = [ @@ -76,8 +91,8 @@ describe("computeCacheLeakage", () => { ]; const { rows, discountPerToken } = computeCacheLeakage(results); expect(discountPerToken).toBeCloseTo(0.002, 6); - expect(rows.map((r) => r.keyAlias)).toEqual(["leaker"]); - expect(rows[0].estSavingsLeft).toBeCloseTo(1.0, 6); + expect(rows.map((r) => r.label)).toEqual(["leaker"]); + expect(rows[0].potentialSavings).toBeCloseTo(1.0, 6); }); it("returns null estimate and ranks by uncached tokens when nobody used caching", () => { @@ -89,8 +104,8 @@ describe("computeCacheLeakage", () => { ]; const { rows, discountPerToken } = computeCacheLeakage(results); expect(discountPerToken).toBeNull(); - expect(rows.map((r) => r.keyAlias)).toEqual(["big", "small"]); - expect(rows.every((r) => r.estSavingsLeft === null)).toBe(true); + expect(rows.map((r) => r.label)).toEqual(["big", "small"]); + expect(rows.every((r) => r.potentialSavings === null)).toBe(true); }); it("computes cache hit ratio against total prompt tokens and clamps inconsistent data at zero", () => { @@ -101,7 +116,7 @@ describe("computeCacheLeakage", () => { }), ]; const { rows } = computeCacheLeakage(results); - expect(rows.map((r) => r.keyAlias)).toEqual(["mixed"]); + expect(rows.map((r) => r.label)).toEqual(["mixed"]); expect(rows[0].cacheHitRatio).toBeCloseTo(0.75, 6); expect(rows[0].uncachedPromptTokens).toBe(250); }); @@ -110,11 +125,63 @@ describe("computeCacheLeakage", () => { const keys = Object.fromEntries( Array.from({ length: 15 }, (_, i) => [`h${i}`, { alias: `k${i}`, metrics: { prompt_tokens: i + 1 } }]), ); - const { rows } = computeCacheLeakage([day("2026-07-01", keys)], 5); + const { rows } = computeCacheLeakage([day("2026-07-01", keys)], "key", 5); expect(rows).toHaveLength(5); }); }); +describe("computeCacheLeakage by model", () => { + it("aggregates only Anthropic models and ignores other providers", () => { + const models: Record> = { + "claude-sonnet-5": { prompt_tokens: 10000, cache_read_input_tokens: 0 }, + "anthropic/claude-haiku-4-5": { prompt_tokens: 4000, cache_read_input_tokens: 0 }, + "bedrock/anthropic.claude-3-5-sonnet": { prompt_tokens: 2000, cache_read_input_tokens: 0 }, + "gpt-4o": { prompt_tokens: 9000, cache_read_input_tokens: 0 }, + "deepseek-chat": { prompt_tokens: 8000, cache_read_input_tokens: 0 }, + }; + const { rows } = computeCacheLeakage([modelDay("2026-07-01", models)], "model"); + expect(rows.map((r) => r.id)).toEqual([ + "claude-sonnet-5", + "anthropic/claude-haiku-4-5", + "bedrock/anthropic.claude-3-5-sonnet", + ]); + }); + + it("labels model rows by model name with no sublabel", () => { + const results = [modelDay("2026-07-01", { "claude-sonnet-5": { prompt_tokens: 1000 } })]; + const { rows } = computeCacheLeakage(results, "model"); + expect(rows[0].label).toBe("claude-sonnet-5"); + expect(rows[0].sublabel).toBeNull(); + }); + + it("prices model leakage at the Anthropic realized cache-read discount", () => { + const results = [ + modelDay("2026-07-01", { + "claude-sonnet-5": { prompt_tokens: 1000, cache_read_input_tokens: 1000, prompt_caching_savings_spend: 2.0 }, + "claude-haiku-4-5": { prompt_tokens: 500 }, + }), + ]; + const { rows, discountPerToken } = computeCacheLeakage(results, "model"); + expect(discountPerToken).toBeCloseTo(0.002, 6); + expect(rows.map((r) => r.id)).toEqual(["claude-haiku-4-5"]); + expect(rows[0].potentialSavings).toBeCloseTo(1.0, 6); + }); +}); + +describe("isAnthropicModel", () => { + it("matches Claude-family models across providers and rejects others", () => { + const anthropic = [ + "claude-sonnet-5", + "anthropic/claude-haiku-4-5", + "bedrock/anthropic.claude-3-5-sonnet", + "vertex_ai/claude-opus-4-8", + ]; + const others = ["gpt-4o", "deepseek-chat", "gemini-2.5-pro", "mistral-large"]; + expect(anthropic.every(isAnthropicModel)).toBe(true); + expect(others.some(isAnthropicModel)).toBe(false); + }); +}); + describe("buildDailyToolSeries", () => { const daily: ToolSpendDailyEntry[] = [ { date: "2026-07-01", tool_name: "search", spend: 1.0, call_count: 1 }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 2868bdac880..30f851bfeca 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -1,4 +1,4 @@ -import { DailyData } from "@/components/UsagePage/types"; +import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; @@ -9,15 +9,15 @@ export const usd = (value: number): string => { export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; +export type CacheLeakageDimension = "key" | "model"; + export interface CacheLeakageRow { - apiKey: string; - keyAlias: string | null; - teamId: string | null; + id: string; + label: string; + sublabel: string | null; uncachedPromptTokens: number; - cacheReadTokens: number; cacheHitRatio: number; - realizedCachingSavings: number; - estSavingsLeft: number | null; + potentialSavings: number | null; } export interface CacheLeakageResult { @@ -25,8 +25,10 @@ export interface CacheLeakageResult { discountPerToken: number | null; } -interface KeyAccumulator { - keyAlias: string | null; +export const isAnthropicModel = (model: string): boolean => /claude|anthropic/i.test(model); + +interface LeakageAccumulator { + alias: string | null; teamId: string | null; promptTokens: number; cacheReadTokens: number; @@ -34,8 +36,8 @@ interface KeyAccumulator { realizedCachingSavings: number; } -const emptyAccumulator = (): KeyAccumulator => ({ - keyAlias: null, +const emptyAccumulator = (): LeakageAccumulator => ({ + alias: null, teamId: null, promptTokens: 0, cacheReadTokens: 0, @@ -43,26 +45,54 @@ const emptyAccumulator = (): KeyAccumulator => ({ realizedCachingSavings: 0, }); -export const computeCacheLeakage = (results: readonly DailyData[], limit = 10): CacheLeakageResult => { - const byKey = new Map(); +const addMetrics = ( + acc: LeakageAccumulator, + m: SpendMetrics, + alias: string | null, + teamId: string | null, +): LeakageAccumulator => ({ + alias: acc.alias ?? alias, + teamId: acc.teamId ?? teamId, + promptTokens: acc.promptTokens + (m.prompt_tokens ?? 0), + cacheReadTokens: acc.cacheReadTokens + (m.cache_read_input_tokens ?? 0), + cacheCreationTokens: acc.cacheCreationTokens + (m.cache_creation_input_tokens ?? 0), + realizedCachingSavings: acc.realizedCachingSavings + (m.prompt_caching_savings_spend ?? 0), +}); + +const aggregateByKey = (results: readonly DailyData[]): Map => { + const byKey = new Map(); for (const day of results) { - const apiKeys = day.breakdown?.api_keys ?? {}; - for (const [apiKey, entry] of Object.entries(apiKeys)) { + for (const [apiKey, entry] of Object.entries(day.breakdown?.api_keys ?? {})) { const acc = byKey.get(apiKey) ?? emptyAccumulator(); - const m = entry.metrics; - const next: KeyAccumulator = { - keyAlias: acc.keyAlias ?? entry.metadata?.key_alias ?? null, - teamId: acc.teamId ?? entry.metadata?.team_id ?? null, - promptTokens: acc.promptTokens + (m.prompt_tokens ?? 0), - cacheReadTokens: acc.cacheReadTokens + (m.cache_read_input_tokens ?? 0), - cacheCreationTokens: acc.cacheCreationTokens + (m.cache_creation_input_tokens ?? 0), - realizedCachingSavings: acc.realizedCachingSavings + (m.prompt_caching_savings_spend ?? 0), - }; - byKey.set(apiKey, next); + byKey.set( + apiKey, + addMetrics(acc, entry.metrics, entry.metadata?.key_alias ?? null, entry.metadata?.team_id ?? null), + ); } } + return byKey; +}; - const totals = [...byKey.values()].reduce( +const aggregateByModel = (results: readonly DailyData[]): Map => { + const byModel = new Map(); + for (const day of results) { + for (const [model, entry] of Object.entries(day.breakdown?.models ?? {})) { + if (!isAnthropicModel(model)) continue; + const acc = byModel.get(model) ?? emptyAccumulator(); + byModel.set(model, addMetrics(acc, entry.metrics, null, null)); + } + } + return byModel; +}; + +export const computeCacheLeakage = ( + results: readonly DailyData[], + dimension: CacheLeakageDimension = "key", + limit = 10, +): CacheLeakageResult => { + const byEntity = dimension === "model" ? aggregateByModel(results) : aggregateByKey(results); + + const totals = [...byEntity.values()].reduce( (agg, a) => ({ cacheReadTokens: agg.cacheReadTokens + a.cacheReadTokens, realizedCachingSavings: agg.realizedCachingSavings + a.realizedCachingSavings, @@ -71,25 +101,23 @@ export const computeCacheLeakage = (results: readonly DailyData[], limit = 10): ); const discountPerToken = totals.cacheReadTokens > 0 ? totals.realizedCachingSavings / totals.cacheReadTokens : null; - const rows: CacheLeakageRow[] = [...byKey.entries()] - .map(([apiKey, a]) => { + const rows: CacheLeakageRow[] = [...byEntity.entries()] + .map(([id, a]) => { const uncachedPromptTokens = Math.max(0, a.promptTokens - a.cacheReadTokens - a.cacheCreationTokens); return { - apiKey, - keyAlias: a.keyAlias, - teamId: a.teamId, + id, + label: dimension === "model" ? id : a.alias ?? `${id.slice(0, 8)}...`, + sublabel: dimension === "model" ? null : a.teamId, uncachedPromptTokens, - cacheReadTokens: a.cacheReadTokens, cacheHitRatio: a.promptTokens > 0 ? a.cacheReadTokens / a.promptTokens : 0, - realizedCachingSavings: a.realizedCachingSavings, - estSavingsLeft: discountPerToken != null ? uncachedPromptTokens * discountPerToken : null, + potentialSavings: discountPerToken != null ? uncachedPromptTokens * discountPerToken : null, }; }) .filter((row) => row.uncachedPromptTokens > 0); const sorted = rows.sort((x, y) => discountPerToken != null - ? (y.estSavingsLeft ?? 0) - (x.estSavingsLeft ?? 0) + ? (y.potentialSavings ?? 0) - (x.potentialSavings ?? 0) : y.uncachedPromptTokens - x.uncachedPromptTokens, ); From 090fd491d4b655a8b3c6374545987d0dee7f108f Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 13:22:03 -0700 Subject: [PATCH 12/49] feat(cost-optimization): sortable cache leakage columns and clearer token column name Makes the three metric columns on the cache leakage table sortable, each with a sensible first-click direction: most uncached tokens and biggest potential savings first, worst cache hit rate first. Repeat clicks toggle the direction. Renames Uncached input to Uncached input tokens, since the column is a token count --- .../_components/CacheLeakageCard.test.tsx | 26 ++++ .../_components/CacheLeakageCard.tsx | 128 +++++++++++++----- 2 files changed, 122 insertions(+), 32 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx index bb40c341786..07e5e4edf50 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx @@ -91,6 +91,32 @@ describe("CacheLeakageCard", () => { ].forEach((info) => expect(getByLabelText(info)).toBeInTheDocument()); }); + it("sorts by the clicked column, worst cache hit rate first", () => { + const { getAllByRole, getByText } = renderWith([ + dayWithKeys("2026-07-12", { + "hash-a": key("alpha", { + prompt_tokens: 10000, + cache_read_input_tokens: 9000, + prompt_caching_savings_spend: 9.0, + }), + "hash-b": key("bravo", { + prompt_tokens: 500, + cache_read_input_tokens: 50, + prompt_caching_savings_spend: 0.05, + }), + }), + ]); + const firstDataRow = () => getAllByRole("row")[1]; + + expect(firstDataRow()).toHaveTextContent("alpha"); + + fireEvent.click(getByText("Cache hit rate")); + expect(firstDataRow()).toHaveTextContent("bravo"); + + fireEvent.click(getByText("Cache hit rate")); + expect(firstDataRow()).toHaveTextContent("alpha"); + }); + it("switches to the model view and lists only Anthropic models", () => { const { getByText, queryByText } = renderWith([ dayWithModels("2026-07-12", { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index ad1ff303dbc..bd0ecea9483 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -1,7 +1,7 @@ "use client"; import React, { useMemo, useState } from "react"; -import { Info } from "lucide-react"; +import { ArrowDown, ArrowUp, ArrowUpDown, Info } from "lucide-react"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -9,29 +9,90 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@ import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { CacheLeakageDimension, computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; +import { CacheLeakageDimension, CacheLeakageRow, computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; import { DailyActivityRange } from "./useDailyActivityRange"; interface CacheLeakageCardProps { activity: DailyActivityRange; } -const HeadWithInfo = ({ label, info }: { label: string; info: string }) => ( - - {label} - - }> - - - {info} - - +type SortColumn = "uncachedPromptTokens" | "cacheHitRatio" | "potentialSavings"; +interface SortState { + column: SortColumn; + dir: "asc" | "desc"; +} + +const NATURAL_DIR: Record = { + uncachedPromptTokens: "desc", + cacheHitRatio: "asc", + potentialSavings: "desc", +}; + +const compareRows = (a: CacheLeakageRow, b: CacheLeakageRow, sort: SortState): number => { + const av = a[sort.column]; + const bv = b[sort.column]; + if (av == null && bv == null) return 0; + if (av == null) return 1; + if (bv == null) return -1; + return sort.dir === "asc" ? av - bv : bv - av; +}; + +const InfoTooltip = ({ info }: { info: string }) => ( + + }> + + + {info} + ); +const SortableHead = ({ + column, + label, + info, + sort, + onSort, +}: { + column: SortColumn; + label: string; + info: string; + sort: SortState; + onSort: (column: SortColumn) => void; +}) => { + const active = sort.column === column; + const ActiveArrow = sort.dir === "asc" ? ArrowUp : ArrowDown; + const Arrow = active ? ActiveArrow : ArrowUpDown; + return ( + + + + + + + ); +}; + const CacheLeakageCard: React.FC = ({ activity }) => { const { dateValue, onDateChange, results, loading, isFetchingMore } = activity; const [dimension, setDimension] = useState("key"); + const [sort, setSort] = useState({ column: "potentialSavings", dir: "desc" }); const leakage = useMemo(() => computeCacheLeakage(results, dimension), [results, dimension]); + const rows = useMemo(() => [...leakage.rows].sort((a, b) => compareRows(a, b, sort)), [leakage.rows, sort]); + + const onSort = (column: SortColumn) => + setSort((prev) => + prev.column === column + ? { column, dir: prev.dir === "asc" ? "desc" : "asc" } + : { column, dir: NATURAL_DIR[column] }, + ); const subject = dimension === "model" ? "Models" : "Keys"; const firstColumn = dimension === "model" ? "Model" : "Key"; @@ -64,7 +125,7 @@ const CacheLeakageCard: React.FC = ({ activity }) => { - {leakage.rows.length === 0 ? ( + {rows.length === 0 ? (

{loading || isFetchingMore ? "Loading..." : `No ${emptyNoun} usage in this range.`}

@@ -73,28 +134,31 @@ const CacheLeakageCard: React.FC = ({ activity }) => { {firstColumn} - - - - - - - - - + + + - {leakage.rows.map((row) => ( + {rows.map((row) => ( {row.label} From 2b77e8c4dba4567676c46bc19717877908ce91c1 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 23 Jul 2026 15:38:42 -0700 Subject: [PATCH 13/49] fix(ui): keep cache leakage time range picker inline at narrow widths The card header used flex-wrap, so the date picker was the element that gave way when the row ran out of room; at higher browser zoom it dropped onto its own line under the description. Pin the picker with shrink-0 and let the title/description block shrink instead (min-w-0), so the copy wraps to a second line and the picker stays on the right. Below md the header stacks, since a 300px input plus its nowrap label leaves nothing usable beside it. --- .../cost-optimization/_components/CacheLeakageCard.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index bd0ecea9483..4f1cfc49569 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -102,8 +102,8 @@ const CacheLeakageCard: React.FC = ({ activity }) => { -
-
+
+
Cache leakage by {dimension === "model" ? "model" : "virtual key"}

{subject} sending large volumes of uncached input with a low cache hit rate are likely missing prompt @@ -111,7 +111,9 @@ const CacheLeakageCard: React.FC = ({ activity }) => { {dimension === "model" ? " Limited to Anthropic (Claude) models, which support prompt caching." : ""}

- +
+ +
Date: Thu, 23 Jul 2026 16:35:56 -0700 Subject: [PATCH 14/49] refactor(ui): migrate agents to shadcn (#34365) * test(ui): make the agents route's tests markup-agnostic before migration Rewrites the two assertions that were coupled to antd's DOM and adds the missing characterisation test for agent_cost_view, so the suite describes behaviour rather than antd markup and can stay untouched across the shadcn migration. The skill selection test reached the checkbox with a querySelector on input[type=checkbox]; antd renders an input while Base UI renders a span[role=checkbox], so it now queries by role and accessible name, which both libraries derive from the wrapping label. The delete confirmation test queried role=dialog; antd Modal is a dialog while Base UI AlertDialog is an alertdialog, so it now anchors on the confirmation text and accepts either role. agent_cost_view had no test at all; it gets one covering the null render, the dollar-prefixed values, the omitted rows, and a zero cost that must not be mistaken for unset. All 55 tests pass against the current antd components. * refactor(ui): migrate agents to shadcn Replaces antd and Tremor with shadcn (base-vega) primitives across the five files the agents route exclusively owns. Markup only; no behaviour, data fetching or route structure changes. Modal becomes AlertDialog, with a plain destructive Button in the footer rather than AlertDialogAction, because that action is AlertDialog.Close and would dismiss the dialog before the delete request settles, losing the in-flight state. Alert, Tag, Spin, Space, Collapse, Descriptions, Typography and the antd icons map onto alert, badge, ui-loading-spinner, flex/grid utilities, collapsible, a definition list, semantic headings and lucide. The shadcn CLI emits alert.tsx importing cva from class-variance-authority, which this project does not depend on; it uses the cva object syntax from lib/cva.config. The generated file fails to typecheck, so the adapted copy lives in components/shared instead, per the convention that ui/ stays CLI-managed. Colour comes from tokens throughout, so the info callout is now the neutral card style rather than antd's blue, and nothing hardcodes a colour in the way of a later theme change. The 55 tests in the route pass unchanged from the previous commit. The visual gate re-baselined agents and all 34 other routes stayed pixel-identical. --- ui/litellm-dashboard/eslint-suppressions.json | 19 - .../agents/_components/AgentsPanel.test.tsx | 5 +- .../agents/_components/AgentsPanel.tsx | 61 ++-- .../agents/_components/AgentsTable.tsx | 37 +- .../_components/agent_card_discovery.test.tsx | 5 +- .../_components/agent_card_discovery.tsx | 339 ++++++++++-------- .../_components/agent_cost_view.test.tsx | 54 +++ .../agents/_components/agent_cost_view.tsx | 33 +- .../agents/_components/agent_virtual_keys.tsx | 45 +-- .../src/components/shared/Alert.tsx | 59 +++ 10 files changed, 402 insertions(+), 255 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_cost_view.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/Alert.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 4fa5528aab8..8bd7f675a0c 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -37,16 +37,6 @@ "count": 1 } }, - "src/app/(dashboard)/agents/_components/AgentsPanel.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/agents/_components/AgentsTable.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/agents/_components/add_agent_form.tsx": { "local/filename-pascal-case": { "count": 1 @@ -71,9 +61,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/refs": { "count": 3 }, @@ -84,9 +71,6 @@ "src/app/(dashboard)/agents/_components/agent_cost_view.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/app/(dashboard)/agents/_components/agent_form_fields.tsx": { @@ -123,9 +107,6 @@ }, "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/agents/_components/cost_config_fields.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx index 441d300436a..d873687378b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx @@ -140,8 +140,9 @@ describe("AgentsPanel", () => { await user.click(await screen.findByTestId("agent-actions-agent-9")); await user.click(await screen.findByTestId("agent-action-delete")); - const modal = await screen.findByRole("dialog"); - await user.click(within(modal).getByRole("button", { name: /^delete$/i })); + const confirmPrompt = await screen.findByText(/are you sure you want to delete agent: Doomed Agent\?/i); + const confirmDialog = confirmPrompt.closest('[role="dialog"],[role="alertdialog"]') as HTMLElement; + await user.click(within(confirmDialog).getByRole("button", { name: /^delete$/i })); await waitFor(() => { expect(networking.deleteAgentCall).toHaveBeenCalledWith("test-token", "agent-9"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx index a4a71530c84..4459ee0c377 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx @@ -1,6 +1,5 @@ import React, { useState, useEffect } from "react"; -import { Modal, Alert } from "antd"; -import { Plus } from "lucide-react"; +import { Info, Plus } from "lucide-react"; import { getAgentsList, deleteAgentCall } from "@/components/networking"; import AddAgentForm from "./add_agent_form"; import { isAdminRole } from "@/utils/roles"; @@ -9,6 +8,16 @@ import AgentsTable from "./AgentsTable"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { Agent } from "@/components/agents/types"; import { Team } from "@/components/key_team_helpers/key_list"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { + AlertDialog, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; interface AgentsPanelProps { @@ -130,17 +139,18 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams

Agents

-

+

List of A2A-spec agents that are available to be used in your organization. Go to AI Hub, to make agents public.

- + + + Why do agents need keys? + + Keys scope access to an agent and allow it to call MCP tools. Assign a key when creating an agent or from + the Virtual Keys page. + + {isAdmin && (
+ + + )}
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx index 824ae47f3e6..359cb49b910 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx @@ -1,13 +1,13 @@ "use client"; import { SortingState } from "@tanstack/react-table"; -import { Tooltip, Switch } from "antd"; -import { CheckCircleOutlined } from "@ant-design/icons"; -import { Bot } from "lucide-react"; +import { Bot, CircleCheck } from "lucide-react"; import React, { useMemo, useState } from "react"; import { Agent } from "@/components/agents/types"; import { DataTable } from "@/components/shared/DataTable"; +import { Switch } from "@/components/ui/switch"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { getAgentsTableColumns } from "./AgentsTableColumns"; @@ -67,18 +67,27 @@ const AgentsTable: React.FC = ({ size="compact" toolbar={() => (
- -
- - Health Check - + + + + Health Check + +
+ } /> -
- + When enabled, only agents with reachable URLs are shown + +
)} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.test.tsx index 4ee6332c54e..7858bdb1cd4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.test.tsx @@ -126,10 +126,7 @@ describe("AgentCardDiscovery", () => { expect(initialSelection.upstream_url).toBe("https://upstream.example.com"); expect(initialSelection.selected_card.skills).toHaveLength(2); - const summarizeLabel = screen.getByText("Summarize").closest("label"); - expect(summarizeLabel).toBeTruthy(); - const summarizeCheckbox = summarizeLabel!.querySelector("input[type='checkbox']") as HTMLInputElement; - await user.click(summarizeCheckbox); + await user.click(screen.getByRole("checkbox", { name: /Summarize/i })); await waitFor(() => { const latest = onApply.mock.calls.at(-1)?.[0]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.tsx index e979b2dbe3f..017a9928f8b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_card_discovery.tsx @@ -1,18 +1,20 @@ "use client"; import React, { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { Alert, Button, Checkbox, Collapse, Empty, Input, Space, Spin, Switch, Tag, Tooltip, Typography } from "antd"; -// Empty is used in the skills panel below. -import { - CheckCircleTwoTone, - InfoCircleOutlined, - LinkOutlined, - ReloadOutlined, - SearchOutlined, -} from "@ant-design/icons"; +import { ChevronDown, CircleAlert, CircleCheck, Info, Link as LinkIcon, RotateCw, Search, X } from "lucide-react"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { DiscoveredAgentCard, discoverAgentCardCall } from "@/components/networking"; +import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { ALLOWED_CAPABILITY_KEYS, selectionsFromSavedAgentCard, @@ -20,9 +22,6 @@ import { skillId, } from "./agent_discovery_utils"; -const { Text, Paragraph } = Typography; -const { Panel } = Collapse; - const DISCOVERY_DEBOUNCE_WAIT_MS = 400; export interface DiscoveredAgentCardSelection { @@ -243,102 +242,115 @@ const AgentCardDiscovery: React.FC = ({ const skillCount = card?.skills?.length ?? 0; const selectedSkillCount = selectedSkillIds.size; + const renderDiscoverIcon = () => { + if (loading) return ; + if (card) return ; + return ; + }; + const discoverLabel = card ? "Re-discover" : "Discover"; + return ( -
-
- - Discover from agent URL - - - +
+
+ + Discover from agent URL + + + + + + } + /> + + LiteLLM will fetch /.well-known/agent-card.json from this URL and let you pick which skills and + capabilities to expose through the proxy. + + +
{isParentDriven ? ( <> - +

Using the connection details you entered above. We'll fetch: - -

+

+
{discoveryRequest!.display_url || effectiveUrl || ( - Fill in the fields above first + Fill in the fields above first )}
-
) : ( <> - +

Paste the upstream agent's base URL. We'll try /.well-known/agent-card.json,{" "} /.well-known/agent.json, and /agent.json in order. - +

- +
setManualUrl(e.target.value)} - onPressEnter={handleDiscover} - allowClear + onKeyDown={(e) => { + if (e.key === "Enter") handleDiscover(); + }} disabled={loading} /> - - +
)} {error && ( - setError(null)} - /> + + + Discovery failed + {error} + + + + )} {loading && !card && (
- +
)} {card && ( -
-
- - - Upstream card loaded - {card.version && v{card.version}} - {card.provider?.organization && {card.provider.organization}} - +
+
+ + Upstream card loaded + {card.version && v{card.version}} + {card.provider?.organization && {card.provider.organization}}
-
+
- + setEditedName(e.target.value)} placeholder="Agent name" />
- - Description +