Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_/wonderful-northcutt-14b37d

This commit is contained in:
Yuneng Jiang 2026-07-23 23:18:02 -07:00
commit 68bba5ac0d
No known key found for this signature in database
12 changed files with 418 additions and 15 deletions

View file

@ -4251,7 +4251,7 @@
},
"src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": {
"no-nested-ternary": {
"count": 3
"count": 2
},
"no-restricted-imports": {
"count": 1

View file

@ -102,8 +102,8 @@ const CacheLeakageCard: React.FC<CacheLeakageCardProps> = ({ activity }) => {
<TooltipProvider delay={300}>
<Card>
<CardHeader>
<div className="flex flex-wrap items-start justify-between gap-4">
<div>
<div className="flex flex-col gap-4 md:flex-row md:items-start md:justify-between">
<div className="min-w-0">
<CardTitle>Cache leakage by {dimension === "model" ? "model" : "virtual key"}</CardTitle>
<p className="mt-1 text-sm text-muted-foreground">
{subject} sending large volumes of uncached input with a low cache hit rate are likely missing prompt
@ -111,7 +111,9 @@ const CacheLeakageCard: React.FC<CacheLeakageCardProps> = ({ activity }) => {
{dimension === "model" ? " Limited to Anthropic (Claude) models, which support prompt caching." : ""}
</p>
</div>
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
<div className="shrink-0">
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
</div>
</div>
<Tabs
value={dimension}

View file

@ -3,13 +3,17 @@ import { renderHook, waitFor } from "@testing-library/react";
import React, { ReactNode } from "react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
isAutoRouterDeployment,
selectAutoRouterModelGroups,
useAllProxyModels,
useAutoRouterModelGroups,
useInfiniteModelInfo,
useModelHub,
useModelsInfo,
useSelectedTeamModels,
useUserModels,
type AllProxyModelsResponse,
type AutoRouterCandidateDeployment,
type PaginatedModelInfoResponse,
type ProxyModel,
} from "./useModels";
@ -918,3 +922,161 @@ describe("useInfiniteModelInfo", () => {
expect(modelInfoCall).not.toHaveBeenCalled();
});
});
describe("isAutoRouterDeployment", () => {
const cases: [string, string | null | undefined, boolean][] = [
["base semantic auto-router", "auto_router/my_router", true],
["complexity router", "auto_router/complexity_router", true],
["adaptive router", "auto_router/adaptive_router", true],
["quality router", "auto_router/quality_router", true],
["plain provider alias", "anthropic/claude-haiku-4-5", false],
["wildcard deployment", "openai/*", false],
["name merely containing the prefix", "openai/auto_router/nope", false],
["missing model", undefined, false],
["null model", null, false],
];
it.each(cases)("returns %s -> %s", (_label, litellmParamsModel, expected) => {
expect(isAutoRouterDeployment({ model_name: "some-group", litellm_params: { model: litellmParamsModel } })).toBe(
expected,
);
});
it("returns false when litellm_params is absent", () => {
expect(isAutoRouterDeployment({ model_name: "some-group" })).toBe(false);
});
});
describe("selectAutoRouterModelGroups", () => {
it("keeps only the public model_name of auto-router deployments", () => {
const deployments: AutoRouterCandidateDeployment[] = [
{ model_name: "smart-router", litellm_params: { model: "auto_router/complexity_router" } },
{ model_name: "claude-haiku", litellm_params: { model: "anthropic/claude-haiku-4-5" } },
{ model_name: "claude-sonnet", litellm_params: { model: "anthropic/claude-sonnet-4-5" } },
{ model_name: "cheap-router", litellm_params: { model: "auto_router/adaptive_router" } },
];
expect(selectAutoRouterModelGroups(deployments)).toEqual(new Set(["smart-router", "cheap-router"]));
});
it("drops auto-router deployments that have no public model_name", () => {
expect(
selectAutoRouterModelGroups([{ model_name: "", litellm_params: { model: "auto_router/complexity_router" } }]),
).toEqual(new Set());
});
it("returns an empty set for an empty model list", () => {
expect(selectAutoRouterModelGroups([])).toEqual(new Set());
});
});
describe("useAutoRouterModelGroups", () => {
let queryClient: QueryClient;
beforeEach(() => {
queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
vi.clearAllMocks();
mockUseAuthorized.mockReturnValue({
accessToken: "test-access-token",
userId: "test-user-id",
userRole: "Admin",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
});
const wrapper = ({ children }: { children: ReactNode }) =>
React.createElement(QueryClientProvider, { client: queryClient }, children);
it("resolves the set of auto-router model groups from the deployment list", async () => {
(modelInfoCall as any).mockResolvedValue({
data: [
{ model_name: "smart-router", litellm_params: { model: "auto_router/complexity_router" } },
{ model_name: "claude-haiku", litellm_params: { model: "anthropic/claude-haiku-4-5" } },
],
total_count: 2,
current_page: 1,
total_pages: 1,
size: 1000,
});
const { result } = renderHook(() => useAutoRouterModelGroups(), { wrapper });
await waitFor(() => expect(result.current.size).toBe(1));
expect(result.current.has("smart-router")).toBe(true);
expect(result.current.has("claude-haiku")).toBe(false);
});
it("requests a single large page when the proxy reports only one page of deployments", async () => {
(modelInfoCall as any).mockResolvedValue({
data: [],
total_count: 0,
current_page: 1,
total_pages: 1,
size: 1000,
});
renderHook(() => useAutoRouterModelGroups(), { wrapper });
await waitFor(() => expect(modelInfoCall).toHaveBeenCalled());
expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 1, 1000);
expect(modelInfoCall).toHaveBeenCalledTimes(1);
});
it("follows total_pages so an auto-router past the first page is still found", async () => {
(modelInfoCall as any).mockImplementation((_t: string, _u: string, _r: string, page: number) => {
if (page === 1) {
return Promise.resolve({
data: [{ model_name: "claude-haiku", litellm_params: { model: "anthropic/claude-haiku-4-5" } }],
total_count: 3,
current_page: 1,
total_pages: 3,
size: 1000,
});
}
if (page === 2) {
return Promise.resolve({
data: [{ model_name: "claude-sonnet", litellm_params: { model: "anthropic/claude-sonnet-4-5" } }],
total_count: 3,
current_page: 2,
total_pages: 3,
size: 1000,
});
}
return Promise.resolve({
data: [{ model_name: "late-router", litellm_params: { model: "auto_router/complexity_router" } }],
total_count: 3,
current_page: 3,
total_pages: 3,
size: 1000,
});
});
const { result } = renderHook(() => useAutoRouterModelGroups(), { wrapper });
await waitFor(() => expect(result.current.size).toBe(1));
expect(result.current.has("late-router")).toBe(true);
expect(modelInfoCall).toHaveBeenCalledTimes(3);
expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 3, 1000);
});
it("returns an empty set before the model list resolves", () => {
(modelInfoCall as any).mockReturnValue(new Promise(() => {}));
const { result } = renderHook(() => useAutoRouterModelGroups(), { wrapper });
expect(result.current.size).toBe(0);
});
it("returns an empty set when the model list request fails", async () => {
(modelInfoCall as any).mockRejectedValue(new Error("boom"));
const { result } = renderHook(() => useAutoRouterModelGroups(), { wrapper });
await waitFor(() => expect(modelInfoCall).toHaveBeenCalled());
expect(result.current.size).toBe(0);
});
});

View file

@ -24,6 +24,7 @@ export interface PaginatedModelInfoResponse {
const modelKeys = createQueryKeys("models");
const modelHubKeys = createQueryKeys("modelHub");
const autoRouterKeys = createQueryKeys("autoRouterModelGroups");
const allProxyModelsKeys = createQueryKeys("allProxyModels");
const selectedTeamModelsKeys = createQueryKeys("selectedTeamModels");
const infiniteModelKeys = createQueryKeys("infiniteModels");
@ -59,6 +60,65 @@ export const useModelsInfo = (
});
};
const AUTO_ROUTER_MODEL_PREFIX = "auto_router/";
const AUTO_ROUTER_LOOKUP_PAGE_SIZE = 1000;
const NO_AUTO_ROUTERS: ReadonlySet<string> = new Set<string>();
export interface AutoRouterCandidateDeployment {
model_name?: string | null;
litellm_params?: { model?: string | null } | null;
}
export const isAutoRouterDeployment = (deployment: AutoRouterCandidateDeployment): boolean =>
Boolean(deployment?.litellm_params?.model?.startsWith(AUTO_ROUTER_MODEL_PREFIX));
export const selectAutoRouterModelGroups = (deployments: AutoRouterCandidateDeployment[]): ReadonlySet<string> =>
new Set(
deployments
.filter(isAutoRouterDeployment)
.map((deployment) => deployment.model_name)
.filter((modelName): modelName is string => Boolean(modelName)),
);
const fetchAllModelDeployments = async (
accessToken: string,
userId: string,
userRole: string,
): Promise<AutoRouterCandidateDeployment[]> => {
const firstPage: PaginatedModelInfoResponse = await modelInfoCall(
accessToken,
userId,
userRole,
1,
AUTO_ROUTER_LOOKUP_PAGE_SIZE,
);
const totalPages = firstPage?.total_pages ?? 1;
const remainingPages = await Promise.all(
Array.from({ length: Math.max(0, totalPages - 1) }, (_unused, index) =>
modelInfoCall(accessToken, userId, userRole, index + 2, AUTO_ROUTER_LOOKUP_PAGE_SIZE),
),
);
return [firstPage, ...remainingPages].flatMap(
(page: PaginatedModelInfoResponse) => page?.data ?? [],
) as AutoRouterCandidateDeployment[];
};
export const useAutoRouterModelGroups = (): ReadonlySet<string> => {
const { accessToken, userId, userRole } = useAuthorized();
const { data } = useQuery<AutoRouterCandidateDeployment[], Error, ReadonlySet<string>>({
queryKey: autoRouterKeys.list({
filters: {
...(userId && { userId }),
...(userRole && { userRole }),
},
}),
queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!),
enabled: Boolean(accessToken && userId && userRole),
select: selectAutoRouterModelGroups,
});
return data ?? NO_AUTO_ROUTERS;
};
export const useModelHub = () => {
const { accessToken } = useAuthorized();
return useQuery({

View file

@ -0,0 +1,62 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it, vi, beforeEach } from "vitest";
import { AutoRouterModelGroupsProvider, AutoRouterTag } from "./AutoRouterTag";
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
useAutoRouterModelGroups: vi.fn(),
}));
import { useAutoRouterModelGroups } from "@/app/(dashboard)/hooks/models/useModels";
const mockUseAutoRouterModelGroups = vi.mocked(useAutoRouterModelGroups);
const renderInProvider = (ui: React.ReactNode) =>
render(<AutoRouterModelGroupsProvider>{ui}</AutoRouterModelGroupsProvider>);
describe("AutoRouterTag", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("names the router that served the request", () => {
mockUseAutoRouterModelGroups.mockReturnValue(new Set(["smart-router"]));
renderInProvider(<AutoRouterTag modelGroup="smart-router" />);
const tag = screen.getByTitle('Routed by auto-router "smart-router"');
expect(tag).toHaveTextContent("smart-router");
expect(tag.querySelector("svg")).not.toBeNull();
});
it("does not tag a plain alias whose model group differs from the resolved model", () => {
mockUseAutoRouterModelGroups.mockReturnValue(new Set(["smart-router"]));
renderInProvider(<AutoRouterTag modelGroup="claude-haiku" />);
expect(screen.queryByText("claude-haiku")).not.toBeInTheDocument();
});
it("renders nothing while the model list is unavailable, so a routed row is never mislabelled", () => {
mockUseAutoRouterModelGroups.mockReturnValue(new Set<string>());
const { container } = renderInProvider(<AutoRouterTag modelGroup="smart-router" />);
expect(container).toBeEmptyDOMElement();
});
it.each([[undefined], [null], [""]])("renders nothing when the row carries no model group (%s)", (modelGroup) => {
mockUseAutoRouterModelGroups.mockReturnValue(new Set(["smart-router"]));
const { container } = renderInProvider(<AutoRouterTag modelGroup={modelGroup} />);
expect(container).toBeEmptyDOMElement();
});
it("renders nothing outside a provider instead of requiring a QueryClient", () => {
const { container } = render(<AutoRouterTag modelGroup="smart-router" />);
expect(container).toBeEmptyDOMElement();
expect(mockUseAutoRouterModelGroups).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,54 @@
"use client";
import { createContext, useContext, type ReactNode } from "react";
import { Waypoints } from "lucide-react";
import { useAutoRouterModelGroups } from "@/app/(dashboard)/hooks/models/useModels";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/cva.config";
const NO_AUTO_ROUTERS: ReadonlySet<string> = new Set<string>();
const AutoRouterModelGroupsContext = createContext<ReadonlySet<string>>(NO_AUTO_ROUTERS);
export function AutoRouterModelGroupsProvider({ children }: { children: ReactNode }) {
const autoRouterModelGroups = useAutoRouterModelGroups();
return (
<AutoRouterModelGroupsContext.Provider value={autoRouterModelGroups}>
{children}
</AutoRouterModelGroupsContext.Provider>
);
}
export function useIsAutoRoutedModelGroup(modelGroup?: string | null): boolean {
const autoRouterModelGroups = useContext(AutoRouterModelGroupsContext);
return Boolean(modelGroup) && autoRouterModelGroups.has(modelGroup as string);
}
export function AutoRouterIcon({ size = 12, className }: { size?: number; className?: string }) {
return <Waypoints size={size} className={className} aria-hidden />;
}
export interface AutoRouterTagProps {
modelGroup?: string | null;
className?: string;
}
export function AutoRouterTag({ modelGroup, className }: AutoRouterTagProps) {
const isAutoRouted = useIsAutoRoutedModelGroup(modelGroup);
if (!isAutoRouted) return null;
return (
<Badge
variant="secondary"
title={`Routed by auto-router "${modelGroup}"`}
className={cn("gap-1.5 px-2.5 py-1 text-sm font-normal text-foreground", className)}
>
<Waypoints aria-hidden />
{modelGroup}
</Badge>
);
}

View file

@ -1,3 +1,10 @@
export {
AutoRouterTag,
AutoRouterIcon,
AutoRouterModelGroupsProvider,
useIsAutoRoutedModelGroup,
type AutoRouterTagProps,
} from "./AutoRouterTag";
export { CellTooltip } from "./cell_tooltip";
export { DateCell, formatCellDate, formatFullTimestamp, type DatePrecision } from "./date_cell";
export { IdCell, type IdCellVariant } from "./id_cell";

View file

@ -2,6 +2,7 @@ import { Button, Space, Tag, Tooltip, Typography } from "antd";
import { CloseOutlined, UpOutlined, DownOutlined } from "@ant-design/icons";
import moment from "moment";
import { LogEntry } from "../columns";
import { AutoRouterTag } from "@/components/shared/table_cells";
import { getProviderLogoAndName } from "../../provider_info_helpers";
import {
DRAWER_HEADER_PADDING,
@ -57,6 +58,7 @@ export function DrawerHeader({
{/* Row 0: Model + Provider with Logo */}
<ModelProviderSection
model={log.model}
modelGroup={log.model_group}
providerLogo={providerInfo?.logo}
providerName={providerInfo?.displayName}
/>
@ -80,10 +82,12 @@ export function DrawerHeader({
*/
function ModelProviderSection({
model,
modelGroup,
providerLogo,
providerName,
}: {
model: string;
modelGroup?: string;
providerLogo?: string;
providerName?: string;
}) {
@ -109,6 +113,7 @@ function ModelProviderSection({
{providerName}
</Text>
)}
<AutoRouterTag modelGroup={modelGroup} />
</Space>
</Space>
);

View file

@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest";
import { LogDetailsDrawer } from "./LogDetailsDrawer";
import { sessionSpendLogsCall } from "../../networking";
import { LogEntry } from "../columns";
import { AutoRouterModelGroupsProvider } from "@/components/shared/table_cells";
vi.mock("../../networking", () => ({
sessionSpendLogsCall: vi.fn(),
@ -22,6 +23,10 @@ vi.mock("./DrawerHeader", () => ({
DrawerHeader: () => null,
}));
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
useAutoRouterModelGroups: vi.fn(() => new Set(["smart-router"])),
}));
const makeLog = (overrides: Partial<LogEntry>): LogEntry => ({
request_id: "req",
api_key: "",
@ -118,3 +123,44 @@ describe("LogDetailsDrawer session sidebar sorting", () => {
await waitFor(() => expect(sidebarEventNames()).toEqual(["tool-early", "llm-late", "llm-early", "tool-late"]));
});
});
describe("LogDetailsDrawer session sidebar auto-router icon", () => {
const routedSessionLogs = [
makeLog({ request_id: "routed", model: "claude-opus-4-8", model_group: "smart-router" }),
makeLog({ request_id: "direct", model: "claude-haiku-4-5", model_group: "claude-haiku" }),
];
const renderRoutedSession = () => {
vi.mocked(sessionSpendLogsCall).mockResolvedValue({ data: routedSessionLogs, total: 2, total_pages: 1 });
const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
return render(
<QueryClientProvider client={queryClient}>
<AutoRouterModelGroupsProvider>
<LogDetailsDrawer open onClose={() => {}} logEntry={null} sessionId="session-1" accessToken="token" />
</AutoRouterModelGroupsProvider>
</QueryClientProvider>,
);
};
const rowFor = (label: string): HTMLElement => {
const row = Array.from(document.body.querySelectorAll("button")).find((button) =>
button.textContent?.includes(label),
);
if (!row) throw new Error(`no sidebar row for ${label}`);
return row;
};
it("marks the auto-routed entry with the router icon and leaves a direct call on the default icon", async () => {
renderRoutedSession();
await waitFor(() => expect(screen.queryByText("claude-opus-4-8")).not.toBeNull());
const routedRow = rowFor("claude-opus-4-8");
const directRow = rowFor("claude-haiku-4-5");
expect(routedRow.querySelector(".lucide-waypoints")).not.toBeNull();
expect(routedRow.querySelector(".lucide-sparkles")).toBeNull();
expect(directRow.querySelector(".lucide-sparkles")).not.toBeNull();
expect(directRow.querySelector(".lucide-waypoints")).toBeNull();
});
});

View file

@ -3,6 +3,7 @@ import { Button, Drawer, Segmented } from "antd";
import { CheckOutlined, CopyOutlined, LeftOutlined, RightOutlined } from "@ant-design/icons";
import { Bot, Sparkles, Wrench } from "lucide-react";
import { LogEntry } from "../columns";
import { AutoRouterIcon, useIsAutoRoutedModelGroup } from "@/components/shared/table_cells";
import { AGENT_CALL_TYPES, MCP_CALL_TYPES } from "../constants";
import { getEventDisplayName } from "../utils";
import { DrawerHeader } from "./DrawerHeader";
@ -46,9 +47,17 @@ interface TraceEventRowProps {
onClick: () => void;
}
const TRACE_EVENT_ICON_CLASS = "text-slate-500 shrink-0";
function TraceEventIcon({ callType, isAutoRouted }: { callType: string; isAutoRouted: boolean }) {
if (MCP_CALL_TYPES.includes(callType)) return <Wrench size={12} className={TRACE_EVENT_ICON_CLASS} />;
if (AGENT_CALL_TYPES.includes(callType)) return <Bot size={12} className={TRACE_EVENT_ICON_CLASS} />;
if (isAutoRouted) return <AutoRouterIcon size={12} className={TRACE_EVENT_ICON_CLASS} />;
return <Sparkles size={12} className={TRACE_EVENT_ICON_CLASS} />;
}
function TraceEventRow({ row, isSelected, onClick }: TraceEventRowProps) {
const isMcp = MCP_CALL_TYPES.includes(row.call_type);
const isAgent = AGENT_CALL_TYPES.includes(row.call_type);
const isAutoRouted = useIsAutoRoutedModelGroup(row.model_group);
const durationValue =
row.request_duration_ms != null
? (row.request_duration_ms / 1000).toFixed(3)
@ -65,13 +74,7 @@ function TraceEventRow({ row, isSelected, onClick }: TraceEventRowProps) {
onClick={onClick}
>
<div className="flex items-center gap-1">
{isMcp ? (
<Wrench size={12} className="text-slate-500 shrink-0" />
) : isAgent ? (
<Bot size={12} className="text-slate-500 shrink-0" />
) : (
<Sparkles size={12} className="text-slate-500 shrink-0" />
)}
<TraceEventIcon callType={row.call_type} isAutoRouted={isAutoRouted} />
<span className="text-xs font-medium text-slate-900 truncate">
{getEventDisplayName(row.call_type, row.model)}
</span>

View file

@ -5,6 +5,7 @@ import type { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } fr
import moment from "moment";
import { useCallback, useEffect, useMemo, useState } from "react";
import { AutoRouterModelGroupsProvider } from "@/components/shared/table_cells";
import { internalUserRoles } from "../../utils/roles";
import type { KeyResponse } from "../key_team_helpers/key_list";
import { keyInfoV1Call } from "../networking";
@ -206,7 +207,7 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID,
}
return (
<>
<AutoRouterModelGroupsProvider>
<div className="flex items-center justify-between mb-4">
<h1 className="text-xl font-semibold">Request Logs</h1>
</div>
@ -263,6 +264,6 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID,
onSelectLog={setSelectedLog}
startTime={moment(startTime).utc().format("YYYY-MM-DD HH:mm:ss")}
/>
</>
</AutoRouterModelGroupsProvider>
);
}

View file

@ -16,6 +16,7 @@ export type LogEntry = {
team_id: string;
model: string;
model_id: string;
model_group?: string;
api_base?: string;
call_type: string;
spend: number;