From ddc13b331a83d49eb7c101eebde4d830ae86ea71 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:04:04 -0700 Subject: [PATCH] fix(ui): probe auto-router tiers via real proxy routing, not /health/test_connection Live testing showed the first cut was broken: /health/test_connection merges {...configParams, ...requestParams}, so passing the public model_group name as the request model overrode the resolved provider model and every tier failed with "LLM Provider NOT provided". The frontend only has the public group name, not the underlying litellm_params, so it cannot build the request that endpoint needs. Switch to testing each model group the way production actually routes it: send a minimal request to /v1/chat/completions (or /v1/embeddings for the embedding model) by public group name through the shared apiClient. The router resolves the group, credentials, and provider itself, so a green row means the tier is genuinely reachable. Verified live: voyage embedding returns 200, a tier with a bad key returns the real provider auth error. Also address Greptile feedback: rows now update progressively as each probe settles instead of all at once, and TIER_ORDER is derived through a `satisfies Record` guard so adding a tier without listing it is a compile error. --- .../auto_router_connection_test.test.tsx | 32 ++++++----- .../add_model/auto_router_connection_test.tsx | 55 +++++-------------- .../build_auto_router_test_targets.ts | 9 ++- .../src/components/networking.tsx | 28 ++++++++++ 4 files changed, 69 insertions(+), 55 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.test.tsx b/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.test.tsx index 9b872d5edee..b07270b5ced 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.test.tsx @@ -7,11 +7,11 @@ vi.mock("../networking", async () => { const actual = await vi.importActual("../networking"); return { ...actual, - testConnectionRequest: vi.fn(), + testModelGroupConnection: vi.fn(), }; }); -const getMock = async () => vi.mocked((await import("../networking")).testConnectionRequest); +const getMock = async () => vi.mocked((await import("../networking")).testModelGroupConnection); const targets: AutoRouterTestTarget[] = [ { labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" }, @@ -32,12 +32,12 @@ describe("AutoRouterConnectionTest", () => { await waitFor(() => expect(mock).toHaveBeenCalledTimes(3)); - expect(mock).toHaveBeenCalledWith("sk-test", { model: "gpt-4o-mini" }, {}, "chat"); - expect(mock).toHaveBeenCalledWith("sk-test", { model: "claude-sonnet-4" }, {}, "chat"); - expect(mock).toHaveBeenCalledWith("sk-test", { model: "voyage-3-5" }, {}, "embedding"); + expect(mock).toHaveBeenCalledWith("sk-test", "gpt-4o-mini", "chat"); + expect(mock).toHaveBeenCalledWith("sk-test", "claude-sonnet-4", "chat"); + expect(mock).toHaveBeenCalledWith("sk-test", "voyage-3-5", "embedding"); }); - it("shows a success indicator per target when the health check passes", async () => { + it("shows a success indicator per target when the routing probe passes", async () => { const mock = await getMock(); mock.mockResolvedValue({ status: "success" }); @@ -48,12 +48,14 @@ describe("AutoRouterConnectionTest", () => { expect(screen.getByText("MEDIUM, COMPLEX")).toBeInTheDocument(); }); - it("renders the provider error message for a failing target while others pass", async () => { + it("renders the provider error message (litellm prefix stripped) for a failing target while others pass", async () => { const mock = await getMock(); - mock.mockImplementation((_token, litellmParams) => - litellmParams.model === "claude-sonnet-4" - ? Promise.resolve({ status: "error", result: { error: "litellm.AuthenticationError: invalid api key" } }) - : Promise.resolve({ status: "success" }), + mock.mockImplementation((_token, modelGroup) => + Promise.resolve( + modelGroup === "claude-sonnet-4" + ? { status: "error", error: "litellm.AuthenticationError: invalid api key" } + : { status: "success" }, + ), ); renderWithProviders(); @@ -64,9 +66,9 @@ describe("AutoRouterConnectionTest", () => { expect(screen.getAllByTestId("test-status-success")).toHaveLength(2); }); - it("surfaces a thrown network error as a failing row", async () => { + it("renders a non-litellm error string verbatim", async () => { const mock = await getMock(); - mock.mockRejectedValue(new Error("Network request failed")); + mock.mockResolvedValue({ status: "error", error: "Connection test failed: 404 Not Found" }); renderWithProviders( { />, ); - await waitFor(() => expect(screen.getByTestId("test-error-message")).toHaveTextContent("Network request failed")); + await waitFor(() => + expect(screen.getByTestId("test-error-message")).toHaveTextContent("Connection test failed: 404 Not Found"), + ); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.tsx b/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.tsx index 77588006e35..5badd155da8 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.tsx @@ -1,7 +1,7 @@ import React from "react"; import { Typography } from "antd"; import { CheckCircleTwoTone, CloseCircleTwoTone, LoadingOutlined } from "@ant-design/icons"; -import { testConnectionRequest } from "../networking"; +import { testModelGroupConnection, ModelGroupConnectionResult } from "../networking"; import { AutoRouterTestTarget } from "./build_auto_router_test_targets"; const { Text } = Typography; @@ -12,45 +12,13 @@ interface AutoRouterConnectionTestProps { onTestComplete?: () => void; } -type TargetResult = { status: "pending" } | { status: "success" } | { status: "error"; error: string }; - -interface NormalizedResponse { - ok: boolean; - error?: string; -} - -const normalizeTestConnectionResponse = (response: unknown): NormalizedResponse => { - if (typeof response !== "object" || response === null) { - return { ok: false, error: "Unexpected response from connection test" }; - } - const record = response as Record; - if (record.status === "success") { - return { ok: true }; - } - const result = - typeof record.result === "object" && record.result !== null ? (record.result as Record) : {}; - const resultError = typeof result.error === "string" ? result.error : undefined; - const recordMessage = typeof record.message === "string" ? record.message : undefined; - return { ok: false, error: resultError ?? recordMessage ?? "Unknown error" }; -}; +type TargetResult = { status: "pending" } | ModelGroupConnectionResult; const cleanErrorMessage = (error: string): string => { const mainError = error.split("stack trace:")[0].trim(); return mainError.replace(/^litellm\.(.*?)Error: /, ""); }; -const runTarget = async (accessToken: string, target: AutoRouterTestTarget): Promise => { - try { - const response = await testConnectionRequest(accessToken, { model: target.modelGroup }, {}, target.mode); - const normalized = normalizeTestConnectionResponse(response); - return normalized.ok - ? { status: "success" } - : { status: "error", error: cleanErrorMessage(normalized.error ?? "Unknown error") }; - } catch (error) { - return { status: "error", error: cleanErrorMessage(error instanceof Error ? error.message : String(error)) }; - } -}; - const AutoRouterConnectionTest: React.FC = ({ accessToken, targets, @@ -61,16 +29,22 @@ const AutoRouterConnectionTest: React.FC = ({ React.useEffect(() => { let cancelled = false; const run = async () => { - const settled = await Promise.all(targets.map((target) => runTarget(accessToken, target))); - if (cancelled) return; - setResults(settled); - if (onTestComplete) onTestComplete(); + await Promise.all( + targets.map(async (target, index) => { + const result = await testModelGroupConnection(accessToken, target.modelGroup, target.mode); + if (cancelled) return; + const cleaned: TargetResult = + result.status === "error" ? { status: "error", error: cleanErrorMessage(result.error) } : result; + setResults((prev) => prev.map((r, i) => (i === index ? cleaned : r))); + }), + ); + if (!cancelled && onTestComplete) onTestComplete(); }; run(); return () => { cancelled = true; }; - // eslint-disable-next-line react-hooks/exhaustive-deps -- probes run once per mount; the parent remounts via `key` to start a fresh test, and re-running on prop identity changes would refire paid health checks + // eslint-disable-next-line react-hooks/exhaustive-deps -- probes run once per mount; the parent remounts via `key` to start a fresh test, and re-running on prop identity changes would refire paid requests }, []); if (targets.length === 0) { @@ -80,7 +54,8 @@ const AutoRouterConnectionTest: React.FC = ({ return (
- Each configured tier routes to a saved model group. Test Connection runs a live health check against each one. + Each configured tier routes to a saved model group. Test Connection sends a minimal request through the proxy to + each one, exactly as the auto router would. {targets.map((target, index) => { const result = results[index] ?? { status: "pending" }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts index 0104b6bc9c5..b2a3cc10012 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_test_targets.ts @@ -14,7 +14,14 @@ export interface BuildAutoRouterTestTargetsParams { embeddingModel: string | undefined; } -const TIER_ORDER: (keyof ComplexityTiers)[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; +// Keys drive iteration order; `satisfies Record` makes it a +// compile error to add a tier to ComplexityTiers without listing it here (and vice versa). +const TIER_ORDER = Object.keys({ + SIMPLE: null, + MEDIUM: null, + COMPLEX: null, + REASONING: null, +} satisfies Record) as (keyof ComplexityTiers)[]; export const buildAutoRouterTestTargets = ({ tiers, diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index da6bb079876..ae263727569 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2315,6 +2315,34 @@ export const testConnectionRequest = async ( } }; +export type ModelGroupConnectionResult = { status: "success" } | { status: "error"; error: string }; + +/** + * Test an existing model group by routing a minimal request through the proxy + * exactly as production would (by public model_group name). Unlike + * /health/test_connection, this needs no litellm_params resolution: the router + * resolves the group, credentials, and provider. Used by the auto-router Test + * Connection to probe each tier's model group and the embedding model. + */ +export const testModelGroupConnection = async ( + accessToken: string, + modelGroup: string, + mode: "chat" | "embedding", +): Promise => { + const path = mode === "embedding" ? "/v1/embeddings" : "/v1/chat/completions"; + const body = + mode === "embedding" + ? { model: modelGroup, input: "test from litellm" } + : { model: modelGroup, messages: [{ role: "user", content: "test from litellm" }], max_tokens: 1 }; + + try { + await apiClient.post(path, { accessToken, body }); + return { status: "success" }; + } catch (error) { + return { status: "error", error: error instanceof Error ? error.message : String(error) }; + } +}; + // ... existing code ... export const keyInfoV1Call = async (accessToken: string, key: string) => { try {