diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx index 9cfc375aada..d27416540af 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx @@ -54,6 +54,8 @@ const modelInfoResponse = (deployments: Deployment[]) => JSON.stringify({ data: deployments.map(({ model_name, mode }) => ({ model_name, + model_group: model_name, + mode, litellm_params: { model: `openai/${model_name}` }, model_info: { id: model_name, mode }, })), @@ -67,7 +69,7 @@ const EXPIRED_KEY_RESPONSE = { JSON.stringify({ error: { message: "Authentication Error - Expired Key.", type: "expired_key", code: "401" } }), } as Response; -const isModelLookup = (input: RequestInfo | URL) => String(input).endsWith("/model/info"); +const isModelLookup = (input: RequestInfo | URL) => /\/model(_group)?\/info$/.test(String(input)); const lookupAuthHeaders = (calls: Parameters[]) => calls.map(([, init]) => Object.values((init?.headers ?? {}) as Record)); @@ -218,7 +220,10 @@ describe("SystemOneUI integration", () => { render(); await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); - expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([expect.arrayContaining(["Bearer session-key"])]); + expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([ + expect.arrayContaining(["Bearer session-key"]), + expect.arrayContaining(["Bearer session-key"]), + ]); await user.click(screen.getByRole("button", { name: "Send" })); expect(await screen.findByText("Selected choice")).toBeInTheDocument(); @@ -474,11 +479,11 @@ describe("SystemOneUI integration", () => { expect(await screen.findByRole("link", { name: "Add a decision model" })).toBeInTheDocument(); first.unmount(); - mockModelLookup.mockResolvedValueOnce(modelInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); rtlRender(page); const picker = await screen.findByRole("combobox", { name: "Decision model" }); await waitFor(() => expect(picker).toHaveValue("jev-latest")); - expect(mockModelLookup).toHaveBeenCalledTimes(2); + expect(mockModelLookup).toHaveBeenCalledTimes(4); }); it("routes a configured TypeSafe model through the same proxy endpoint", async () => { @@ -518,7 +523,10 @@ describe("SystemOneUI integration", () => { await user.type(screen.getByLabelText("Virtual Key", { exact: true }), "sk-typed-key"); await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); - expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([expect.arrayContaining(["Bearer sk-typed-key"])]); + expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([ + expect.arrayContaining(["Bearer sk-typed-key"]), + expect.arrayContaining(["Bearer sk-typed-key"]), + ]); }); it("keeps the admin signed in when a typed virtual key has expired", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts index c37af290d99..11f87a11c0a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts @@ -196,6 +196,11 @@ describe("/v1/decisions requests", () => { describe("fetchDecisionModels", () => { const mockFetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise>(); const modelInfo = (data: unknown) => ({ ok: true, text: async () => JSON.stringify({ data }) }) as Response; + const group = (model_group: string, mode: string | null) => ({ model_group, mode }); + const serve = (deployments: unknown[], groups: unknown[] = []) => + mockFetch.mockImplementation(async (input) => + String(input).endsWith("/model_group/info") ? modelInfo(groups) : modelInfo(deployments), + ); const deployment = (model_name: string, mode: string | null, supported_endpoints?: string[]) => ({ model_name, litellm_params: { model: `openai/${model_name}` }, @@ -212,50 +217,58 @@ describe("fetchDecisionModels", () => { }); it("lists the key's decision models from the proxy that answers the request, sorted by name", async () => { - mockFetch.mockResolvedValueOnce( - modelInfo([ - deployment("pplx-decider", "evaluation"), - deployment("gpt-5.5", "chat", ["/v1/chat/completions", "/v1/responses"]), - deployment("jev-latest", "evaluation"), - deployment("unknown-mode", null), - ]), - ); + serve([ + deployment("pplx-decider", "evaluation"), + deployment("gpt-5.5", "chat", ["/v1/chat/completions", "/v1/responses"]), + deployment("jev-latest", "evaluation"), + deployment("unknown-mode", null), + ]); await expect(fetchDecisionModels("virtual-key", "https://custom.example.com/")).resolves.toEqual([ "jev-latest", "pplx-decider", ]); - expect(mockFetch).toHaveBeenCalledWith( - "https://custom.example.com/model/info", - expect.objectContaining({ - method: "GET", - headers: expect.objectContaining({ Authorization: "Bearer virtual-key" }), - }), + for (const path of ["/model/info", "/model_group/info"]) { + expect(mockFetch).toHaveBeenCalledWith( + `https://custom.example.com${path}`, + expect.objectContaining({ + method: "GET", + headers: expect.objectContaining({ Authorization: "Bearer virtual-key" }), + }), + ); + } + }); + + it("lists a routing group whose members serve decisions, next to the deployments", async () => { + serve( + [deployment("decision-primary", "evaluation")], + [group("decision-balanced", "evaluation"), group("chat-pool", "chat"), group("unknown-mode", null)], ); + + await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["decision-balanced", "decision-primary"]); }); it("lists a chat-mode model whose cost map entry serves a decision endpoint", async () => { - mockFetch.mockResolvedValueOnce( - modelInfo([ - deployment("gpt-6-luna", "chat", ["/v1/chat/completions", "/v1/responses", "/v1/decisions"]), - deployment("clef", "chat", ["/v1/systemone"]), - deployment("gpt-5.5", "chat", ["/v1/chat/completions"]), - ]), - ); + serve([ + deployment("gpt-6-luna", "chat", ["/v1/chat/completions", "/v1/responses", "/v1/decisions"]), + deployment("clef", "chat", ["/v1/systemone"]), + deployment("gpt-5.5", "chat", ["/v1/chat/completions"]), + ]); await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["clef", "gpt-6-luna"]); }); it("lists a model group once when several deployments share its name", async () => { - mockFetch.mockResolvedValueOnce( - modelInfo([deployment("jev-latest", "evaluation"), deployment("jev-latest", "evaluation")]), + serve( + [deployment("jev-latest", "evaluation"), deployment("jev-latest", "evaluation")], + [group("jev-latest", "evaluation")], ); await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["jev-latest"]); }); it("rejects a response that is not a model list", async () => { - mockFetch.mockResolvedValueOnce(modelInfo([{ model_info: { mode: "evaluation" } }])); + serve([{ model_info: { mode: "evaluation" } }]); await expect(fetchDecisionModels("virtual-key")).rejects.toThrow(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts index 1236ca5b4f7..3ac9286d89c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking"; import { withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { createApiClient } from "@/lib/http/client"; -import { isDecisionModel } from "@/lib/decisionModels"; +import { isDecisionMode, isDecisionModel } from "@/lib/decisionModels"; import { systemOneResponseSchema, type PlaygroundRequest, @@ -36,6 +36,10 @@ const modelInfoSchema = z.object({ ), }); +const modelGroupInfoSchema = z.object({ + data: z.array(z.object({ model_group: z.string(), mode: z.string().nullish() })), +}); + function proxyClient(customBaseUrl?: string) { const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl; @@ -57,15 +61,21 @@ export async function fetchDecisionModels( customBaseUrl?: string, signal?: AbortSignal, ): Promise { - const body = await proxyClient(customBaseUrl).get("/model/info", { - headers: authHeaders(accessToken), - signal, - }); - const decisionModelNames = modelInfoSchema - .parse(body) + const client = proxyClient(customBaseUrl); + const options = { headers: authHeaders(accessToken), signal }; + const [deployments, groups] = await Promise.all([ + client.get("/model/info", options), + client.get("/model_group/info", options), + ]); + const deploymentNames = modelInfoSchema + .parse(deployments) .data.filter((deployment) => isDecisionModel(deployment.model_info ?? {})) .map((deployment) => deployment.model_name); - return [...new Set(decisionModelNames)].sort((a, b) => a.localeCompare(b)); + const groupNames = modelGroupInfoSchema + .parse(groups) + .data.filter((group) => isDecisionMode(group.mode)) + .map((group) => group.model_group); + return [...new Set([...deploymentNames, ...groupNames])].sort((a, b) => a.localeCompare(b)); } async function sendDecisionRequest(