diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1c7cf54ab4d..16596835c86 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16400,7 +16400,8 @@ async def model_info_v1( # when v1 resolved models only via public model_name strings. all_models: list[dict] = copy.deepcopy(llm_router.model_list) alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) - all_models.extend(alias_models) + routing_group_models: Final = copy.deepcopy(list(llm_router.get_model_list_from_routing_groups())) + all_models.extend([*alias_models, *routing_group_models]) all_models = expand_wildcard_deployments_for_model_info(all_models) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index fae4a6470cf..14a3d394fc0 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -596,6 +596,43 @@ def test_model_info_team_key_cannot_see_other_teams_byok_model( assert _BYOK_INTERNAL_NAME not in surfaced_names +@pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"]) +def test_model_info_lists_routing_group_rows(client, auth_as, monkeypatch, path): + """A routing group is callable by its own name, so /model/info surfaces the + rows a configured router materializes for it next to the member deployment. + """ + from litellm.proxy._types import LitellmUserRoles + + member = { + "model_name": "gpt-6-luna", + "litellm_params": {"model": "openai/gpt-6-luna", "api_key": "sk-test"}, + "model_info": {"id": "luna-deployment-id", "mode": "chat"}, + } + router = litellm.Router( + model_list=[member], + routing_groups=[ + { + "group_name": "decision-balanced", + "models": ["gpt-6-luna"], + "routing_strategy": "simple-shuffle", + } + ], + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", [member]) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + + with auth_as(role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"): + response = client.get(path) + + assert response.status_code == 200, response.text + surfaced = {m["model_name"]: m for m in response.json()["data"]} + assert set(surfaced) == {"gpt-6-luna", "decision-balanced"} + assert surfaced["decision-balanced"]["model_info"]["id"] == "luna-deployment-id" + assert surfaced["decision-balanced"]["model_info"]["mode"] == "chat" + + # --------------------------------------------------------------------------- # GET /model_group/info # --------------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx index 8158e5782bb..59d16fa4d15 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx @@ -16,15 +16,15 @@ import type { SystemOneResponse } from "./lib/schemas"; import type { OpenAIDecisionsResponse } from "./lib/openAIDecisions"; import { curlPayload } from "./lib/curlCommand"; -interface ModelGroupInfo { - model_group: string; +interface Deployment { + model_name: string; mode: string; } -const DECISION_AND_CHAT_MODELS: ModelGroupInfo[] = [ - { model_group: "gpt-5.5", mode: "chat" }, - { model_group: "jev-latest", mode: "evaluation" }, - { model_group: "pplx-decider", mode: "evaluation" }, +const DECISION_AND_CHAT_MODELS: Deployment[] = [ + { model_name: "gpt-5.5", mode: "chat" }, + { model_name: "jev-latest", mode: "evaluation" }, + { model_name: "pplx-decider", mode: "evaluation" }, ]; const responseBody: SystemOneResponse = { @@ -46,8 +46,19 @@ const render = (ui: ReactElement) => , ); -const modelGroupInfoResponse = (groups: ModelGroupInfo[]) => - ({ ok: true, status: 200, text: async () => JSON.stringify({ data: groups }) }) as Response; +const modelInfoResponse = (deployments: Deployment[]) => + ({ + ok: true, + status: 200, + text: async () => + JSON.stringify({ + data: deployments.map(({ model_name, mode }) => ({ + model_name, + litellm_params: { model: `openai/${model_name}` }, + model_info: { id: model_name, mode }, + })), + }), + }) as Response; const EXPIRED_KEY_RESPONSE = { ok: false, @@ -56,7 +67,7 @@ const EXPIRED_KEY_RESPONSE = { JSON.stringify({ error: { message: "Authentication Error - Expired Key.", type: "expired_key", code: "401" } }), } as Response; -const isModelLookup = (input: RequestInfo | URL) => String(input).endsWith("/model_group/info"); +const isModelLookup = (input: RequestInfo | URL) => String(input).endsWith("/model/info"); const lookupAuthHeaders = (calls: Parameters[]) => calls.map(([, init]) => Object.values((init?.headers ?? {}) as Record)); @@ -99,7 +110,7 @@ describe("SystemOneUI integration", () => { isModelLookup(input) ? mockModelLookup(input, init) : mockFetch(input, init), ); mockFetch.mockResolvedValue(createResponse(responseBody)); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse([])); + mockModelLookup.mockResolvedValue(modelInfoResponse([])); }); afterEach(() => { @@ -147,7 +158,7 @@ describe("SystemOneUI integration", () => { it("keeps the preset label after picking a model and keeps that model when clearing", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); render(); const modelPicker = screen.getByRole("combobox", { name: "Decision model" }); await waitFor(() => expect(modelPicker).toHaveValue("jev-latest")); @@ -203,7 +214,7 @@ describe("SystemOneUI integration", () => { it("opens on /v1/systemone with the first decision model the key can call", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); render(); await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); @@ -221,7 +232,7 @@ describe("SystemOneUI integration", () => { it("sends each endpoint's real shape and switches existing responses between cards and JSON without resending", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); render(); await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); fireEvent.change(screen.getByRole("textbox", { name: "Input" }), { target: { value: "An outage" } }); @@ -431,7 +442,7 @@ describe("SystemOneUI integration", () => { it("offers only decision models and writes the picked one into the request", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); render(); const picker = await screen.findByRole("combobox", { name: "Decision model" }); @@ -451,11 +462,28 @@ describe("SystemOneUI integration", () => { expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("pplx-decider"); }); + it("looks the decision models up again when the playground is reopened within the cache window", async () => { + mockModelLookup.mockResolvedValueOnce(modelInfoResponse([{ model_name: "gpt-5.5", mode: "chat" }])); + const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } }); + const page = ( + + + + ); + const first = rtlRender(page); + expect(await screen.findByRole("link", { name: "Add a decision model" })).toBeInTheDocument(); + first.unmount(); + + mockModelLookup.mockResolvedValueOnce(modelInfoResponse(DECISION_AND_CHAT_MODELS)); + rtlRender(page); + const picker = await screen.findByRole("combobox", { name: "Decision model" }); + await waitFor(() => expect(picker).toHaveValue("jev-latest")); + expect(mockModelLookup).toHaveBeenCalledTimes(2); + }); + it("routes a configured TypeSafe model through the same proxy endpoint", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue( - modelGroupInfoResponse([{ model_group: "typesafe/jev-latest", mode: "evaluation" }]), - ); + mockModelLookup.mockResolvedValue(modelInfoResponse([{ model_name: "typesafe/jev-latest", mode: "evaluation" }])); render(); await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("typesafe/jev-latest"), @@ -484,7 +512,7 @@ describe("SystemOneUI integration", () => { it("looks up a typed virtual key's decision models once, after typing settles", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); render(); await user.type(screen.getByLabelText("Virtual Key", { exact: true }), "sk-typed-key"); @@ -515,7 +543,7 @@ describe("SystemOneUI integration", () => { fireEvent.change(screen.getByRole("textbox", { name: "Input" }), { target: { value: "edited" } }); await act(async () => { - models.resolve(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + models.resolve(modelInfoResponse(DECISION_AND_CHAT_MODELS)); await new Promise((resolve) => setTimeout(resolve, 0)); }); @@ -544,7 +572,7 @@ describe("SystemOneUI integration", () => { expect(mockFetch.mock.calls[0]?.[0]).toMatch(new RegExp(`${endpoint}$`)); expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).not.toHaveProperty("model"); await act(async () => { - models.resolve(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + models.resolve(modelInfoResponse(DECISION_AND_CHAT_MODELS)); await models.promise; }); expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue(""); @@ -616,7 +644,7 @@ describe("SystemOneUI integration", () => { it("uses the requested model when the response omits it", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); mockFetch.mockResolvedValueOnce( createResponse({ answers: responseBody.answers, @@ -861,7 +889,7 @@ describe("SystemOneUI integration", () => { it("edits score levels and keeps instructions when a question changes type", async () => { const user = userEvent.setup(); - mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS)); render(); await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/useDecisionModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/useDecisionModels.ts index c694b6384e5..51477554f18 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/useDecisionModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/useDecisionModels.ts @@ -26,6 +26,7 @@ export function useDecisionModels( enabled: lookupApiKey !== "", retry: false, staleTime: 60 * 1000, + refetchOnMount: "always" as const, }; const query = useQuery(queryOptions); return { decisionModels: query.data ?? NO_DECISION_MODELS, isLoaded: query.isSuccess }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts index 720152d3143..ae5637f4f27 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts @@ -195,7 +195,12 @@ describe("/v1/decisions requests", () => { describe("fetchDecisionModels", () => { const mockFetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise>(); - const modelGroupInfo = (data: unknown) => ({ ok: true, text: async () => JSON.stringify({ data }) }) as Response; + const modelInfo = (data: unknown) => ({ ok: true, text: async () => JSON.stringify({ data }) }) as Response; + const deployment = (model_name: string, mode: string | null, supported_endpoints?: string[]) => ({ + model_name, + litellm_params: { model: `openai/${model_name}` }, + model_info: { id: `${model_name}-${mode}`, mode, supported_endpoints }, + }); beforeEach(() => { vi.stubGlobal("fetch", mockFetch); @@ -208,11 +213,11 @@ describe("fetchDecisionModels", () => { it("lists the key's decision models from the proxy that answers the request, sorted by name", async () => { mockFetch.mockResolvedValueOnce( - modelGroupInfo([ - { model_group: "pplx-decider", mode: "evaluation" }, - { model_group: "gpt-5.5", mode: "chat" }, - { model_group: "jev-latest", mode: "evaluation" }, - { model_group: "unknown-mode", mode: null }, + modelInfo([ + deployment("pplx-decider", "evaluation"), + deployment("gpt-5.5", "chat", ["/v1/chat/completions", "/v1/responses"]), + deployment("jev-latest", "evaluation"), + deployment("unknown-mode", null), ]), ); @@ -221,7 +226,7 @@ describe("fetchDecisionModels", () => { "pplx-decider", ]); expect(mockFetch).toHaveBeenCalledWith( - "https://custom.example.com/model_group/info", + "https://custom.example.com/model/info", expect.objectContaining({ method: "GET", headers: expect.objectContaining({ Authorization: "Bearer virtual-key" }), @@ -229,8 +234,40 @@ describe("fetchDecisionModels", () => { ); }); - it("rejects a response that is not a model group list", async () => { - mockFetch.mockResolvedValueOnce(modelGroupInfo([{ mode: "evaluation" }])); + it("lists a routing group row the proxy materializes over a default-mode decision deployment", async () => { + mockFetch.mockResolvedValueOnce( + modelInfo([ + deployment("gpt-6-luna", "chat", ["/v1/chat/completions", "/v1/decisions"]), + deployment("decision-balanced", "chat", ["/v1/chat/completions", "/v1/decisions"]), + deployment("chat-pool", "chat", ["/v1/chat/completions"]), + ]), + ); + + await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["decision-balanced", "gpt-6-luna"]); + }); + + it("lists a chat-mode model whose cost map entry serves a decision endpoint", async () => { + mockFetch.mockResolvedValueOnce( + modelInfo([ + deployment("gpt-6-luna", "chat", ["/v1/chat/completions", "/v1/responses", "/v1/decisions"]), + deployment("clef", "chat", ["/v1/systemone"]), + deployment("gpt-5.5", "chat", ["/v1/chat/completions"]), + ]), + ); + + await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["clef", "gpt-6-luna"]); + }); + + it("lists a model group once when several deployments share its name", async () => { + mockFetch.mockResolvedValueOnce( + modelInfo([deployment("jev-latest", "evaluation"), deployment("jev-latest", "evaluation")]), + ); + + await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["jev-latest"]); + }); + + it("rejects a response that is not a model list", async () => { + mockFetch.mockResolvedValueOnce(modelInfo([{ model_info: { mode: "evaluation" } }])); await expect(fetchDecisionModels("virtual-key")).rejects.toThrow(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts index f49874b4735..1236ca5b4f7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking"; import { withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { createApiClient } from "@/lib/http/client"; -import { isDecisionMode } from "@/lib/decisionModels"; +import { isDecisionModel } from "@/lib/decisionModels"; import { systemOneResponseSchema, type PlaygroundRequest, @@ -25,8 +25,15 @@ export type PlaygroundDecisionRequest = | { endpoint: "/v1/systemone"; payload: PlaygroundRequest } | { endpoint: "/v1/decisions"; payload: OpenAIDecisionsRequest }; -const modelGroupInfoSchema = z.object({ - data: z.array(z.object({ model_group: z.string(), mode: z.string().nullish() })), +const modelInfoSchema = z.object({ + data: z.array( + z.object({ + model_name: z.string(), + model_info: z + .object({ mode: z.string().nullish(), supported_endpoints: z.array(z.string()).nullish() }) + .nullish(), + }), + ), }); function proxyClient(customBaseUrl?: string) { @@ -50,15 +57,15 @@ export async function fetchDecisionModels( customBaseUrl?: string, signal?: AbortSignal, ): Promise { - const body = await proxyClient(customBaseUrl).get("/model_group/info", { + const body = await proxyClient(customBaseUrl).get("/model/info", { headers: authHeaders(accessToken), signal, }); - return modelGroupInfoSchema + const decisionModelNames = modelInfoSchema .parse(body) - .data.filter((group) => isDecisionMode(group.mode)) - .map((group) => group.model_group) - .sort((a, b) => a.localeCompare(b)); + .data.filter((deployment) => isDecisionModel(deployment.model_info ?? {})) + .map((deployment) => deployment.model_name); + return [...new Set(decisionModelNames)].sort((a, b) => a.localeCompare(b)); } async function sendDecisionRequest( diff --git a/ui/litellm-dashboard/src/lib/decisionModels.ts b/ui/litellm-dashboard/src/lib/decisionModels.ts index a14ce1948a9..ae08e4a9c03 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.ts @@ -28,7 +28,12 @@ export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), prov export const isDecisionMode = (mode: string | null | undefined): boolean => mode === "decisions" || mode === "evaluation"; -const isDecisionEntry = (entry: CostMapEntry): boolean => +export interface DecisionModelFields { + readonly mode?: string | null; + readonly supported_endpoints?: readonly string[] | null; +} + +export const isDecisionModel = (entry: DecisionModelFields): boolean => isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint)); const parsedEntries = (costMap: unknown): ReadonlyArray => { @@ -44,12 +49,12 @@ const parsedEntries = (costMap: unknown): ReadonlyArray isDecisionEntry(entry)); + const decisionEntries = entries.filter(([, entry]) => isDecisionModel(entry)); const decisionProviders = new Set(decisionEntries.map(([, entry]) => entry.litellm_provider)); const providers = new Map( [...decisionProviders].map((provider) => { const providerEntries = entries.filter(([, entry]) => entry.litellm_provider === provider); - return [provider, { decisionOnly: providerEntries.every(([, entry]) => isDecisionEntry(entry)) }] as const; + return [provider, { decisionOnly: providerEntries.every(([, entry]) => isDecisionModel(entry)) }] as const; }), ); return { models: new Set(decisionEntries.map(([key]) => key)), providers };