Merge 7bdcf728da into 4b8a846788

litellm/proxy/proxy_server.py

@@ -16400,7 +16400,8 @@ async def model_info_v1(
     # when v1 resolved models only via public model_name strings.
     all_models: list[dict] = copy.deepcopy(llm_router.model_list)
     alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias())
-    all_models.extend(alias_models)
+    routing_group_models: Final = copy.deepcopy(list(llm_router.get_model_list_from_routing_groups()))
+    all_models.extend([*alias_models, *routing_group_models])
 
     all_models = expand_wildcard_deployments_for_model_info(all_models)
 

tests/unit/proxy/proxy_server/test_routes_model_info.py

@@ -596,6 +596,43 @@ def test_model_info_team_key_cannot_see_other_teams_byok_model(
     assert _BYOK_INTERNAL_NAME not in surfaced_names
 
 
+@pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"])
+def test_model_info_lists_routing_group_rows(client, auth_as, monkeypatch, path):
+    """A routing group is callable by its own name, so /model/info surfaces the
+    rows a configured router materializes for it next to the member deployment.
+    """
+    from litellm.proxy._types import LitellmUserRoles
+
+    member = {
+        "model_name": "gpt-6-luna",
+        "litellm_params": {"model": "openai/gpt-6-luna", "api_key": "sk-test"},
+        "model_info": {"id": "luna-deployment-id", "mode": "chat"},
+    }
+    router = litellm.Router(
+        model_list=[member],
+        routing_groups=[
+            {
+                "group_name": "decision-balanced",
+                "models": ["gpt-6-luna"],
+                "routing_strategy": "simple-shuffle",
+            }
+        ],
+    )
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", [member])
+    monkeypatch.setattr(proxy_server, "user_model", None)
+    monkeypatch.setattr(proxy_server, "prisma_client", None)
+
+    with auth_as(role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"):
+        response = client.get(path)
+
+    assert response.status_code == 200, response.text
+    surfaced = {m["model_name"]: m for m in response.json()["data"]}
+    assert set(surfaced) == {"gpt-6-luna", "decision-balanced"}
+    assert surfaced["decision-balanced"]["model_info"]["id"] == "luna-deployment-id"
+    assert surfaced["decision-balanced"]["model_info"]["mode"] == "chat"
+
+
 # ---------------------------------------------------------------------------
 # GET /model_group/info
 # ---------------------------------------------------------------------------

ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx

@@ -16,15 +16,15 @@ import type { SystemOneResponse } from "./lib/schemas";
 import type { OpenAIDecisionsResponse } from "./lib/openAIDecisions";
 import { curlPayload } from "./lib/curlCommand";
 
-interface ModelGroupInfo {
-  model_group: string;
+interface Deployment {
+  model_name: string;
   mode: string;
 }
 
-const DECISION_AND_CHAT_MODELS: ModelGroupInfo[] = [
-  { model_group: "gpt-5.5", mode: "chat" },
-  { model_group: "jev-latest", mode: "evaluation" },
-  { model_group: "pplx-decider", mode: "evaluation" },
+const DECISION_AND_CHAT_MODELS: Deployment[] = [
+  { model_name: "gpt-5.5", mode: "chat" },
+  { model_name: "jev-latest", mode: "evaluation" },
+  { model_name: "pplx-decider", mode: "evaluation" },
 ];
 
 const responseBody: SystemOneResponse = {
@@ -46,8 +46,19 @@ const render = (ui: ReactElement) =>
     </QueryClientProvider>,
   );
 
-const modelGroupInfoResponse = (groups: ModelGroupInfo[]) =>
-  ({ ok: true, status: 200, text: async () => JSON.stringify({ data: groups }) }) as Response;
+const modelInfoResponse = (deployments: Deployment[]) =>
+  ({
+    ok: true,
+    status: 200,
+    text: async () =>
+      JSON.stringify({
+        data: deployments.map(({ model_name, mode }) => ({
+          model_name,
+          litellm_params: { model: `openai/${model_name}` },
+          model_info: { id: model_name, mode },
+        })),
+      }),
+  }) as Response;
 
 const EXPIRED_KEY_RESPONSE = {
   ok: false,
@@ -56,7 +67,7 @@ const EXPIRED_KEY_RESPONSE = {
     JSON.stringify({ error: { message: "Authentication Error - Expired Key.", type: "expired_key", code: "401" } }),
 } as Response;
 
-const isModelLookup = (input: RequestInfo | URL) => String(input).endsWith("/model_group/info");
+const isModelLookup = (input: RequestInfo | URL) => String(input).endsWith("/model/info");
 
 const lookupAuthHeaders = (calls: Parameters<typeof fetch>[]) =>
   calls.map(([, init]) => Object.values((init?.headers ?? {}) as Record<string, string>));
@@ -99,7 +110,7 @@ describe("SystemOneUI integration", () => {
       isModelLookup(input) ? mockModelLookup(input, init) : mockFetch(input, init),
     );
     mockFetch.mockResolvedValue(createResponse(responseBody));
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse([]));
+    mockModelLookup.mockResolvedValue(modelInfoResponse([]));
   });
 
   afterEach(() => {
@@ -147,7 +158,7 @@ describe("SystemOneUI integration", () => {
 
   it("keeps the preset label after picking a model and keeps that model when clearing", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+    mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
     render(<SystemOneUI accessToken="session-key" />);
     const modelPicker = screen.getByRole("combobox", { name: "Decision model" });
     await waitFor(() => expect(modelPicker).toHaveValue("jev-latest"));
@@ -203,7 +214,7 @@ describe("SystemOneUI integration", () => {
 
   it("opens on /v1/systemone with the first decision model the key can call", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+    mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
     render(<SystemOneUI accessToken="session-key" />);
 
     await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
@@ -221,7 +232,7 @@ describe("SystemOneUI integration", () => {
 
   it("sends each endpoint's real shape and switches existing responses between cards and JSON without resending", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+    mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
     render(<SystemOneUI accessToken="session-key" />);
     await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
     fireEvent.change(screen.getByRole("textbox", { name: "Input" }), { target: { value: "An outage" } });
@@ -431,7 +442,7 @@ describe("SystemOneUI integration", () => {
 
   it("offers only decision models and writes the picked one into the request", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+    mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
     render(<SystemOneUI accessToken="session-key" />);
     const picker = await screen.findByRole("combobox", { name: "Decision model" });
 
@@ -451,11 +462,28 @@ describe("SystemOneUI integration", () => {
     expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("pplx-decider");
   });
 
+  it("looks the decision models up again when the playground is reopened within the cache window", async () => {
+    mockModelLookup.mockResolvedValueOnce(modelInfoResponse([{ model_name: "gpt-5.5", mode: "chat" }]));
+    const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } });
+    const page = (
+      <QueryClientProvider client={queryClient}>
+        <SystemOneUI accessToken="session-key" />
+      </QueryClientProvider>
+    );
+    const first = rtlRender(page);
+    expect(await screen.findByRole("link", { name: "Add a decision model" })).toBeInTheDocument();
+    first.unmount();
+
+    mockModelLookup.mockResolvedValueOnce(modelInfoResponse(DECISION_AND_CHAT_MODELS));
+    rtlRender(page);
+    const picker = await screen.findByRole("combobox", { name: "Decision model" });
+    await waitFor(() => expect(picker).toHaveValue("jev-latest"));
+    expect(mockModelLookup).toHaveBeenCalledTimes(2);
+  });
+
   it("routes a configured TypeSafe model through the same proxy endpoint", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(
-      modelGroupInfoResponse([{ model_group: "typesafe/jev-latest", mode: "evaluation" }]),
-    );
+    mockModelLookup.mockResolvedValue(modelInfoResponse([{ model_name: "typesafe/jev-latest", mode: "evaluation" }]));
     render(<SystemOneUI accessToken="session-key" />);
     await waitFor(() =>
       expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("typesafe/jev-latest"),
@@ -484,7 +512,7 @@ describe("SystemOneUI integration", () => {
 
   it("looks up a typed virtual key's decision models once, after typing settles", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+    mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
     render(<SystemOneUI accessToken={null} disabledPersonalKeyCreation />);
 
     await user.type(screen.getByLabelText("Virtual Key", { exact: true }), "sk-typed-key");
@@ -515,7 +543,7 @@ describe("SystemOneUI integration", () => {
     fireEvent.change(screen.getByRole("textbox", { name: "Input" }), { target: { value: "edited" } });
 
     await act(async () => {
-      models.resolve(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+      models.resolve(modelInfoResponse(DECISION_AND_CHAT_MODELS));
       await new Promise((resolve) => setTimeout(resolve, 0));
     });
 
@@ -544,7 +572,7 @@ describe("SystemOneUI integration", () => {
       expect(mockFetch.mock.calls[0]?.[0]).toMatch(new RegExp(`${endpoint}$`));
       expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).not.toHaveProperty("model");
       await act(async () => {
-        models.resolve(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+        models.resolve(modelInfoResponse(DECISION_AND_CHAT_MODELS));
         await models.promise;
       });
       expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("");
@@ -616,7 +644,7 @@ describe("SystemOneUI integration", () => {
 
   it("uses the requested model when the response omits it", async () => {
     const user = userEvent.setup();
-    mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+    mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
     mockFetch.mockResolvedValueOnce(
       createResponse({
         answers: responseBody.answers,
@@ -861,7 +889,7 @@ describe("SystemOneUI integration", () => {
 
     it("edits score levels and keeps instructions when a question changes type", async () => {
       const user = userEvent.setup();
-      mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
+      mockModelLookup.mockResolvedValue(modelInfoResponse(DECISION_AND_CHAT_MODELS));
       render(<SystemOneUI accessToken="session-key" />);
       await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest"));
 

ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/useDecisionModels.ts

@@ -26,6 +26,7 @@ export function useDecisionModels(
     enabled: lookupApiKey !== "",
     retry: false,
     staleTime: 60 * 1000,
+    refetchOnMount: "always" as const,
   };
   const query = useQuery(queryOptions);
   return { decisionModels: query.data ?? NO_DECISION_MODELS, isLoaded: query.isSuccess };

ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts

@@ -195,7 +195,12 @@ describe("/v1/decisions requests", () => {
 
 describe("fetchDecisionModels", () => {
   const mockFetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>();
-  const modelGroupInfo = (data: unknown) => ({ ok: true, text: async () => JSON.stringify({ data }) }) as Response;
+  const modelInfo = (data: unknown) => ({ ok: true, text: async () => JSON.stringify({ data }) }) as Response;
+  const deployment = (model_name: string, mode: string | null, supported_endpoints?: string[]) => ({
+    model_name,
+    litellm_params: { model: `openai/${model_name}` },
+    model_info: { id: `${model_name}-${mode}`, mode, supported_endpoints },
+  });
 
   beforeEach(() => {
     vi.stubGlobal("fetch", mockFetch);
@@ -208,11 +213,11 @@ describe("fetchDecisionModels", () => {
 
   it("lists the key's decision models from the proxy that answers the request, sorted by name", async () => {
     mockFetch.mockResolvedValueOnce(
-      modelGroupInfo([
-        { model_group: "pplx-decider", mode: "evaluation" },
-        { model_group: "gpt-5.5", mode: "chat" },
-        { model_group: "jev-latest", mode: "evaluation" },
-        { model_group: "unknown-mode", mode: null },
+      modelInfo([
+        deployment("pplx-decider", "evaluation"),
+        deployment("gpt-5.5", "chat", ["/v1/chat/completions", "/v1/responses"]),
+        deployment("jev-latest", "evaluation"),
+        deployment("unknown-mode", null),
       ]),
     );
 
@@ -221,7 +226,7 @@ describe("fetchDecisionModels", () => {
       "pplx-decider",
     ]);
     expect(mockFetch).toHaveBeenCalledWith(
-      "https://custom.example.com/model_group/info",
+      "https://custom.example.com/model/info",
       expect.objectContaining({
         method: "GET",
         headers: expect.objectContaining({ Authorization: "Bearer virtual-key" }),
@@ -229,8 +234,40 @@ describe("fetchDecisionModels", () => {
     );
   });
 
-  it("rejects a response that is not a model group list", async () => {
-    mockFetch.mockResolvedValueOnce(modelGroupInfo([{ mode: "evaluation" }]));
+  it("lists a routing group row the proxy materializes over a default-mode decision deployment", async () => {
+    mockFetch.mockResolvedValueOnce(
+      modelInfo([
+        deployment("gpt-6-luna", "chat", ["/v1/chat/completions", "/v1/decisions"]),
+        deployment("decision-balanced", "chat", ["/v1/chat/completions", "/v1/decisions"]),
+        deployment("chat-pool", "chat", ["/v1/chat/completions"]),
+      ]),
+    );
+
+    await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["decision-balanced", "gpt-6-luna"]);
+  });
+
+  it("lists a chat-mode model whose cost map entry serves a decision endpoint", async () => {
+    mockFetch.mockResolvedValueOnce(
+      modelInfo([
+        deployment("gpt-6-luna", "chat", ["/v1/chat/completions", "/v1/responses", "/v1/decisions"]),
+        deployment("clef", "chat", ["/v1/systemone"]),
+        deployment("gpt-5.5", "chat", ["/v1/chat/completions"]),
+      ]),
+    );
+
+    await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["clef", "gpt-6-luna"]);
+  });
+
+  it("lists a model group once when several deployments share its name", async () => {
+    mockFetch.mockResolvedValueOnce(
+      modelInfo([deployment("jev-latest", "evaluation"), deployment("jev-latest", "evaluation")]),
+    );
+
+    await expect(fetchDecisionModels("virtual-key")).resolves.toEqual(["jev-latest"]);
+  });
+
+  it("rejects a response that is not a model list", async () => {
+    mockFetch.mockResolvedValueOnce(modelInfo([{ model_info: { mode: "evaluation" } }]));
 
     await expect(fetchDecisionModels("virtual-key")).rejects.toThrow();
   });

ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts

@@ -2,7 +2,7 @@ import { z } from "zod";
 import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking";
 import { withRequiredHeaders } from "@/components/llm_calls/request_headers";
 import { createApiClient } from "@/lib/http/client";
-import { isDecisionMode } from "@/lib/decisionModels";
+import { isDecisionModel } from "@/lib/decisionModels";
 import {
   systemOneResponseSchema,
   type PlaygroundRequest,
@@ -25,8 +25,15 @@ export type PlaygroundDecisionRequest =
   | { endpoint: "/v1/systemone"; payload: PlaygroundRequest }
   | { endpoint: "/v1/decisions"; payload: OpenAIDecisionsRequest };
 
-const modelGroupInfoSchema = z.object({
-  data: z.array(z.object({ model_group: z.string(), mode: z.string().nullish() })),
+const modelInfoSchema = z.object({
+  data: z.array(
+    z.object({
+      model_name: z.string(),
+      model_info: z
+        .object({ mode: z.string().nullish(), supported_endpoints: z.array(z.string()).nullish() })
+        .nullish(),
+    }),
+  ),
 });
 
 function proxyClient(customBaseUrl?: string) {
@@ -50,15 +57,15 @@ export async function fetchDecisionModels(
   customBaseUrl?: string,
   signal?: AbortSignal,
 ): Promise<string[]> {
-  const body = await proxyClient(customBaseUrl).get<unknown>("/model_group/info", {
+  const body = await proxyClient(customBaseUrl).get<unknown>("/model/info", {
     headers: authHeaders(accessToken),
     signal,
   });
-  return modelGroupInfoSchema
+  const decisionModelNames = modelInfoSchema
     .parse(body)
-    .data.filter((group) => isDecisionMode(group.mode))
-    .map((group) => group.model_group)
-    .sort((a, b) => a.localeCompare(b));
+    .data.filter((deployment) => isDecisionModel(deployment.model_info ?? {}))
+    .map((deployment) => deployment.model_name);
+  return [...new Set(decisionModelNames)].sort((a, b) => a.localeCompare(b));
 }
 
 async function sendDecisionRequest<T>(

ui/litellm-dashboard/src/lib/decisionModels.ts

@@ -28,7 +28,12 @@ export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), prov
 export const isDecisionMode = (mode: string | null | undefined): boolean =>
   mode === "decisions" || mode === "evaluation";
 
-const isDecisionEntry = (entry: CostMapEntry): boolean =>
+export interface DecisionModelFields {
+  readonly mode?: string | null;
+  readonly supported_endpoints?: readonly string[] | null;
+}
+
+export const isDecisionModel = (entry: DecisionModelFields): boolean =>
   isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint));
 
 const parsedEntries = (costMap: unknown): ReadonlyArray<readonly [string, CostMapEntry]> => {
@@ -44,12 +49,12 @@ const parsedEntries = (costMap: unknown): ReadonlyArray<readonly [string, CostMa
 
 export function buildDecisionCatalog(costMap: unknown): DecisionCatalog {
   const entries = parsedEntries(costMap);
-  const decisionEntries = entries.filter(([, entry]) => isDecisionEntry(entry));
+  const decisionEntries = entries.filter(([, entry]) => isDecisionModel(entry));
   const decisionProviders = new Set(decisionEntries.map(([, entry]) => entry.litellm_provider));
   const providers = new Map(
     [...decisionProviders].map((provider) => {
       const providerEntries = entries.filter(([, entry]) => entry.litellm_provider === provider);
-      return [provider, { decisionOnly: providerEntries.every(([, entry]) => isDecisionEntry(entry)) }] as const;
+      return [provider, { decisionOnly: providerEntries.every(([, entry]) => isDecisionModel(entry)) }] as const;
     }),
   );
   return { models: new Set(decisionEntries.map(([key]) => key)), providers };