diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx new file mode 100644 index 00000000000..0f23e91207a --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx @@ -0,0 +1,44 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, render, screen } from "@testing-library/react"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; + +vi.mock("../networking", () => ({ + getRouterSettingsCall: vi.fn().mockResolvedValue({}), + modelInfoCall: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]), + fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]), +})); + +const EchoingCreateForm = () => { + const [value, setValue] = useState(undefined); + return ; +}; + +describe("RouterSettingsAccordion on a create form", () => { + it("should render every reliability input after the parent echoes the first value back", async () => { + render( + + + , + ); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 250)); + }); + + for (const field of [ + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "retry_after", + "model_group_alias", + ]) { + expect(screen.getByRole("textbox", { name: new RegExp(field) })).toBeEnabled(); + } + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx index 6f819607e3f..827c1b60636 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx @@ -1,13 +1,18 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; -import type { ReactElement, ReactNode } from "react"; +import { createRef, useState, type ReactElement, type ReactNode } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm"; import { fetchAvailableModels, fetchAvailableModelsForTeam } from "@/components/llm_calls/fetch_models"; -import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; +import { getRouterSettingsCall, modelInfoCall } from "../networking"; +import RouterSettingsAccordion, { + RouterSettingsAccordionRef, + RouterSettingsAccordionValue, +} from "./RouterSettingsAccordion"; vi.mock("../networking", () => ({ getRouterSettingsCall: vi.fn().mockResolvedValue({}), + modelInfoCall: vi.fn(), })); vi.mock("@/components/llm_calls/fetch_models", () => ({ @@ -130,6 +135,186 @@ describe("RouterSettingsAccordion", () => { expect(screen.getByTestId("available-models")).not.toHaveTextContent("slow-team-model"); }); + describe("deployment weights", () => { + const gpt5Deployments = { + data: [ + { model_name: "gpt-5", litellm_params: { model: "openai/gpt-5" }, model_info: { id: "dep-openai" } }, + { model_name: "gpt-5", litellm_params: { model: "azure/gpt-5" }, model_info: { id: "dep-azure" } }, + ], + }; + + beforeEach(() => { + vi.mocked(modelInfoCall).mockResolvedValue(gpt5Deployments); + }); + + it("should save stored weights together with a newly entered one", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + + fireEvent.change(await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"), { target: { value: "1" } }); + + expect(await screen.findByText("90% of traffic")).toBeInTheDocument(); + expect(screen.getByText("10% of traffic")).toBeInTheDocument(); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ + "gpt-5": { "dep-openai": 9, "dep-azure": 1 }, + }); + expect(vi.mocked(modelInfoCall).mock.calls[0][7]).toBe("team-123"); + expect(vi.mocked(modelInfoCall).mock.calls[0][11]).toBe("gpt-5"); + }); + + it("should save null once every weight is cleared, so the stored weights are removed", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + + fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } }); + + await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull()); + }); + + it("should keep a model group on screen when the parent echoes back the value it was sent", async () => { + const EchoingParent = () => { + const [value, setValue] = useState({ + router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } }, + }); + return ; + }; + renderWithQueryClient(); + + fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } }); + await act(async () => { + vi.advanceTimersByTime(200); + }); + + expect(screen.getByLabelText("Weight for openai/gpt-5 (dep-openai)")).toHaveValue(""); + expect(screen.getByRole("button", { name: "Remove weights for gpt-5" })).toBeInTheDocument(); + }); + + it("should say the deployments failed to load instead of calling saved weights stale", async () => { + vi.mocked(modelInfoCall).mockRejectedValue(new Error("403")); + const ref = createRef(); + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Could not load this model group's deployments. Saved weights are kept."), + ).toBeInTheDocument(); + expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument(); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } }); + }); + + it("should warn that weights are ignored when the inherited global strategy is not simple-shuffle", async () => { + vi.mocked(getRouterSettingsCall).mockResolvedValueOnce({ + fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"], field_value: "latency-based-routing" }], + }); + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Weights are ignored with latency-based-routing. Select simple-shuffle to use them."), + ).toBeInTheDocument(); + }); + + it("should accept a decimal weight typed one character at a time", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + const input = await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"); + + fireEvent.change(input, { target: { value: "1." } }); + expect(input).toHaveValue("1."); + fireEvent.change(input, { target: { value: "1.5" } }); + + expect(input).toHaveValue("1.5"); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ + "gpt-5": { "dep-openai": 9, "dep-azure": 1.5 }, + }); + }); + + it("should ignore keystrokes that would not save as a weight, so the field always shows the saved value", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + const input = await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"); + + fireEvent.change(input, { target: { value: "9x" } }); + fireEvent.change(input, { target: { value: "-1" } }); + + expect(input).toHaveValue("9"); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } }); + }); + + it("should not call saved weights stale while the deployments are still loading", async () => { + vi.mocked(modelInfoCall).mockReturnValue(new Promise(() => {})); + renderWithQueryClient( + , + ); + + expect(await screen.findByText("dep-openai")).toBeInTheDocument(); + expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument(); + }); + + it("should drop weights picked for one team when the form switches to another team", async () => { + const ref = createRef(); + const value = { router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }; + const { rerender } = renderWithQueryClient( + , + ); + await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"); + + rerender(); + + await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull()); + }); + + it("should flag a group whose weights are all zero", async () => { + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Set at least one weight above 0, or remove this model group."), + ).toBeInTheDocument(); + }); + }); + it("does not call onChange when unmounted mid-wait", async () => { const onChange = vi.fn<(value: RouterSettingsAccordionValue) => void>(); const { unmount } = renderWithQueryClient(); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx index 68fbd294b7c..a89c4138970 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx @@ -9,6 +9,8 @@ import { Fallbacks } from "../Settings/RouterSettings/Fallbacks/AddFallbacks"; import { FallbackSelectionForm } from "../Settings/RouterSettings/Fallbacks/FallbackSelectionForm"; import { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig"; import { fetchAvailableModels, fetchAvailableModelsForTeam, ModelGroup } from "@/components/llm_calls/fetch_models"; +import RouterWeightsSection from "./RouterWeightsSection"; +import { RouterWeights, routerWeightsFrom, savedRouterWeights } from "./routerWeights"; export interface RouterSettingsAccordionValue { router_settings: { @@ -24,6 +26,7 @@ export interface RouterSettingsAccordionValue { model_group_alias?: { [key: string]: any } | null; enable_tag_filtering?: boolean; routing_strategy_args?: { [key: string]: any } | null; + weights?: RouterWeights | null; }; } @@ -41,20 +44,38 @@ export interface RouterSettingsAccordionRef { const PROPAGATE_WAIT_MS = 100; +const EMPTY_RELIABILITY_SETTINGS = { + allowed_fails: null, + cooldown_time: null, + num_retries: null, + timeout: null, + retry_after: null, + model_group_alias: null, +}; + +const valueKeyOf = (settings: RouterSettingsAccordionValue["router_settings"]): string => { + const { routing_strategy, fallbacks, enable_tag_filtering, weights } = settings; + const identity = { routing_strategy, fallbacks, enable_tag_filtering, weights }; + return JSON.stringify(identity); +}; + const RouterSettingsAccordion = forwardRef( ({ accessToken, value, onChange, modelData, teamId }, ref) => { const { t } = useTranslation(); const [formValue, setFormValue] = useState({ - routerSettings: {}, + routerSettings: EMPTY_RELIABILITY_SETTINGS, selectedStrategy: null, enableTagFiltering: false, }); const [fallbacks, setFallbacks] = useState([]); const [fallbackGroups, setFallbackGroups] = useState([]); + const [weights, setWeights] = useState({}); const [availableRoutingStrategies, setAvailableRoutingStrategies] = useState([]); + const [globalRoutingStrategy, setGlobalRoutingStrategy] = useState(null); const [routerFieldsMetadata, setRouterFieldsMetadata] = useState<{ [key: string]: any }>({}); const [routingStrategyDescriptions, setRoutingStrategyDescriptions] = useState<{ [key: string]: string }>({}); const isInternalUpdateRef = useRef(false); + const weightsTeamIdRef = useRef(teamId); const lastInitializedValueRef = useRef(null); // Convert fallbacks format to groups format @@ -90,13 +111,7 @@ const RouterSettingsAccordion = forwardRef { // Create a stable key from the value to detect actual external changes - const valueKey = value?.router_settings - ? JSON.stringify({ - routing_strategy: value.router_settings.routing_strategy, - fallbacks: value.router_settings.fallbacks, - enable_tag_filtering: value.router_settings.enable_tag_filtering, - }) - : null; + const valueKey = value?.router_settings ? valueKeyOf(value.router_settings) : null; // Skip if this is an internal update (from our own onChange) and the value hasn't actually changed if (isInternalUpdateRef.current && valueKey === lastInitializedValueRef.current) { @@ -118,7 +133,7 @@ const RouterSettingsAccordion = forwardRef { + if (weightsTeamIdRef.current === teamId) return; + weightsTeamIdRef.current = teamId; + setWeights({}); + }, [teamId]); + // Fetch router settings metadata useEffect(() => { if (!accessToken) { @@ -169,6 +192,7 @@ const RouterSettingsAccordion = forwardRef { setFallbackGroups(newGroups); @@ -360,6 +386,16 @@ const RouterSettingsAccordion = forwardRef +
+ +
{ expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument(); }); + it("should list each weighted model group with its deployment weights", () => { + render(); + + expect(screen.getByText("Deployment Weights:")).toBeInTheDocument(); + expect(screen.getByText("gpt-5")).toBeInTheDocument(); + expect(screen.getByText("dep-a: 9, dep-b: 1")).toBeInTheDocument(); + expect(screen.queryByText("empty")).not.toBeInTheDocument(); + }); + it("should show the empty state when every setting is null", () => { render(); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx index f99a154fe3d..89ec0873920 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx @@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import i18n from "@/i18n/i18n"; import { hasRouterSettings } from "./routerSettingsPayload"; +import { routerWeightsFrom, savedRouterWeights } from "./routerWeights"; interface RouterSettingsSummaryProps { routerSettings: Record | null | undefined; @@ -15,6 +16,28 @@ const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => { ); }; +export function RouterWeightsSummary({ weights }: { weights: unknown }) { + const { t } = useTranslation(); + const saved = savedRouterWeights(routerWeightsFrom(weights)); + if (!saved) return null; + return ( +
+
{t("routerWeights.summary")}
+
+ {Object.entries(saved).map(([group, groupWeights]) => ( +
+ {group} + -> + {Object.entries(groupWeights) + .map(([deploymentId, weight]) => `${deploymentId}: ${weight}`) + .join(", ")} +
+ ))} +
+
+ ); +} + export default function RouterSettingsSummary({ routerSettings, emptyText = i18n.t("routerSettingsSummary.empty"), @@ -63,6 +86,7 @@ export default function RouterSettingsSummary({ )} + ); } diff --git a/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx b/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx new file mode 100644 index 00000000000..32c5e70a71f --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx @@ -0,0 +1,206 @@ +import { useQuery } from "@tanstack/react-query"; +import { AlertCircle, X } from "lucide-react"; +import { useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { modelInfoCall } from "../networking"; +import { + GroupWeights, + RouterWeights, + WeightableDeployment, + hasNoPositiveWeight, + parseWeightInput, + weightRows, + withDeploymentWeight, + withGroup, + withoutGroup, +} from "./routerWeights"; + +const WEIGHTED_STRATEGY = "simple-shuffle"; +const DEPLOYMENTS_PAGE_SIZE = 1000; + +interface RouterWeightsSectionProps { + accessToken: string; + teamId?: string | null; + availableModels: string[]; + selectedStrategy: string | null; + weights: RouterWeights; + onChange: (weights: RouterWeights) => void; +} + +interface WeightGroupProps { + accessToken: string; + teamId?: string | null; + group: string; + groupWeights: GroupWeights; + onWeightChange: (deploymentId: string, weight: number | null) => void; + onRemove: () => void; +} + +interface WeightInputProps { + label: string; + weight: number | null; + onChange: (weight: number | null) => void; +} + +const formatWeight = (weight: number | null): string => (weight === null ? "" : String(weight)); + +function WeightInput({ label, weight, onChange }: WeightInputProps) { + const [draft, setDraft] = useState(formatWeight(weight)); + const [shownWeight, setShownWeight] = useState(weight); + if (weight !== shownWeight) { + setShownWeight(weight); + if (parseWeightInput(draft) !== weight) setDraft(formatWeight(weight)); + } + + const handleChange = (event: React.ChangeEvent) => { + const parsed = parseWeightInput(event.target.value); + if (parsed === undefined) return; + setDraft(event.target.value); + onChange(parsed); + }; + + return ( + + ); +} + +function WeightGroup({ accessToken, teamId, group, groupWeights, onWeightChange, onRemove }: WeightGroupProps) { + const { t } = useTranslation(); + const { + data: deployments = [], + isLoading, + isError, + } = useQuery({ + queryKey: ["routerWeightDeployments", accessToken, teamId ?? null, group], + queryFn: async () => + ( + await modelInfoCall( + accessToken, + "", + "", + 1, + DEPLOYMENTS_PAGE_SIZE, + undefined, + undefined, + teamId ?? undefined, + undefined, + undefined, + undefined, + group, + ) + ).data ?? [], + enabled: Boolean(accessToken), + }); + const rows = weightRows(group, groupWeights, deployments, teamId ?? null); + const deploymentsKnown = !isLoading && !isError; + const noDeployments = deploymentsKnown && rows.length === 0; + + return ( +
+
+ {group} + +
+ {isError && ( +

+ {t("routerWeights.loadFailed")} +

+ )} + {noDeployments &&

{t("routerWeights.noDeployments")}

} + {rows.map((row) => ( +
+
+
+ {row.label ?? (deploymentsKnown ? t("routerWeights.unknownDeployment") : "")} +
+
{row.deploymentId}
+
+ onWeightChange(row.deploymentId, weight)} + /> + + {row.sharePercent === null ? "" : t("routerWeights.share", { percent: row.sharePercent })} + +
+ ))} + {hasNoPositiveWeight(groupWeights) && ( +

+ + {t("routerWeights.noPositiveWeight")} +

+ )} +
+ ); +} + +export default function RouterWeightsSection({ + accessToken, + teamId, + availableModels, + selectedStrategy, + weights, + onChange, +}: RouterWeightsSectionProps) { + const { t } = useTranslation(); + const addGroupInputId = useId(); + const unweightedModels = availableModels.filter((model) => !(model in weights)); + const ignoredByStrategy = Boolean(selectedStrategy) && selectedStrategy !== WEIGHTED_STRATEGY; + + return ( +
+
+

{t("routerWeights.title")}

+

{t("routerWeights.description")}

+
+ {ignoredByStrategy && Object.keys(weights).length > 0 && ( +
+ + {t("routerWeights.ignoredForStrategy", { strategy: selectedStrategy })} +
+ )} + {Object.entries(weights).map(([group, groupWeights]) => ( + + onChange(withDeploymentWeight(weights, group, deploymentId, weight)) + } + onRemove={() => onChange(withoutGroup(weights, group))} + /> + ))} +
+ + ({ label: model, value: model }))} + value={null} + onValueChange={(group) => { + if (group) onChange(withGroup(weights, group)); + }} + placeholder={t("routerWeights.addGroupPlaceholder")} + emptyText={t("routerWeights.noModels")} + /> +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts index 542ed17d7a2..05645a50b41 100644 --- a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts @@ -101,9 +101,18 @@ describe("routerSettingsUpdate", () => { model_group_alias: null, enable_tag_filtering: null, routing_strategy_args: null, + weights: null, }); }); + it("should replace stored weights with the edited ones, including clearing them", () => { + const stored = { weights: { "gpt-4": { "dep-a": 9, "dep-b": 1 } }, num_retries: 2 }; + expect(routerSettingsUpdate({ num_retries: 2, weights: { "gpt-4": { "dep-a": 1 } } }, stored)).toMatchObject({ + weights: { "gpt-4": { "dep-a": 1 } }, + }); + expect(routerSettingsUpdate({ num_retries: 2, weights: null }, stored)).toMatchObject({ weights: null }); + }); + it("should send the edited settings when the user configured something", () => { expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts index 1d048425a6d..1bf6d0ef252 100644 --- a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts @@ -15,6 +15,7 @@ const EDITOR_OWNED_FIELDS: Record = { model_group_alias: true, enable_tag_filtering: true, routing_strategy_args: true, + weights: true, }; const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array; diff --git a/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts b/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts new file mode 100644 index 00000000000..3df42281bf5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { + hasNoPositiveWeight, + parseWeightInput, + routerWeightsFrom, + savedRouterWeights, + weightRows, + withDeploymentWeight, + withGroup, + withoutGroup, +} from "./routerWeights"; + +const deployment = (id: string, modelName: string, model: string, team?: { id: string; publicName: string }) => ({ + model_name: modelName, + litellm_params: { model }, + model_info: { id, ...(team ? { team_id: team.id, team_public_model_name: team.publicName } : {}) }, +}); + +describe("routerWeightsFrom", () => { + it("should keep well-formed groups and drop malformed weights from the stored blob", () => { + expect( + routerWeightsFrom({ + "gpt-5": { a: 9, b: 1, bad: "1", negative: -1, inf: Infinity }, + broken: [1, 2], + }), + ).toStrictEqual({ "gpt-5": { a: 9, b: 1 } }); + }); + + it("should read missing or non-object weights as none", () => { + expect(routerWeightsFrom(undefined)).toStrictEqual({}); + expect(routerWeightsFrom(null)).toStrictEqual({}); + expect(routerWeightsFrom("x")).toStrictEqual({}); + }); +}); + +describe("weight edits", () => { + it("should set, replace and clear one deployment without touching others", () => { + const start = { "gpt-5": { a: 9 }, other: { c: 2 } }; + const added = withDeploymentWeight(start, "gpt-5", "b", 1); + expect(added).toStrictEqual({ "gpt-5": { a: 9, b: 1 }, other: { c: 2 } }); + expect(withDeploymentWeight(added, "gpt-5", "a", 3)).toStrictEqual({ "gpt-5": { a: 3, b: 1 }, other: { c: 2 } }); + expect(withDeploymentWeight(added, "gpt-5", "a", null)).toStrictEqual({ "gpt-5": { b: 1 }, other: { c: 2 } }); + expect(start).toStrictEqual({ "gpt-5": { a: 9 }, other: { c: 2 } }); + }); + + it("should add an empty group once and remove a group entirely", () => { + const withEmpty = withGroup({ "gpt-5": { a: 9 } }, "claude"); + expect(withEmpty).toStrictEqual({ "gpt-5": { a: 9 }, claude: {} }); + expect(withGroup(withEmpty, "gpt-5")).toBe(withEmpty); + expect(withoutGroup(withEmpty, "gpt-5")).toStrictEqual({ claude: {} }); + }); +}); + +describe("savedRouterWeights", () => { + it("should drop groups with no weights so they are not saved", () => { + expect(savedRouterWeights({ "gpt-5": { a: 0, b: 2 }, claude: {} })).toStrictEqual({ "gpt-5": { a: 0, b: 2 } }); + }); + + it("should save no weights as null so the field is cleared", () => { + expect(savedRouterWeights({})).toBeNull(); + expect(savedRouterWeights({ claude: {} })).toBeNull(); + }); +}); + +describe("hasNoPositiveWeight", () => { + it("should flag a group whose weights are all zero, as the proxy rejects it", () => { + expect(hasNoPositiveWeight({ a: 0, b: 0 })).toBe(true); + expect(hasNoPositiveWeight({ a: 0, b: 0.5 })).toBe(false); + expect(hasNoPositiveWeight({})).toBe(false); + }); +}); + +describe("weightRows", () => { + it("should list the group's deployments with each one's share of traffic", () => { + const rows = weightRows( + "gpt-5", + { a: 9, b: 1 }, + [ + deployment("a", "gpt-5", "openai/gpt-5"), + deployment("b", "gpt-5", "azure/gpt-5"), + deployment("c", "gpt-5", "bedrock/gpt-5"), + ], + null, + ); + expect(rows).toStrictEqual([ + { deploymentId: "a", label: "openai/gpt-5", weight: 9, sharePercent: 90 }, + { deploymentId: "b", label: "azure/gpt-5", weight: 1, sharePercent: 10 }, + { deploymentId: "c", label: "bedrock/gpt-5", weight: null, sharePercent: 0 }, + ]); + }); + + it("should match team deployments by their public model name, not their internal routing name", () => { + const rows = weightRows( + "gpt-5", + {}, + [ + deployment("team-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }), + deployment("other-group", "gpt-5-mini", "openai/gpt-5-mini"), + ], + "team-1", + ); + expect(rows.map((row) => row.deploymentId)).toStrictEqual(["team-dep"]); + }); + + it("should group a deployment with no team by its model name even if it carries a public name", () => { + const orphanedPublicName = { + model_name: "gpt-5-internal", + litellm_params: { model: "openai/gpt-5" }, + model_info: { id: "x", team_public_model_name: "gpt-5" }, + }; + expect(weightRows("gpt-5", {}, [orphanedPublicName], null)).toStrictEqual([]); + expect(weightRows("gpt-5-internal", {}, [orphanedPublicName], null)).toHaveLength(1); + }); + + it("should leave out another team's deployments, which the proxy rejects", () => { + const deployments = [ + deployment("global", "gpt-5", "openai/gpt-5"), + deployment("team-1-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }), + deployment("team-2-dep", "model_name_team-2_def", "openai/gpt-5", { id: "team-2", publicName: "gpt-5" }), + ]; + expect(weightRows("gpt-5", {}, deployments, "team-1").map((row) => row.deploymentId)).toStrictEqual([ + "global", + "team-1-dep", + ]); + expect(weightRows("gpt-5", {}, deployments, null).map((row) => row.deploymentId)).toStrictEqual(["global"]); + }); + + it("should show no share until some weight is set", () => { + const rows = weightRows("gpt-5", {}, [deployment("a", "gpt-5", "openai/gpt-5")], null); + expect(rows[0].sharePercent).toBeNull(); + }); + + it("should keep a weighted id whose deployment is gone so it can still be cleared", () => { + const rows = weightRows("gpt-5", { a: 1, deleted: 1 }, [deployment("a", "gpt-5", "openai/gpt-5")], null); + const staleRow = { deploymentId: "deleted", label: null, weight: 1, sharePercent: 50 }; + expect(rows[1]).toStrictEqual(staleRow); + }); +}); + +describe("parseWeightInput", () => { + it("should accept non-negative numbers, clear on empty and reject the rest", () => { + expect(parseWeightInput("2.5")).toBe(2.5); + expect(parseWeightInput("0")).toBe(0); + expect(parseWeightInput(" ")).toBeNull(); + expect(parseWeightInput("-1")).toBeUndefined(); + expect(parseWeightInput("abc")).toBeUndefined(); + expect(parseWeightInput("1.")).toBe(1); + expect(parseWeightInput(".")).toBeUndefined(); + expect(parseWeightInput("1e3")).toBeUndefined(); + expect(parseWeightInput("0x10")).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/routerWeights.ts b/ui/litellm-dashboard/src/components/common_components/routerWeights.ts new file mode 100644 index 00000000000..16f428cade8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerWeights.ts @@ -0,0 +1,105 @@ +export type GroupWeights = Readonly>; +export type RouterWeights = Readonly>; + +export interface WeightableDeployment { + model_name?: string | null; + litellm_params?: { model?: string | null } | null; + model_info?: { id?: string | null; team_id?: string | null; team_public_model_name?: string | null } | null; +} + +export interface WeightRow { + deploymentId: string; + label: string | null; + weight: number | null; + sharePercent: number | null; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const isWeight = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0; + +export const routerWeightsFrom = (stored: unknown): RouterWeights => + isRecord(stored) + ? Object.fromEntries( + Object.entries(stored) + .filter((entry): entry is [string, Record] => isRecord(entry[1])) + .map(([group, weights]) => [ + group, + Object.fromEntries( + Object.entries(weights).filter((entry): entry is [string, number] => isWeight(entry[1])), + ), + ]), + ) + : {}; + +const deploymentGroup = (deployment: WeightableDeployment): string | null => + (deployment.model_info?.team_id ? deployment.model_info.team_public_model_name : null) || + deployment.model_name || + null; + +const isWeightable = (deployment: WeightableDeployment, group: string, teamId: string | null): boolean => { + const ownerTeamId = deployment.model_info?.team_id ?? null; + const visibleToCaller = ownerTeamId === null || ownerTeamId === teamId; + return deploymentGroup(deployment) === group && Boolean(deployment.model_info?.id) && visibleToCaller; +}; + +export const withGroup = (weights: RouterWeights, group: string): RouterWeights => + group in weights ? weights : { ...weights, [group]: {} }; + +export const withoutGroup = (weights: RouterWeights, group: string): RouterWeights => + Object.fromEntries(Object.entries(weights).filter(([name]) => name !== group)); + +export const withDeploymentWeight = ( + weights: RouterWeights, + group: string, + deploymentId: string, + weight: number | null, +): RouterWeights => { + const others = Object.fromEntries(Object.entries(weights[group] ?? {}).filter(([id]) => id !== deploymentId)); + return { ...weights, [group]: weight === null ? others : { ...others, [deploymentId]: weight } }; +}; + +export const savedRouterWeights = (weights: RouterWeights): RouterWeights | null => { + const nonEmpty = Object.entries(weights).filter(([, group]) => Object.keys(group).length > 0); + return nonEmpty.length > 0 ? Object.fromEntries(nonEmpty) : null; +}; + +export const hasNoPositiveWeight = (group: GroupWeights): boolean => + Object.keys(group).length > 0 && !Object.values(group).some((weight) => weight > 0); + +export const weightRows = ( + group: string, + groupWeights: GroupWeights, + deployments: readonly WeightableDeployment[], + teamId: string | null, +): WeightRow[] => { + const total = Object.values(groupWeights).reduce((sum, weight) => sum + weight, 0); + const row = (deploymentId: string, label: string | null): WeightRow => { + const weight = groupWeights[deploymentId] ?? null; + return { + deploymentId, + label, + weight, + sharePercent: total > 0 ? Math.round(((weight ?? 0) / total) * 100) : null, + }; + }; + const current = deployments + .filter((deployment) => isWeightable(deployment, group, teamId)) + .map((deployment) => row(deployment.model_info!.id!, deployment.litellm_params?.model ?? null)); + const currentIds = new Set(current.map(({ deploymentId }) => deploymentId)); + const stale = Object.keys(groupWeights) + .filter((deploymentId) => !currentIds.has(deploymentId)) + .map((deploymentId) => row(deploymentId, null)); + return [...current, ...stale]; +}; + +const WEIGHT_DRAFT = /^(\d+\.?\d*|\.\d+)?$/; + +export const parseWeightInput = (raw: string): number | null | undefined => { + const trimmed = raw.trim(); + if (!WEIGHT_DRAFT.test(trimmed)) return undefined; + if (trimmed === "") return null; + const weight = Number(trimmed); + return isWeight(weight) ? weight : undefined; +}; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 0fbe043c0d1..bbc849167cc 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -173,6 +173,7 @@ const ROUTER_SETTINGS_DEFAULT = { model_group_alias: null, enable_tag_filtering: false, routing_strategy_args: null, + weights: null, }; const SECTION_PAYLOAD_ADDITIONS: Record> = { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index ce780a221fd..dee936c4be0 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1784,6 +1784,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp key={routerSettingsKey} ref={routerSettingsRef} accessToken={accessToken || ""} + teamId={selectedCreateKeyTeam?.team_id} value={routerSettings || undefined} onChange={setRouterSettings} modelData={ diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index c6b485a57a1..3d67bed5d93 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -114,6 +114,7 @@ import SearchToolSelector from "../search_tools/SearchToolSelector"; import SkillSelector from "../skills/SkillSelector"; import EditLoggingSettings from "./EditLoggingSettings"; import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion"; +import { RouterWeightsSummary } from "../common_components/RouterSettingsSummary"; import { StorePromptsToggle } from "../common_components/StorePromptsToggle"; import MemberModal from "./EditMembership"; import MemberPermissions from "./member_permissions"; @@ -2673,6 +2674,7 @@ const TeamInfoView: React.FC = ({ {info.router_settings.enable_tag_filtering && (
{t("teams.teamInfo.settingsView.router.tagFilteringEnabled")}
)} + ) : (
{t("teams.teamInfo.settingsView.router.none")}
diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index d4ed30d51b2..06b3e4d6b19 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -20380,5 +20380,21 @@ "query": "query", "request": "request" } + }, + "routerWeights": { + "title": "Deployment Weights", + "description": "Split a model group's traffic across its deployments. A deployment with no weight gets no traffic once any weight in its group is set. Weights only apply with the simple-shuffle routing strategy.", + "ignoredForStrategy": "Weights are ignored with {{strategy}}. Select simple-shuffle to use them.", + "addGroup": "Add a model group", + "addGroupPlaceholder": "Select a model group to weight", + "noModels": "No model groups found", + "noDeployments": "No deployments found for this model group", + "unknownDeployment": "Deployment no longer exists", + "weightFor": "Weight for {{deployment}}", + "share": "{{percent}}% of traffic", + "noPositiveWeight": "Set at least one weight above 0, or remove this model group.", + "removeGroup": "Remove weights for {{group}}", + "summary": "Deployment Weights:", + "loadFailed": "Could not load this model group's deployments. Saved weights are kept." } }