mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
feat(ui): set deployment weights in key and team router settings (#50552)
* feat(ui): set deployment weights in key and team router settings The proxy already applies router_settings.weights per key and team, but no dashboard screen could set them, and saving a team in the UI dropped weights set through the API Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(ui): only offer deployments the key or team may weight Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(ui): show weight load errors, inherited strategy and team deployments on key create Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(ui): keep reliability fields on create forms and label weight inputs by deployment id Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * style(ui): name the accordion value identity object Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(ui): expect weights in the key create router settings payload Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(ui): keep weight drafts, loading state and team switches correct Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(ui): only accept weight keystrokes that save as a number Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
joshua-berri
GitHub
parent
093caa52fa
commit
152a5cac7b
14 changed files with 804 additions and 13 deletions
|
|
@ -0,0 +1,44 @@
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { act, render, screen } from "@testing-library/react";
|
||||
import { useState } from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion";
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
getRouterSettingsCall: vi.fn().mockResolvedValue({}),
|
||||
modelInfoCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/llm_calls/fetch_models", () => ({
|
||||
fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]),
|
||||
fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]),
|
||||
}));
|
||||
|
||||
const EchoingCreateForm = () => {
|
||||
const [value, setValue] = useState<RouterSettingsAccordionValue | undefined>(undefined);
|
||||
return <RouterSettingsAccordion accessToken="test-token" value={value} onChange={setValue} />;
|
||||
};
|
||||
|
||||
describe("RouterSettingsAccordion on a create form", () => {
|
||||
it("should render every reliability input after the parent echoes the first value back", async () => {
|
||||
render(
|
||||
<QueryClientProvider client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}>
|
||||
<EchoingCreateForm />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
await act(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
});
|
||||
|
||||
for (const field of [
|
||||
"allowed_fails",
|
||||
"cooldown_time",
|
||||
"num_retries",
|
||||
"timeout",
|
||||
"retry_after",
|
||||
"model_group_alias",
|
||||
]) {
|
||||
expect(screen.getByRole("textbox", { name: new RegExp(field) })).toBeEnabled();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -1,13 +1,18 @@
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import type { ReactElement, ReactNode } from "react";
|
||||
import { createRef, useState, type ReactElement, type ReactNode } from "react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm";
|
||||
import { fetchAvailableModels, fetchAvailableModelsForTeam } from "@/components/llm_calls/fetch_models";
|
||||
import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion";
|
||||
import { getRouterSettingsCall, modelInfoCall } from "../networking";
|
||||
import RouterSettingsAccordion, {
|
||||
RouterSettingsAccordionRef,
|
||||
RouterSettingsAccordionValue,
|
||||
} from "./RouterSettingsAccordion";
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
getRouterSettingsCall: vi.fn().mockResolvedValue({}),
|
||||
modelInfoCall: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/llm_calls/fetch_models", () => ({
|
||||
|
|
@ -130,6 +135,186 @@ describe("RouterSettingsAccordion", () => {
|
|||
expect(screen.getByTestId("available-models")).not.toHaveTextContent("slow-team-model");
|
||||
});
|
||||
|
||||
describe("deployment weights", () => {
|
||||
const gpt5Deployments = {
|
||||
data: [
|
||||
{ model_name: "gpt-5", litellm_params: { model: "openai/gpt-5" }, model_info: { id: "dep-openai" } },
|
||||
{ model_name: "gpt-5", litellm_params: { model: "azure/gpt-5" }, model_info: { id: "dep-azure" } },
|
||||
],
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(modelInfoCall).mockResolvedValue(gpt5Deployments);
|
||||
});
|
||||
|
||||
it("should save stored weights together with a newly entered one", async () => {
|
||||
const ref = createRef<RouterSettingsAccordionRef>();
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
ref={ref}
|
||||
accessToken="test-token"
|
||||
teamId="team-123"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
|
||||
fireEvent.change(await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"), { target: { value: "1" } });
|
||||
|
||||
expect(await screen.findByText("90% of traffic")).toBeInTheDocument();
|
||||
expect(screen.getByText("10% of traffic")).toBeInTheDocument();
|
||||
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({
|
||||
"gpt-5": { "dep-openai": 9, "dep-azure": 1 },
|
||||
});
|
||||
expect(vi.mocked(modelInfoCall).mock.calls[0][7]).toBe("team-123");
|
||||
expect(vi.mocked(modelInfoCall).mock.calls[0][11]).toBe("gpt-5");
|
||||
});
|
||||
|
||||
it("should save null once every weight is cleared, so the stored weights are removed", async () => {
|
||||
const ref = createRef<RouterSettingsAccordionRef>();
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
ref={ref}
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
|
||||
fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } });
|
||||
|
||||
await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull());
|
||||
});
|
||||
|
||||
it("should keep a model group on screen when the parent echoes back the value it was sent", async () => {
|
||||
const EchoingParent = () => {
|
||||
const [value, setValue] = useState<RouterSettingsAccordionValue>({
|
||||
router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } },
|
||||
});
|
||||
return <RouterSettingsAccordion accessToken="test-token" value={value} onChange={setValue} />;
|
||||
};
|
||||
renderWithQueryClient(<EchoingParent />);
|
||||
|
||||
fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } });
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(200);
|
||||
});
|
||||
|
||||
expect(screen.getByLabelText("Weight for openai/gpt-5 (dep-openai)")).toHaveValue("");
|
||||
expect(screen.getByRole("button", { name: "Remove weights for gpt-5" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should say the deployments failed to load instead of calling saved weights stale", async () => {
|
||||
vi.mocked(modelInfoCall).mockRejectedValue(new Error("403"));
|
||||
const ref = createRef<RouterSettingsAccordionRef>();
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
ref={ref}
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(
|
||||
await screen.findByText("Could not load this model group's deployments. Saved weights are kept."),
|
||||
).toBeInTheDocument();
|
||||
expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument();
|
||||
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } });
|
||||
});
|
||||
|
||||
it("should warn that weights are ignored when the inherited global strategy is not simple-shuffle", async () => {
|
||||
vi.mocked(getRouterSettingsCall).mockResolvedValueOnce({
|
||||
fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"], field_value: "latency-based-routing" }],
|
||||
});
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(
|
||||
await screen.findByText("Weights are ignored with latency-based-routing. Select simple-shuffle to use them."),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should accept a decimal weight typed one character at a time", async () => {
|
||||
const ref = createRef<RouterSettingsAccordionRef>();
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
ref={ref}
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
const input = await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)");
|
||||
|
||||
fireEvent.change(input, { target: { value: "1." } });
|
||||
expect(input).toHaveValue("1.");
|
||||
fireEvent.change(input, { target: { value: "1.5" } });
|
||||
|
||||
expect(input).toHaveValue("1.5");
|
||||
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({
|
||||
"gpt-5": { "dep-openai": 9, "dep-azure": 1.5 },
|
||||
});
|
||||
});
|
||||
|
||||
it("should ignore keystrokes that would not save as a weight, so the field always shows the saved value", async () => {
|
||||
const ref = createRef<RouterSettingsAccordionRef>();
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
ref={ref}
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
const input = await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)");
|
||||
|
||||
fireEvent.change(input, { target: { value: "9x" } });
|
||||
fireEvent.change(input, { target: { value: "-1" } });
|
||||
|
||||
expect(input).toHaveValue("9");
|
||||
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } });
|
||||
});
|
||||
|
||||
it("should not call saved weights stale while the deployments are still loading", async () => {
|
||||
vi.mocked(modelInfoCall).mockReturnValue(new Promise(() => {}));
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(await screen.findByText("dep-openai")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should drop weights picked for one team when the form switches to another team", async () => {
|
||||
const ref = createRef<RouterSettingsAccordionRef>();
|
||||
const value = { router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } };
|
||||
const { rerender } = renderWithQueryClient(
|
||||
<RouterSettingsAccordion ref={ref} accessToken="test-token" teamId="team-a" value={value} />,
|
||||
);
|
||||
await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)");
|
||||
|
||||
rerender(<RouterSettingsAccordion ref={ref} accessToken="test-token" teamId="team-b" value={value} />);
|
||||
|
||||
await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull());
|
||||
});
|
||||
|
||||
it("should flag a group whose weights are all zero", async () => {
|
||||
renderWithQueryClient(
|
||||
<RouterSettingsAccordion
|
||||
accessToken="test-token"
|
||||
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 0 } } } }}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(
|
||||
await screen.findByText("Set at least one weight above 0, or remove this model group."),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it("does not call onChange when unmounted mid-wait", async () => {
|
||||
const onChange = vi.fn<(value: RouterSettingsAccordionValue) => void>();
|
||||
const { unmount } = renderWithQueryClient(<RouterSettingsAccordion accessToken="test-token" onChange={onChange} />);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ import { Fallbacks } from "../Settings/RouterSettings/Fallbacks/AddFallbacks";
|
|||
import { FallbackSelectionForm } from "../Settings/RouterSettings/Fallbacks/FallbackSelectionForm";
|
||||
import { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig";
|
||||
import { fetchAvailableModels, fetchAvailableModelsForTeam, ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import RouterWeightsSection from "./RouterWeightsSection";
|
||||
import { RouterWeights, routerWeightsFrom, savedRouterWeights } from "./routerWeights";
|
||||
|
||||
export interface RouterSettingsAccordionValue {
|
||||
router_settings: {
|
||||
|
|
@ -24,6 +26,7 @@ export interface RouterSettingsAccordionValue {
|
|||
model_group_alias?: { [key: string]: any } | null;
|
||||
enable_tag_filtering?: boolean;
|
||||
routing_strategy_args?: { [key: string]: any } | null;
|
||||
weights?: RouterWeights | null;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -41,20 +44,38 @@ export interface RouterSettingsAccordionRef {
|
|||
|
||||
const PROPAGATE_WAIT_MS = 100;
|
||||
|
||||
const EMPTY_RELIABILITY_SETTINGS = {
|
||||
allowed_fails: null,
|
||||
cooldown_time: null,
|
||||
num_retries: null,
|
||||
timeout: null,
|
||||
retry_after: null,
|
||||
model_group_alias: null,
|
||||
};
|
||||
|
||||
const valueKeyOf = (settings: RouterSettingsAccordionValue["router_settings"]): string => {
|
||||
const { routing_strategy, fallbacks, enable_tag_filtering, weights } = settings;
|
||||
const identity = { routing_strategy, fallbacks, enable_tag_filtering, weights };
|
||||
return JSON.stringify(identity);
|
||||
};
|
||||
|
||||
const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSettingsAccordionProps>(
|
||||
({ accessToken, value, onChange, modelData, teamId }, ref) => {
|
||||
const { t } = useTranslation();
|
||||
const [formValue, setFormValue] = useState<RouterSettingsFormValue>({
|
||||
routerSettings: {},
|
||||
routerSettings: EMPTY_RELIABILITY_SETTINGS,
|
||||
selectedStrategy: null,
|
||||
enableTagFiltering: false,
|
||||
});
|
||||
const [fallbacks, setFallbacks] = useState<Fallbacks>([]);
|
||||
const [fallbackGroups, setFallbackGroups] = useState<FallbackGroup[]>([]);
|
||||
const [weights, setWeights] = useState<RouterWeights>({});
|
||||
const [availableRoutingStrategies, setAvailableRoutingStrategies] = useState<string[]>([]);
|
||||
const [globalRoutingStrategy, setGlobalRoutingStrategy] = useState<string | null>(null);
|
||||
const [routerFieldsMetadata, setRouterFieldsMetadata] = useState<{ [key: string]: any }>({});
|
||||
const [routingStrategyDescriptions, setRoutingStrategyDescriptions] = useState<{ [key: string]: string }>({});
|
||||
const isInternalUpdateRef = useRef(false);
|
||||
const weightsTeamIdRef = useRef(teamId);
|
||||
const lastInitializedValueRef = useRef<string | null>(null);
|
||||
|
||||
// Convert fallbacks format to groups format
|
||||
|
|
@ -90,13 +111,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
// Initialize from value prop if provided (only when value actually changes externally)
|
||||
useEffect(() => {
|
||||
// Create a stable key from the value to detect actual external changes
|
||||
const valueKey = value?.router_settings
|
||||
? JSON.stringify({
|
||||
routing_strategy: value.router_settings.routing_strategy,
|
||||
fallbacks: value.router_settings.fallbacks,
|
||||
enable_tag_filtering: value.router_settings.enable_tag_filtering,
|
||||
})
|
||||
: null;
|
||||
const valueKey = value?.router_settings ? valueKeyOf(value.router_settings) : null;
|
||||
|
||||
// Skip if this is an internal update (from our own onChange) and the value hasn't actually changed
|
||||
if (isInternalUpdateRef.current && valueKey === lastInitializedValueRef.current) {
|
||||
|
|
@ -118,7 +133,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
|
||||
if (value?.router_settings) {
|
||||
const rs = value.router_settings;
|
||||
const { fallbacks: _, ...routerSettingsWithoutFallbacks } = rs;
|
||||
const { fallbacks: _, weights: storedWeights, ...routerSettingsWithoutFallbacks } = rs;
|
||||
setFormValue({
|
||||
routerSettings: routerSettingsWithoutFallbacks as { [key: string]: any },
|
||||
selectedStrategy: rs.routing_strategy || null,
|
||||
|
|
@ -127,10 +142,11 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
const initialFallbacks = rs.fallbacks || [];
|
||||
setFallbacks(initialFallbacks);
|
||||
setFallbackGroups(fallbacksToGroups(initialFallbacks));
|
||||
setWeights(routerWeightsFrom(storedWeights));
|
||||
} else {
|
||||
// Initialize with empty defaults if no value provided
|
||||
setFormValue({
|
||||
routerSettings: {},
|
||||
routerSettings: EMPTY_RELIABILITY_SETTINGS,
|
||||
selectedStrategy: null,
|
||||
enableTagFiltering: false,
|
||||
});
|
||||
|
|
@ -142,9 +158,16 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
fallbackModels: [],
|
||||
},
|
||||
]);
|
||||
setWeights({});
|
||||
}
|
||||
}, [value]);
|
||||
|
||||
useEffect(() => {
|
||||
if (weightsTeamIdRef.current === teamId) return;
|
||||
weightsTeamIdRef.current = teamId;
|
||||
setWeights({});
|
||||
}, [teamId]);
|
||||
|
||||
// Fetch router settings metadata
|
||||
useEffect(() => {
|
||||
if (!accessToken) {
|
||||
|
|
@ -169,6 +192,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
if (routingStrategyField?.options) {
|
||||
setAvailableRoutingStrategies(routingStrategyField.options);
|
||||
}
|
||||
setGlobalRoutingStrategy(routingStrategyField?.field_value ?? null);
|
||||
|
||||
// Store routing strategy descriptions
|
||||
if (data.routing_strategy_descriptions) {
|
||||
|
|
@ -297,6 +321,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
model_group_alias: normalizeValue(updatedVariables.model_group_alias),
|
||||
enable_tag_filtering: formValue.enableTagFiltering,
|
||||
routing_strategy_args: normalizeValue(updatedVariables.routing_strategy_args),
|
||||
weights: savedRouterWeights(weights),
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -307,6 +332,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
}
|
||||
isInternalUpdateRef.current = true;
|
||||
const finalRouterSettings = buildRouterSettings();
|
||||
lastInitializedValueRef.current = valueKeyOf(finalRouterSettings);
|
||||
onChange({
|
||||
router_settings: finalRouterSettings,
|
||||
});
|
||||
|
|
@ -321,7 +347,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
}
|
||||
debouncedPropagate();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [formValue, fallbacks]);
|
||||
}, [formValue, fallbacks, weights]);
|
||||
|
||||
const handleFallbackGroupsChange = (newGroups: FallbackGroup[]): void => {
|
||||
setFallbackGroups(newGroups);
|
||||
|
|
@ -360,6 +386,16 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
availableRoutingStrategies={availableRoutingStrategies}
|
||||
routingStrategyDescriptions={routingStrategyDescriptions}
|
||||
/>
|
||||
<div className="mt-8 border-t border-border pt-8">
|
||||
<RouterWeightsSection
|
||||
accessToken={accessToken}
|
||||
teamId={teamId}
|
||||
availableModels={availableModels}
|
||||
selectedStrategy={formValue.selectedStrategy ?? globalRoutingStrategy}
|
||||
weights={weights}
|
||||
onChange={setWeights}
|
||||
/>
|
||||
</div>
|
||||
</TabsContent>
|
||||
<TabsContent value="2" keepMounted>
|
||||
<FallbackSelectionForm
|
||||
|
|
|
|||
|
|
@ -20,6 +20,15 @@ describe("RouterSettingsSummary", () => {
|
|||
expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should list each weighted model group with its deployment weights", () => {
|
||||
render(<RouterSettingsSummary routerSettings={{ weights: { "gpt-5": { "dep-a": 9, "dep-b": 1 }, empty: {} } }} />);
|
||||
|
||||
expect(screen.getByText("Deployment Weights:")).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-5")).toBeInTheDocument();
|
||||
expect(screen.getByText("dep-a: 9, dep-b: 1")).toBeInTheDocument();
|
||||
expect(screen.queryByText("empty")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show the empty state when every setting is null", () => {
|
||||
render(<RouterSettingsSummary routerSettings={{ fallbacks: null, num_retries: null }} />);
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next";
|
|||
import { Badge } from "@/components/ui/badge";
|
||||
import i18n from "@/i18n/i18n";
|
||||
import { hasRouterSettings } from "./routerSettingsPayload";
|
||||
import { routerWeightsFrom, savedRouterWeights } from "./routerWeights";
|
||||
|
||||
interface RouterSettingsSummaryProps {
|
||||
routerSettings: Record<string, unknown> | null | undefined;
|
||||
|
|
@ -15,6 +16,28 @@ const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => {
|
|||
);
|
||||
};
|
||||
|
||||
export function RouterWeightsSummary({ weights }: { weights: unknown }) {
|
||||
const { t } = useTranslation();
|
||||
const saved = savedRouterWeights(routerWeightsFrom(weights));
|
||||
if (!saved) return null;
|
||||
return (
|
||||
<div>
|
||||
<div>{t("routerWeights.summary")}</div>
|
||||
<div className="mt-1 space-y-1">
|
||||
{Object.entries(saved).map(([group, groupWeights]) => (
|
||||
<div key={group} className="text-xs text-muted-foreground">
|
||||
<span className="font-medium">{group}</span>
|
||||
<span className="mx-1 text-muted-foreground">-></span>
|
||||
{Object.entries(groupWeights)
|
||||
.map(([deploymentId, weight]) => `${deploymentId}: ${weight}`)
|
||||
.join(", ")}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function RouterSettingsSummary({
|
||||
routerSettings,
|
||||
emptyText = i18n.t("routerSettingsSummary.empty"),
|
||||
|
|
@ -63,6 +86,7 @@ export default function RouterSettingsSummary({
|
|||
</div>
|
||||
</div>
|
||||
)}
|
||||
<RouterWeightsSummary weights={settings.weights} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,206 @@
|
|||
import { useQuery } from "@tanstack/react-query";
|
||||
import { AlertCircle, X } from "lucide-react";
|
||||
import { useId, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { modelInfoCall } from "../networking";
|
||||
import {
|
||||
GroupWeights,
|
||||
RouterWeights,
|
||||
WeightableDeployment,
|
||||
hasNoPositiveWeight,
|
||||
parseWeightInput,
|
||||
weightRows,
|
||||
withDeploymentWeight,
|
||||
withGroup,
|
||||
withoutGroup,
|
||||
} from "./routerWeights";
|
||||
|
||||
const WEIGHTED_STRATEGY = "simple-shuffle";
|
||||
const DEPLOYMENTS_PAGE_SIZE = 1000;
|
||||
|
||||
interface RouterWeightsSectionProps {
|
||||
accessToken: string;
|
||||
teamId?: string | null;
|
||||
availableModels: string[];
|
||||
selectedStrategy: string | null;
|
||||
weights: RouterWeights;
|
||||
onChange: (weights: RouterWeights) => void;
|
||||
}
|
||||
|
||||
interface WeightGroupProps {
|
||||
accessToken: string;
|
||||
teamId?: string | null;
|
||||
group: string;
|
||||
groupWeights: GroupWeights;
|
||||
onWeightChange: (deploymentId: string, weight: number | null) => void;
|
||||
onRemove: () => void;
|
||||
}
|
||||
|
||||
interface WeightInputProps {
|
||||
label: string;
|
||||
weight: number | null;
|
||||
onChange: (weight: number | null) => void;
|
||||
}
|
||||
|
||||
const formatWeight = (weight: number | null): string => (weight === null ? "" : String(weight));
|
||||
|
||||
function WeightInput({ label, weight, onChange }: WeightInputProps) {
|
||||
const [draft, setDraft] = useState(formatWeight(weight));
|
||||
const [shownWeight, setShownWeight] = useState(weight);
|
||||
if (weight !== shownWeight) {
|
||||
setShownWeight(weight);
|
||||
if (parseWeightInput(draft) !== weight) setDraft(formatWeight(weight));
|
||||
}
|
||||
|
||||
const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const parsed = parseWeightInput(event.target.value);
|
||||
if (parsed === undefined) return;
|
||||
setDraft(event.target.value);
|
||||
onChange(parsed);
|
||||
};
|
||||
|
||||
return (
|
||||
<Input type="text" inputMode="decimal" className="w-28" aria-label={label} value={draft} onChange={handleChange} />
|
||||
);
|
||||
}
|
||||
|
||||
function WeightGroup({ accessToken, teamId, group, groupWeights, onWeightChange, onRemove }: WeightGroupProps) {
|
||||
const { t } = useTranslation();
|
||||
const {
|
||||
data: deployments = [],
|
||||
isLoading,
|
||||
isError,
|
||||
} = useQuery<WeightableDeployment[]>({
|
||||
queryKey: ["routerWeightDeployments", accessToken, teamId ?? null, group],
|
||||
queryFn: async () =>
|
||||
(
|
||||
await modelInfoCall(
|
||||
accessToken,
|
||||
"",
|
||||
"",
|
||||
1,
|
||||
DEPLOYMENTS_PAGE_SIZE,
|
||||
undefined,
|
||||
undefined,
|
||||
teamId ?? undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
group,
|
||||
)
|
||||
).data ?? [],
|
||||
enabled: Boolean(accessToken),
|
||||
});
|
||||
const rows = weightRows(group, groupWeights, deployments, teamId ?? null);
|
||||
const deploymentsKnown = !isLoading && !isError;
|
||||
const noDeployments = deploymentsKnown && rows.length === 0;
|
||||
|
||||
return (
|
||||
<div className="rounded-md border border-border p-4 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm font-semibold text-foreground">{group}</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-xs"
|
||||
aria-label={t("routerWeights.removeGroup", { group })}
|
||||
onClick={onRemove}
|
||||
>
|
||||
<X />
|
||||
</Button>
|
||||
</div>
|
||||
{isError && (
|
||||
<p role="alert" className="text-xs text-destructive">
|
||||
{t("routerWeights.loadFailed")}
|
||||
</p>
|
||||
)}
|
||||
{noDeployments && <p className="text-xs text-muted-foreground">{t("routerWeights.noDeployments")}</p>}
|
||||
{rows.map((row) => (
|
||||
<div key={row.deploymentId} className="flex items-center gap-3">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate text-sm text-foreground">
|
||||
{row.label ?? (deploymentsKnown ? t("routerWeights.unknownDeployment") : "")}
|
||||
</div>
|
||||
<div className="truncate text-xs text-muted-foreground">{row.deploymentId}</div>
|
||||
</div>
|
||||
<WeightInput
|
||||
label={t("routerWeights.weightFor", {
|
||||
deployment: row.label ? `${row.label} (${row.deploymentId})` : row.deploymentId,
|
||||
})}
|
||||
weight={row.weight}
|
||||
onChange={(weight) => onWeightChange(row.deploymentId, weight)}
|
||||
/>
|
||||
<span className="w-24 text-right text-xs text-muted-foreground">
|
||||
{row.sharePercent === null ? "" : t("routerWeights.share", { percent: row.sharePercent })}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
{hasNoPositiveWeight(groupWeights) && (
|
||||
<p role="alert" className="flex items-center gap-2 text-xs text-destructive">
|
||||
<AlertCircle className="size-4" />
|
||||
{t("routerWeights.noPositiveWeight")}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function RouterWeightsSection({
|
||||
accessToken,
|
||||
teamId,
|
||||
availableModels,
|
||||
selectedStrategy,
|
||||
weights,
|
||||
onChange,
|
||||
}: RouterWeightsSectionProps) {
|
||||
const { t } = useTranslation();
|
||||
const addGroupInputId = useId();
|
||||
const unweightedModels = availableModels.filter((model) => !(model in weights));
|
||||
const ignoredByStrategy = Boolean(selectedStrategy) && selectedStrategy !== WEIGHTED_STRATEGY;
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="max-w-3xl">
|
||||
<h3 className="text-sm font-medium text-foreground">{t("routerWeights.title")}</h3>
|
||||
<p className="mt-1 text-xs text-muted-foreground">{t("routerWeights.description")}</p>
|
||||
</div>
|
||||
{ignoredByStrategy && Object.keys(weights).length > 0 && (
|
||||
<div className="flex items-center gap-2 rounded-sm bg-warning/10 p-2 text-xs text-warning">
|
||||
<AlertCircle className="size-4" />
|
||||
<span>{t("routerWeights.ignoredForStrategy", { strategy: selectedStrategy })}</span>
|
||||
</div>
|
||||
)}
|
||||
{Object.entries(weights).map(([group, groupWeights]) => (
|
||||
<WeightGroup
|
||||
key={group}
|
||||
accessToken={accessToken}
|
||||
teamId={teamId}
|
||||
group={group}
|
||||
groupWeights={groupWeights}
|
||||
onWeightChange={(deploymentId, weight) =>
|
||||
onChange(withDeploymentWeight(weights, group, deploymentId, weight))
|
||||
}
|
||||
onRemove={() => onChange(withoutGroup(weights, group))}
|
||||
/>
|
||||
))}
|
||||
<div className="max-w-md">
|
||||
<label htmlFor={addGroupInputId} className="mb-2 block text-sm font-medium text-foreground">
|
||||
{t("routerWeights.addGroup")}
|
||||
</label>
|
||||
<SearchSelect
|
||||
inputId={addGroupInputId}
|
||||
options={unweightedModels.map((model) => ({ label: model, value: model }))}
|
||||
value={null}
|
||||
onValueChange={(group) => {
|
||||
if (group) onChange(withGroup(weights, group));
|
||||
}}
|
||||
placeholder={t("routerWeights.addGroupPlaceholder")}
|
||||
emptyText={t("routerWeights.noModels")}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -101,9 +101,18 @@ describe("routerSettingsUpdate", () => {
|
|||
model_group_alias: null,
|
||||
enable_tag_filtering: null,
|
||||
routing_strategy_args: null,
|
||||
weights: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("should replace stored weights with the edited ones, including clearing them", () => {
|
||||
const stored = { weights: { "gpt-4": { "dep-a": 9, "dep-b": 1 } }, num_retries: 2 };
|
||||
expect(routerSettingsUpdate({ num_retries: 2, weights: { "gpt-4": { "dep-a": 1 } } }, stored)).toMatchObject({
|
||||
weights: { "gpt-4": { "dep-a": 1 } },
|
||||
});
|
||||
expect(routerSettingsUpdate({ num_retries: 2, weights: null }, stored)).toMatchObject({ weights: null });
|
||||
});
|
||||
|
||||
it("should send the edited settings when the user configured something", () => {
|
||||
expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ const EDITOR_OWNED_FIELDS: Record<keyof RouterSettings, true> = {
|
|||
model_group_alias: true,
|
||||
enable_tag_filtering: true,
|
||||
routing_strategy_args: true,
|
||||
weights: true,
|
||||
};
|
||||
|
||||
const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array<keyof RouterSettings>;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,152 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
hasNoPositiveWeight,
|
||||
parseWeightInput,
|
||||
routerWeightsFrom,
|
||||
savedRouterWeights,
|
||||
weightRows,
|
||||
withDeploymentWeight,
|
||||
withGroup,
|
||||
withoutGroup,
|
||||
} from "./routerWeights";
|
||||
|
||||
const deployment = (id: string, modelName: string, model: string, team?: { id: string; publicName: string }) => ({
|
||||
model_name: modelName,
|
||||
litellm_params: { model },
|
||||
model_info: { id, ...(team ? { team_id: team.id, team_public_model_name: team.publicName } : {}) },
|
||||
});
|
||||
|
||||
describe("routerWeightsFrom", () => {
|
||||
it("should keep well-formed groups and drop malformed weights from the stored blob", () => {
|
||||
expect(
|
||||
routerWeightsFrom({
|
||||
"gpt-5": { a: 9, b: 1, bad: "1", negative: -1, inf: Infinity },
|
||||
broken: [1, 2],
|
||||
}),
|
||||
).toStrictEqual({ "gpt-5": { a: 9, b: 1 } });
|
||||
});
|
||||
|
||||
it("should read missing or non-object weights as none", () => {
|
||||
expect(routerWeightsFrom(undefined)).toStrictEqual({});
|
||||
expect(routerWeightsFrom(null)).toStrictEqual({});
|
||||
expect(routerWeightsFrom("x")).toStrictEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("weight edits", () => {
|
||||
it("should set, replace and clear one deployment without touching others", () => {
|
||||
const start = { "gpt-5": { a: 9 }, other: { c: 2 } };
|
||||
const added = withDeploymentWeight(start, "gpt-5", "b", 1);
|
||||
expect(added).toStrictEqual({ "gpt-5": { a: 9, b: 1 }, other: { c: 2 } });
|
||||
expect(withDeploymentWeight(added, "gpt-5", "a", 3)).toStrictEqual({ "gpt-5": { a: 3, b: 1 }, other: { c: 2 } });
|
||||
expect(withDeploymentWeight(added, "gpt-5", "a", null)).toStrictEqual({ "gpt-5": { b: 1 }, other: { c: 2 } });
|
||||
expect(start).toStrictEqual({ "gpt-5": { a: 9 }, other: { c: 2 } });
|
||||
});
|
||||
|
||||
it("should add an empty group once and remove a group entirely", () => {
|
||||
const withEmpty = withGroup({ "gpt-5": { a: 9 } }, "claude");
|
||||
expect(withEmpty).toStrictEqual({ "gpt-5": { a: 9 }, claude: {} });
|
||||
expect(withGroup(withEmpty, "gpt-5")).toBe(withEmpty);
|
||||
expect(withoutGroup(withEmpty, "gpt-5")).toStrictEqual({ claude: {} });
|
||||
});
|
||||
});
|
||||
|
||||
describe("savedRouterWeights", () => {
|
||||
it("should drop groups with no weights so they are not saved", () => {
|
||||
expect(savedRouterWeights({ "gpt-5": { a: 0, b: 2 }, claude: {} })).toStrictEqual({ "gpt-5": { a: 0, b: 2 } });
|
||||
});
|
||||
|
||||
it("should save no weights as null so the field is cleared", () => {
|
||||
expect(savedRouterWeights({})).toBeNull();
|
||||
expect(savedRouterWeights({ claude: {} })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("hasNoPositiveWeight", () => {
|
||||
it("should flag a group whose weights are all zero, as the proxy rejects it", () => {
|
||||
expect(hasNoPositiveWeight({ a: 0, b: 0 })).toBe(true);
|
||||
expect(hasNoPositiveWeight({ a: 0, b: 0.5 })).toBe(false);
|
||||
expect(hasNoPositiveWeight({})).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("weightRows", () => {
|
||||
it("should list the group's deployments with each one's share of traffic", () => {
|
||||
const rows = weightRows(
|
||||
"gpt-5",
|
||||
{ a: 9, b: 1 },
|
||||
[
|
||||
deployment("a", "gpt-5", "openai/gpt-5"),
|
||||
deployment("b", "gpt-5", "azure/gpt-5"),
|
||||
deployment("c", "gpt-5", "bedrock/gpt-5"),
|
||||
],
|
||||
null,
|
||||
);
|
||||
expect(rows).toStrictEqual([
|
||||
{ deploymentId: "a", label: "openai/gpt-5", weight: 9, sharePercent: 90 },
|
||||
{ deploymentId: "b", label: "azure/gpt-5", weight: 1, sharePercent: 10 },
|
||||
{ deploymentId: "c", label: "bedrock/gpt-5", weight: null, sharePercent: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("should match team deployments by their public model name, not their internal routing name", () => {
|
||||
const rows = weightRows(
|
||||
"gpt-5",
|
||||
{},
|
||||
[
|
||||
deployment("team-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }),
|
||||
deployment("other-group", "gpt-5-mini", "openai/gpt-5-mini"),
|
||||
],
|
||||
"team-1",
|
||||
);
|
||||
expect(rows.map((row) => row.deploymentId)).toStrictEqual(["team-dep"]);
|
||||
});
|
||||
|
||||
it("should group a deployment with no team by its model name even if it carries a public name", () => {
|
||||
const orphanedPublicName = {
|
||||
model_name: "gpt-5-internal",
|
||||
litellm_params: { model: "openai/gpt-5" },
|
||||
model_info: { id: "x", team_public_model_name: "gpt-5" },
|
||||
};
|
||||
expect(weightRows("gpt-5", {}, [orphanedPublicName], null)).toStrictEqual([]);
|
||||
expect(weightRows("gpt-5-internal", {}, [orphanedPublicName], null)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("should leave out another team's deployments, which the proxy rejects", () => {
|
||||
const deployments = [
|
||||
deployment("global", "gpt-5", "openai/gpt-5"),
|
||||
deployment("team-1-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }),
|
||||
deployment("team-2-dep", "model_name_team-2_def", "openai/gpt-5", { id: "team-2", publicName: "gpt-5" }),
|
||||
];
|
||||
expect(weightRows("gpt-5", {}, deployments, "team-1").map((row) => row.deploymentId)).toStrictEqual([
|
||||
"global",
|
||||
"team-1-dep",
|
||||
]);
|
||||
expect(weightRows("gpt-5", {}, deployments, null).map((row) => row.deploymentId)).toStrictEqual(["global"]);
|
||||
});
|
||||
|
||||
it("should show no share until some weight is set", () => {
|
||||
const rows = weightRows("gpt-5", {}, [deployment("a", "gpt-5", "openai/gpt-5")], null);
|
||||
expect(rows[0].sharePercent).toBeNull();
|
||||
});
|
||||
|
||||
it("should keep a weighted id whose deployment is gone so it can still be cleared", () => {
|
||||
const rows = weightRows("gpt-5", { a: 1, deleted: 1 }, [deployment("a", "gpt-5", "openai/gpt-5")], null);
|
||||
const staleRow = { deploymentId: "deleted", label: null, weight: 1, sharePercent: 50 };
|
||||
expect(rows[1]).toStrictEqual(staleRow);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseWeightInput", () => {
|
||||
it("should accept non-negative numbers, clear on empty and reject the rest", () => {
|
||||
expect(parseWeightInput("2.5")).toBe(2.5);
|
||||
expect(parseWeightInput("0")).toBe(0);
|
||||
expect(parseWeightInput(" ")).toBeNull();
|
||||
expect(parseWeightInput("-1")).toBeUndefined();
|
||||
expect(parseWeightInput("abc")).toBeUndefined();
|
||||
expect(parseWeightInput("1.")).toBe(1);
|
||||
expect(parseWeightInput(".")).toBeUndefined();
|
||||
expect(parseWeightInput("1e3")).toBeUndefined();
|
||||
expect(parseWeightInput("0x10")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
export type GroupWeights = Readonly<Record<string, number>>;
|
||||
export type RouterWeights = Readonly<Record<string, GroupWeights>>;
|
||||
|
||||
export interface WeightableDeployment {
|
||||
model_name?: string | null;
|
||||
litellm_params?: { model?: string | null } | null;
|
||||
model_info?: { id?: string | null; team_id?: string | null; team_public_model_name?: string | null } | null;
|
||||
}
|
||||
|
||||
export interface WeightRow {
|
||||
deploymentId: string;
|
||||
label: string | null;
|
||||
weight: number | null;
|
||||
sharePercent: number | null;
|
||||
}
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
const isWeight = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0;
|
||||
|
||||
export const routerWeightsFrom = (stored: unknown): RouterWeights =>
|
||||
isRecord(stored)
|
||||
? Object.fromEntries(
|
||||
Object.entries(stored)
|
||||
.filter((entry): entry is [string, Record<string, unknown>] => isRecord(entry[1]))
|
||||
.map(([group, weights]) => [
|
||||
group,
|
||||
Object.fromEntries(
|
||||
Object.entries(weights).filter((entry): entry is [string, number] => isWeight(entry[1])),
|
||||
),
|
||||
]),
|
||||
)
|
||||
: {};
|
||||
|
||||
const deploymentGroup = (deployment: WeightableDeployment): string | null =>
|
||||
(deployment.model_info?.team_id ? deployment.model_info.team_public_model_name : null) ||
|
||||
deployment.model_name ||
|
||||
null;
|
||||
|
||||
const isWeightable = (deployment: WeightableDeployment, group: string, teamId: string | null): boolean => {
|
||||
const ownerTeamId = deployment.model_info?.team_id ?? null;
|
||||
const visibleToCaller = ownerTeamId === null || ownerTeamId === teamId;
|
||||
return deploymentGroup(deployment) === group && Boolean(deployment.model_info?.id) && visibleToCaller;
|
||||
};
|
||||
|
||||
export const withGroup = (weights: RouterWeights, group: string): RouterWeights =>
|
||||
group in weights ? weights : { ...weights, [group]: {} };
|
||||
|
||||
export const withoutGroup = (weights: RouterWeights, group: string): RouterWeights =>
|
||||
Object.fromEntries(Object.entries(weights).filter(([name]) => name !== group));
|
||||
|
||||
export const withDeploymentWeight = (
|
||||
weights: RouterWeights,
|
||||
group: string,
|
||||
deploymentId: string,
|
||||
weight: number | null,
|
||||
): RouterWeights => {
|
||||
const others = Object.fromEntries(Object.entries(weights[group] ?? {}).filter(([id]) => id !== deploymentId));
|
||||
return { ...weights, [group]: weight === null ? others : { ...others, [deploymentId]: weight } };
|
||||
};
|
||||
|
||||
export const savedRouterWeights = (weights: RouterWeights): RouterWeights | null => {
|
||||
const nonEmpty = Object.entries(weights).filter(([, group]) => Object.keys(group).length > 0);
|
||||
return nonEmpty.length > 0 ? Object.fromEntries(nonEmpty) : null;
|
||||
};
|
||||
|
||||
export const hasNoPositiveWeight = (group: GroupWeights): boolean =>
|
||||
Object.keys(group).length > 0 && !Object.values(group).some((weight) => weight > 0);
|
||||
|
||||
export const weightRows = (
|
||||
group: string,
|
||||
groupWeights: GroupWeights,
|
||||
deployments: readonly WeightableDeployment[],
|
||||
teamId: string | null,
|
||||
): WeightRow[] => {
|
||||
const total = Object.values(groupWeights).reduce((sum, weight) => sum + weight, 0);
|
||||
const row = (deploymentId: string, label: string | null): WeightRow => {
|
||||
const weight = groupWeights[deploymentId] ?? null;
|
||||
return {
|
||||
deploymentId,
|
||||
label,
|
||||
weight,
|
||||
sharePercent: total > 0 ? Math.round(((weight ?? 0) / total) * 100) : null,
|
||||
};
|
||||
};
|
||||
const current = deployments
|
||||
.filter((deployment) => isWeightable(deployment, group, teamId))
|
||||
.map((deployment) => row(deployment.model_info!.id!, deployment.litellm_params?.model ?? null));
|
||||
const currentIds = new Set(current.map(({ deploymentId }) => deploymentId));
|
||||
const stale = Object.keys(groupWeights)
|
||||
.filter((deploymentId) => !currentIds.has(deploymentId))
|
||||
.map((deploymentId) => row(deploymentId, null));
|
||||
return [...current, ...stale];
|
||||
};
|
||||
|
||||
const WEIGHT_DRAFT = /^(\d+\.?\d*|\.\d+)?$/;
|
||||
|
||||
export const parseWeightInput = (raw: string): number | null | undefined => {
|
||||
const trimmed = raw.trim();
|
||||
if (!WEIGHT_DRAFT.test(trimmed)) return undefined;
|
||||
if (trimmed === "") return null;
|
||||
const weight = Number(trimmed);
|
||||
return isWeight(weight) ? weight : undefined;
|
||||
};
|
||||
|
|
@ -173,6 +173,7 @@ const ROUTER_SETTINGS_DEFAULT = {
|
|||
model_group_alias: null,
|
||||
enable_tag_filtering: false,
|
||||
routing_strategy_args: null,
|
||||
weights: null,
|
||||
};
|
||||
|
||||
const SECTION_PAYLOAD_ADDITIONS: Record<keyof typeof SECTIONS, Record<string, unknown>> = {
|
||||
|
|
|
|||
|
|
@ -1784,6 +1784,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
|
|||
key={routerSettingsKey}
|
||||
ref={routerSettingsRef}
|
||||
accessToken={accessToken || ""}
|
||||
teamId={selectedCreateKeyTeam?.team_id}
|
||||
value={routerSettings || undefined}
|
||||
onChange={setRouterSettings}
|
||||
modelData={
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ import SearchToolSelector from "../search_tools/SearchToolSelector";
|
|||
import SkillSelector from "../skills/SkillSelector";
|
||||
import EditLoggingSettings from "./EditLoggingSettings";
|
||||
import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion";
|
||||
import { RouterWeightsSummary } from "../common_components/RouterSettingsSummary";
|
||||
import { StorePromptsToggle } from "../common_components/StorePromptsToggle";
|
||||
import MemberModal from "./EditMembership";
|
||||
import MemberPermissions from "./member_permissions";
|
||||
|
|
@ -2673,6 +2674,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
{info.router_settings.enable_tag_filtering && (
|
||||
<div>{t("teams.teamInfo.settingsView.router.tagFilteringEnabled")}</div>
|
||||
)}
|
||||
<RouterWeightsSummary weights={info.router_settings.weights} />
|
||||
</div>
|
||||
) : (
|
||||
<div className="text-muted-foreground">{t("teams.teamInfo.settingsView.router.none")}</div>
|
||||
|
|
|
|||
|
|
@ -20380,5 +20380,21 @@
|
|||
"query": "query",
|
||||
"request": "request"
|
||||
}
|
||||
},
|
||||
"routerWeights": {
|
||||
"title": "Deployment Weights",
|
||||
"description": "Split a model group's traffic across its deployments. A deployment with no weight gets no traffic once any weight in its group is set. Weights only apply with the simple-shuffle routing strategy.",
|
||||
"ignoredForStrategy": "Weights are ignored with {{strategy}}. Select simple-shuffle to use them.",
|
||||
"addGroup": "Add a model group",
|
||||
"addGroupPlaceholder": "Select a model group to weight",
|
||||
"noModels": "No model groups found",
|
||||
"noDeployments": "No deployments found for this model group",
|
||||
"unknownDeployment": "Deployment no longer exists",
|
||||
"weightFor": "Weight for {{deployment}}",
|
||||
"share": "{{percent}}% of traffic",
|
||||
"noPositiveWeight": "Set at least one weight above 0, or remove this model group.",
|
||||
"removeGroup": "Remove weights for {{group}}",
|
||||
"summary": "Deployment Weights:",
|
||||
"loadFailed": "Could not load this model group's deployments. Saved weights are kept."
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue