refine auto router setup flow

This commit is contained in:
moe-berri 2026-09-03 15:15:24 -07:00
parent 591f8aef1c
commit 9fdf34a3b3
3 changed files with 135 additions and 56 deletions

View file

@ -5,11 +5,24 @@ import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
import type { AutoSetupObjective, AutoSetupQualityLevel } from "./build_complexity_router_config";
const OBJECTIVE_DESCRIPTION: Record<AutoSetupObjective, string> = {
cost: "Uses expected cost per completed benchmark task, including failures",
task_completion_speed: "Uses live response speed for easy requests and benchmark completion time for hard ones",
balanced: "Balances cost per completed task with those same completion-time signals",
cost: "Spend less while staying within your selected quality level",
task_completion_speed: "Finish requests sooner, not just generate tokens faster",
balanced: "Balance expected cost with time to completion",
};
const QUALITY_OPTIONS = [
{ value: "economy", label: "Economy" },
{ value: "balanced", label: "Balanced" },
{ value: "high", label: "High" },
{ value: "max", label: "Max" },
];
const OBJECTIVE_OPTIONS = [
{ value: "cost", label: "Cost" },
{ value: "task_completion_speed", label: "Task completion speed" },
{ value: "balanced", label: "Balanced" },
];
interface AutoSetupControlsProps {
qualityLevel: AutoSetupQualityLevel;
optimizeFor: AutoSetupObjective;
@ -24,7 +37,7 @@ interface AutoSetupControlsProps {
hasAutoPolicy: boolean;
onRetry: () => void;
onRegenerate: () => void;
onUseManual: () => void;
onBack: () => void;
}
const recommendationErrorMessage = (error: unknown): string =>
@ -32,7 +45,18 @@ const recommendationErrorMessage = (error: unknown): string =>
const readyMessage = (recommendation: AutoRouterRecommendationResponse): string => {
const count = recommendation.matched_model_groups.length;
return `Ready with ${count} available model${count === 1 ? "" : "s"}. Review or edit below`;
return `Uses ${count} available model${count === 1 ? "" : "s"} across four complexity levels`;
};
const recommendationDescription = (optimizeFor: AutoSetupObjective): string => {
switch (optimizeFor) {
case "cost":
return "Favors the lowest expected cost while keeping each request within your selected quality level";
case "task_completion_speed":
return "Easy requests use live response speed; difficult requests use benchmark completion time";
case "balanced":
return "Balances expected cost with response speed and benchmark completion time";
}
};
export default function AutoSetupControls({
@ -49,20 +73,26 @@ export default function AutoSetupControls({
hasAutoPolicy,
onRetry,
onRegenerate,
onUseManual,
onBack,
}: AutoSetupControlsProps) {
return (
<div className="space-y-4 rounded-lg border border-border p-4" data-testid="auto-setup-controls">
<div>
<div className="font-medium text-foreground">Auto setup</div>
<div className="text-xs text-muted-foreground">
Builds an editable complexity router from the models available to you
<div className="space-y-4" data-testid="auto-setup-controls">
<div className="flex items-start justify-between gap-4">
<div>
<div className="font-medium text-foreground">Configure automatically</div>
<div className="text-xs text-muted-foreground">
Choose the outcome you want. LiteLLM will build the model tiers for you
</div>
</div>
<Button type="button" variant="ghost" className="h-auto shrink-0 px-0" onClick={onBack}>
Back
</Button>
</div>
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
<div>
<label className="mb-2 block text-sm font-medium text-foreground">Quality level</label>
<Select
items={QUALITY_OPTIONS}
value={qualityLevel}
onValueChange={(value: AutoSetupQualityLevel | null) => value && onQualityLevelChange(value)}
>
@ -83,6 +113,7 @@ export default function AutoSetupControls({
<div>
<label className="mb-2 block text-sm font-medium text-foreground">Optimize for</label>
<Select
items={OBJECTIVE_OPTIONS}
value={optimizeFor}
onValueChange={(value: AutoSetupObjective | null) => value && onOptimizeForChange(value)}
>
@ -115,20 +146,22 @@ export default function AutoSetupControls({
</div>
)}
{recommendation && (
<div className="space-y-1">
<div className={`text-xs ${hasAutoPolicy ? "text-success" : "text-warning"}`}>
{hasAutoPolicy ? readyMessage(recommendation) : "Customized model pools now use standard manual selection"}
<div className="space-y-2 rounded-lg border border-border bg-muted/40 p-3" data-testid="auto-setup-summary">
<div className={`text-sm font-medium ${hasAutoPolicy ? "text-foreground" : "text-warning"}`}>
{hasAutoPolicy ? "Recommended setup" : "Configuration edited"}
</div>
<div className="text-xs text-muted-foreground">
{hasAutoPolicy
? `${readyMessage(recommendation)}. ${recommendationDescription(optimizeFor)}`
: "Your model changes are now treated as a manual configuration"}
</div>
{!hasAutoPolicy && (
<button type="button" className="text-xs text-primary underline" onClick={onRegenerate}>
Regenerate Auto setup
Restore recommended setup
</button>
)}
</div>
)}
<Button type="button" variant="ghost" className="h-auto px-0" onClick={onUseManual}>
Choose a template or set up manually
</Button>
</div>
);
}

View file

@ -43,11 +43,17 @@ const ANTHROPIC_ONLY_MODEL = ANTHROPIC_TIERS.COMPLEX[0];
const openTemplateDropdown = (): void => {
if (!screen.queryByTestId("template-selector")) {
fireEvent.click(screen.getByRole("button", { name: /choose a template or set up manually/i }));
fireEvent.click(screen.getByRole("button", { name: "Back" }));
}
fireEvent.click(screen.getByTestId("template-selector"));
};
const openAutoSetup = (): void => {
if (!screen.queryByTestId("auto-quality-selector")) {
fireEvent.click(screen.getByTestId("configure-automatically-button"));
}
};
// Detailed Configuration is collapsed by default, so any test reaching into it (a tier select, an
// "Advanced: ..." sub-section) has to open it first.
const expandDetailedConfiguration = (): void => {
@ -152,9 +158,23 @@ describe("AddAutoRouterTab", () => {
vi.mocked(useAutoRouterRecommendation).mockReturnValue(EMPTY_RECOMMENDATION_QUERY);
});
it("starts with only quality and optimization choices in Auto setup", () => {
it("keeps the manual form and adds Configure automatically above it", () => {
renderWithProviders(<Harness />);
expect(screen.getByTestId("configure-automatically-button")).toBeInTheDocument();
expect(screen.getByTestId("template-selector")).toBeInTheDocument();
expect(screen.getByTestId("detailed-configuration-toggle")).toBeInTheDocument();
expect(screen.queryByTestId("auto-quality-selector")).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
expect(useAutoRouterRecommendation).toHaveBeenCalledWith(expect.objectContaining({ enabled: false }));
});
it("shows the two outcome choices after Configure automatically is selected", async () => {
const user = userEvent.setup();
renderWithProviders(<Harness />);
await user.click(screen.getByTestId("configure-automatically-button"));
expect(screen.getByTestId("auto-quality-selector")).toBeInTheDocument();
expect(screen.getByTestId("auto-objective-selector")).toBeInTheDocument();
expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument();
@ -163,6 +183,18 @@ describe("AddAutoRouterTab", () => {
);
});
it("returns to the original manual form from automatic setup", async () => {
const user = userEvent.setup();
renderWithProviders(<Harness />);
await user.click(screen.getByTestId("configure-automatically-button"));
await user.click(screen.getByRole("button", { name: "Back" }));
expect(screen.getByTestId("template-selector")).toBeInTheDocument();
expect(screen.getByTestId("configure-automatically-button")).toBeInTheDocument();
expect(screen.queryByTestId("auto-quality-selector")).not.toBeInTheDocument();
});
it("submits the editable policy generated for the caller's available model groups", async () => {
const user = userEvent.setup();
const autoSetup: AutoSetupConfigPayload = {
@ -221,7 +253,8 @@ describe("AddAutoRouterTab", () => {
});
renderWithProviders(<Harness />);
await screen.findByText(/Ready with 1 available model/);
await user.click(screen.getByTestId("configure-automatically-button"));
await screen.findByText(/Uses 1 available model/);
await user.type(screen.getByPlaceholderText(/smart_router/i), "max-speed-router");
await user.click(screen.getByRole("button", { name: /add auto router/i }));
@ -231,8 +264,8 @@ describe("AddAutoRouterTab", () => {
});
});
// Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller
// opts into the full tier/classifier form rather than always seeing it up front.
// Detailed Configuration starts collapsed after manual setup is selected; a caller opts into the
// full tier/classifier form rather than always seeing it up front.
it("keeps Detailed Configuration collapsed until a caller opens it", () => {
renderWithProviders(<Harness />);
@ -243,9 +276,7 @@ describe("AddAutoRouterTab", () => {
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
});
// Nothing is filled in, so there is nothing to submit. The button reports that itself instead of
// accepting a click and answering with a toast.
it("offers no submit at all until every tier has a model", async () => {
it("offers no submit until every tier has a model", async () => {
renderWithProviders(<Harness />);
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
@ -259,6 +290,7 @@ describe("AddAutoRouterTab", () => {
});
const { container } = renderWithProviders(<Harness />);
openAutoSetup();
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "early-router" } });
fireEvent.submit(container.querySelector("form")!);
@ -465,6 +497,7 @@ describe("AddAutoRouterTab", () => {
<AddAutoRouterTab handleOk={vi.fn()} accessToken="token" userRole="Internal User" createScope="team-required" />,
);
openAutoSetup();
expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument();
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
@ -480,6 +513,7 @@ describe("AddAutoRouterTab", () => {
<AddAutoRouterTab handleOk={vi.fn()} accessToken="token" userRole="Internal User" createScope="team-required" />,
);
openAutoSetup();
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
await user.click(screen.getByTestId("team-dropdown-clear"));

View file

@ -186,7 +186,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const watchedName = useWatch({ control: form.control, name: "auto_router_name" });
const watchedTeamId = useWatch({ control: form.control, name: "team_id" });
const [modelAccessGroups, setModelAccessGroups] = useState<string[]>([]);
const [setupMode, setSetupMode] = useState<SetupMode>("auto");
const [setupMode, setSetupMode] = useState<SetupMode>("manual");
const [qualityLevel, setQualityLevel] = useState<AutoSetupQualityLevel>("balanced");
const [optimizeFor, setOptimizeFor] = useState<AutoSetupObjective>("balanced");
const appliedRecommendationKey = useRef<string | null>(null);
@ -338,6 +338,20 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
setEscalationKeywords(prefill.escalationKeywords);
}, []);
const chooseAutoSetup = () => {
appliedRecommendationKey.current = null;
setDetailsExpanded(false);
setSetupMode("auto");
};
const returnToManualSetup = () => {
appliedRecommendationKey.current = null;
setSelectedPreset(undefined);
applyPrefill(buildEmptyPrefill());
setDetailsExpanded(false);
setSetupMode("manual");
};
useEffect(() => {
if (!recommendation || setupMode !== "auto") return;
const key = [
@ -596,22 +610,14 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
hasAutoPolicy={Boolean(complexityRouterConfig.auto_setup)}
onRetry={() => void refetchRecommendation()}
onRegenerate={regenerateAutoSetup}
onUseManual={() => setSetupMode("manual")}
onBack={returnToManualSetup}
/>
) : (
<div>
<div className="mb-2 flex items-center justify-between gap-3">
<label className="block text-sm font-medium text-foreground">Template</label>
<Button
type="button"
variant="ghost"
className="h-auto px-0"
onClick={() => {
appliedRecommendationKey.current = null;
setSetupMode("auto");
}}
>
Use Auto setup
<Button type="button" data-testid="configure-automatically-button" onClick={chooseAutoSetup}>
Configure automatically
</Button>
</div>
<Select
@ -691,11 +697,13 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
) : (
<ChevronRight className="size-3 text-muted-foreground" />
)}
Detailed Configuration
{setupMode === "auto" ? "Review and edit configuration" : "Detailed Configuration"}
</span>
{!detailsExpanded && (
<span className="text-xs text-muted-foreground line-clamp-2">
{tierConfigSummary(complexityRouterConfig)}
{setupMode === "auto"
? "View the generated model tiers and routing rules"
: tierConfigSummary(complexityRouterConfig)}
</span>
)}
</button>
@ -768,27 +776,31 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
<TooltipContent>Get help on our github</TooltipContent>
</Tooltip>
<div className="flex gap-2">
<BlockedReasonTooltip reason={submitBlockedReason}>
{setupMode === "manual" && (
<BlockedReasonTooltip reason={submitBlockedReason}>
<Button
type="button"
variant="outline"
data-testid="auto-router-test-routing-btn"
disabled={submitBlockedReason !== null || isSubmitting}
onClick={() => setIsRoutingTestVisible(true)}
>
Test Routing
</Button>
</BlockedReasonTooltip>
)}
{setupMode === "manual" && (
<Button
type="button"
variant="outline"
data-testid="auto-router-test-routing-btn"
disabled={submitBlockedReason !== null || isSubmitting}
onClick={() => setIsRoutingTestVisible(true)}
data-testid="auto-router-test-connect-btn"
onClick={handleTestConnection}
disabled={isTestingConnection}
>
Test Routing
{isTestingConnection && <UiLoadingSpinner className="size-4" />}
Test Connection
</Button>
</BlockedReasonTooltip>
<Button
type="button"
variant="outline"
data-testid="auto-router-test-connect-btn"
onClick={handleTestConnection}
disabled={isTestingConnection}
>
{isTestingConnection && <UiLoadingSpinner className="size-4" />}
Test Connection
</Button>
)}
<BlockedReasonTooltip reason={submitBlockedReason}>
<Button
type="button"