fix(router): preserve unavailable Fuse presets

This commit is contained in:
Tin Chi Lo 2026-09-19 11:48:40 -07:00
parent 2fa115db2b
commit 4d659135b6
5 changed files with 268 additions and 121 deletions

View file

@ -179,51 +179,6 @@ Configure capability forecasting through YAML or the model-management API.
The dashboard preserves its classifier and calibration on an untouched save;
it does not provide a capability-card editor
### Fuse v2 profile presets
Fuse v2 accepts maintained model and runtime descriptions instead of requiring
custom prose for both solvers and the harness. Select profiles explicitly for
all deployments behind your configured model groups and their actual settings.
Group names do not select profiles automatically
```yaml
complexity_router_config:
classifier_type: llm_v2
classifier_llm_config:
model: your-judge-group
tiers:
SIMPLE: your-efficient-group
REASONING: your-capable-group
llm_v2_config:
efficient_profile_preset: claude-sonnet-5-v1
capable_profile_preset: claude-fable-5-1-v1
harness_preset: claude-code-v1
max_quality_gap: 0.05
```
`GET /public/complexity_router/fuse_presets` returns the catalog version, model
profiles, and runtime descriptions, including source URLs. The bundled catalog
is loaded once per process without network requests. Sources are citations only
Each of `efficient_profile`, `capable_profile`, and `harness` requires either
nonblank custom text or its corresponding preset reference. Custom text wins
when both are supplied, but an unknown or wrong-kind preset is still rejected.
Explicit blank text is invalid even with a valid preset. Custom text remains
limited to 4000 characters
Saved configurations retain preset references and explicit text separately.
Preset text is resolved when building the classifier prompt, not copied into
stored custom fields. Existing all-custom configurations keep the same prompt.
Versioned preset IDs identify immutable content: revised wording receives a new
ID, and older referenced entries must remain available
The runtime presets do not imply a repository, runnable tests, network access,
additional tools, or a step, time, or spending budget. mini-SWE-agent describes
an agent interface, not a SWE-bench task. Model descriptions summarize provider
positioning without solve rates or guaranteed rankings. Wording is an evaluation
input, not a calibrated quality claim. Existing Fuse licensing, policy,
calibration, and prompt version are unchanged
### Heuristic v2
Set `classifier_type: heuristic_v2` to classify with the bundled calibrated

View file

@ -1,6 +1,7 @@
import json
from hashlib import sha256
from importlib.resources import files
from typing import Final
from typing import Final, Literal
import pytest
from pydantic import ValidationError
@ -19,11 +20,36 @@ def test_catalog_is_loaded_once_and_preserves_bundled_content() -> None:
assert first.model_dump(mode="json") == bundled
entries: Final = (*first.models, *first.harnesses)
assert len({entry.id for entry in entries}) == len(entries)
assert len(first.models) == 9
assert len(first.harnesses) == 5
assert all(entry.sources and all(source.startswith("https://") for source in entry.sources) for entry in entries)
@pytest.mark.parametrize(
("kind", "preset_id", "expected_digest"),
(
("model", "gpt-6-astra-v1", "a9403b0c00ea64081b7b08b5b968850670f3a047d219a7e0668f2169146ae96e"),
("model", "gpt-5.6-sol-v1", "2b91a6c43e0e93183aaaf9c355e1bbb8ed2e9817aab6b0c2f50148f53a23247b"),
("model", "gpt-5.6-luna-v1", "fff94a9e01bf4519798d5be4e76a3f9d57b75a2d9966a59dc92cbfeb5cd08d07"),
("model", "gpt-5.6-terra-v1", "75de040f3bea841fa4764885738303893ee7ac0804aed1e932cd3959185ff893"),
("model", "claude-haiku-4-5-v1", "91c1920953073462b6b70ef810596a5325f08286b5e62630cff47938fc4157db"),
("model", "claude-sonnet-5-v1", "133f4414c644a707cd8cf565a486153856f4836ca4e4f75ee0553f2b7a1e3663"),
("model", "claude-opus-5-v1", "9cbfcae45d2e3a2575e44ce5adf618f56614abff4b3221d35900c647200b99ef"),
("model", "claude-fable-5-v1", "25c275d7403f1572ffb4fe899d5feecd9a434ebdc37b4dd9ef601a8ecf4850fc"),
("model", "claude-fable-5-1-v1", "37693107c878ab6266530395bbdc2d2813d676d179bf281d05e5a5ec1b9d4c60"),
("harness", "unspecified-v1", "d9eb30b61509456f0c71ca805b33d821cab6605578d567a29ab421d8f602ce7b"),
("harness", "claude-code-v1", "7ee8e9d50f1cf44a8a58461efff66d6182f245d25499702c144d1c642c101ed9"),
("harness", "codex-cli-v1", "0678047e34562ef05b5e2fba099c1f9e5876304f7eaf3b0d8c3809e707eb3311"),
("harness", "opencode-v1", "8b6cc240d90091ac2ef9b374b535f981a55abb91e25d4c04fdb9fc206eeb907e"),
("harness", "mini-swe-agent-v1", "21e2dc4a8a2320a5a554a498b30516326dc3592ebf20b0f4db20ddb33e879a39"),
),
)
def test_existing_preset_text_is_unchanged(
kind: Literal["model", "harness"], preset_id: str, expected_digest: str
) -> None:
text: Final = resolve_fuse_profile(None, preset_id, kind)
assert text is not None
assert sha256(text.encode("utf-8")).hexdigest() == expected_digest
def test_every_catalog_entry_resolves_without_changing_custom_ownership() -> None:
catalog: Final = get_fuse_presets()
for entry in catalog.models:

View file

@ -159,7 +159,7 @@ describe("forecast classifier form", () => {
const effectiveText = override ?? catalog.models[0].text;
await waitFor(() => expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText));
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
await user.click(screen.getByRole("option", { name: "Custom", exact: true }));
await user.click(screen.getByRole("option", { name: "Custom" }));
expect(screen.getByLabelText("Efficient solver profile")).not.toHaveAttribute("readonly");
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText);
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Custom budget" } });
@ -219,6 +219,81 @@ describe("forecast classifier form", () => {
},
);
it.each([
["efficient_profile", "Efficient solver profile"],
["capable_profile", "Capable solver profile"],
["harness", "Harness and budget"],
] as const)(
"preserves the saved %s reference during a catalog outage until Custom text replaces it",
async (field, label) => {
const user = userEvent.setup();
vi.mocked(fetch).mockImplementation(async () => Response.json({ error: "unavailable" }, { status: 503 }));
renderWithProviders(<Form initialValue={presetInitial} />);
expect(await screen.findByText(/Profile presets could not be loaded/)).toBeInTheDocument();
const save = screen.getByRole("button", { name: "Save configuration" });
const output = screen.getByRole("status", { name: "Saved configuration" });
expect(save).toBeEnabled();
await user.click(save);
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig);
await user.click(screen.getByRole("combobox", { name: `${label} preset` }));
await user.click(screen.getByRole("option", { name: "Custom" }));
expect(screen.getByLabelText(label)).toHaveValue("");
expect(screen.getByLabelText(label)).not.toHaveAttribute("readonly");
expect(screen.getByRole("combobox", { name: `${label} preset` })).toHaveValue("Custom");
expect(save).toBeEnabled();
await user.click(save);
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig);
fireEvent.change(screen.getByLabelText(label), { target: { value: " " } });
expect(save).toBeDisabled();
await user.click(screen.getByRole("button", { name: `Keep saved ${label.toLowerCase()} preset` }));
expect(screen.getByLabelText(label)).toHaveAttribute("readonly");
expect(save).toBeEnabled();
await user.click(save);
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig);
await user.click(screen.getByRole("combobox", { name: `${label} preset` }));
await user.click(screen.getByRole("option", { name: "Custom" }));
const replacement = "Manually authored replacement";
fireEvent.change(screen.getByLabelText(label), { target: { value: replacement } });
expect(save).toBeEnabled();
await user.click(save);
const referenceKey = `${field}_preset` as const;
const { [referenceKey]: _reference, ...remaining } = presetConfig;
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual({ ...remaining, [field]: replacement });
},
);
it.each([true, false])(
"keeps a reference selected as Custom while the catalog settles, success=%s",
async (success) => {
const user = userEvent.setup();
const response = Promise.withResolvers<Response>();
vi.mocked(fetch).mockReturnValue(response.promise);
renderWithProviders(<Form initialValue={presetInitial} />);
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
await user.click(screen.getByRole("option", { name: "Custom" }));
await act(async () => response.resolve(success ? Response.json(catalog) : Response.json({}, { status: 503 })));
if (success) await screen.findAllByText(`Catalog version: ${catalog.version}`);
else await screen.findByText(/Profile presets could not be loaded/);
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(success ? catalog.models[0].text : "");
await user.click(screen.getByRole("button", { name: "Save configuration" }));
expect(
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
).toEqual(presetConfig);
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Replacement" } });
await user.click(screen.getByRole("button", { name: "Save configuration" }));
const { efficient_profile_preset: _reference, ...remaining } = presetConfig;
expect(
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
).toEqual({
...remaining,
efficient_profile: "Replacement",
});
},
);
it("keeps unknown saved IDs visible with unavailable previews rather than replacing them", async () => {
const settings = { ...presetConfig, efficient_profile_preset: "unavailable-v8" };
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
@ -330,7 +405,7 @@ describe("forecast classifier form", () => {
fireEvent.click(screen.getByRole("tab", { name: "Complexity" }));
fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) }));
await user.click(screen.getByRole("combobox", { name: "Classifier Model" }));
await user.click(screen.getByRole("option", { name: "judge", exact: true }));
await user.click(screen.getByRole("option", { name: "judge" }));
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
const output = screen.getByRole("status", { name: "Saved configuration" });
expect(output).toHaveTextContent('"classification_rubric":"agentic"');

View file

@ -245,7 +245,7 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions
value={fuse.max_quality_gap}
min={0}
max={1}
help="Allowed difference between capable and efficient success probabilities, from 0 to 1. This is an estimate, not a measured quality guarantee"
help="Allowed difference between capable and efficient success probabilities, from 0 to 1. Tune on held-out tasks from your workload; this estimate is not a measured quality guarantee. A gap of 0 still selects efficient on tied or higher forecasts. Route directly to one model to avoid judging when you do not want model selection"
onChange={(max_quality_gap) => updateFuse({ ...fuse, max_quality_gap })}
/>
</>

View file

@ -1,9 +1,15 @@
import React from "react";
import { $api } from "@/lib/http/api";
import { SearchSelect } from "@/components/shared/SearchSelect";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import { Textarea } from "@/components/ui/textarea";
import { fuseProfileFields, selectFuseProfile, type FuseSettings } from "./forecast_classifier_config";
import {
fuseProfileFields,
selectFuseProfile,
type FuseProfileField,
type FuseSettings,
} from "./forecast_classifier_config";
const catalogQueryOptions = {
staleTime: Infinity,
@ -14,6 +20,142 @@ const catalogQueryOptions = {
refetchOnReconnect: false,
};
const profileLabels: Readonly<Record<FuseProfileField, string>> = {
efficient_profile: "Efficient solver profile",
capable_profile: "Capable solver profile",
harness: "Harness and budget",
};
type FusePresetEntry = {
id: string;
label: string;
text: string;
sources: readonly string[];
model?: string;
};
type FieldProps = {
id: string;
field: FuseProfileField;
value: FuseSettings;
onChange: (value: FuseSettings) => void;
presets: readonly FusePresetEntry[] | undefined;
catalogVersion: string | undefined;
customWithoutPreview: ReadonlySet<FuseProfileField>;
setCustomWithoutPreview: React.Dispatch<React.SetStateAction<ReadonlySet<FuseProfileField>>>;
};
const profileSelectionLabel = (awaitingCustomText: boolean, custom: boolean): string => {
if (awaitingCustomText) return "Saved preset remains active until replacement text is entered";
if (custom) return "Custom text overrides preset";
return "Preset";
};
function FuseProfilePresetField({
id,
field,
value,
onChange,
presets,
catalogVersion,
customWithoutPreview,
setCustomWithoutPreview,
}: FieldProps) {
const label = profileLabels[field];
const presetId = value[`${field}_preset`];
const preset = presets?.find((entry) => entry.id === presetId);
const awaitingCustomText = customWithoutPreview.has(field) && value[field] == null && presetId != null;
const custom = value[field] != null || presetId == null || awaitingCustomText;
const effectiveText = value[field] ?? preset?.text ?? "";
const selectionLabel = profileSelectionLabel(awaitingCustomText, custom);
const chooseProfile = (selected: string | null) => {
if (!selected) return;
const missingPreview = presetId != null && preset == null && value[field] == null;
if (selected === "custom" && missingPreview) {
setCustomWithoutPreview((fields) => new Set([...fields, field]));
return;
}
setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field)));
onChange(selectFuseProfile(value, field, selected === "custom" ? undefined : selected, effectiveText));
};
const editProfile = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
if (customWithoutPreview.has(field) && event.target.value.trim().length > 0) {
setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field)));
onChange(selectFuseProfile(value, field, undefined, event.target.value));
return;
}
onChange({ ...value, [field]: event.target.value });
};
const keepSavedPreset = () => {
if (presetId == null) return;
setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field)));
onChange(selectFuseProfile(value, field, presetId, ""));
};
const placeholder =
field === "harness"
? "Tools, execution environment, verification, and budget available to each solver"
: "Describe this solver's strengths, limitations, and settings";
return (
<div className="space-y-2 min-w-0">
<Label htmlFor={`${id}-${field}-preset`}>{label} preset</Label>
<SearchSelect
inputId={`${id}-${field}-preset`}
aria-label={`${label} preset`}
value={custom ? "custom" : presetId}
allowClear={false}
options={[
{ value: "custom", label: "Custom" },
...(presets ?? []).map((entry) => ({ value: entry.id, label: entry.label, sublabel: entry.id })),
]}
onValueChange={chooseProfile}
/>
<Textarea
aria-label={label}
value={effectiveText}
readOnly={!custom}
maxLength={4000}
rows={4}
placeholder={placeholder}
onChange={editProfile}
/>
{customWithoutPreview.has(field) && presetId != null && (
<Button
type="button"
variant="outline"
size="sm"
aria-label={`Keep saved ${label.toLowerCase()} preset`}
onClick={keepSavedPreset}
>
Keep saved preset
</Button>
)}
{presetId && (
<div className="space-y-1 text-xs text-muted-foreground break-words">
<p>
{selectionLabel}: {presetId}
</p>
{preset ? (
<>
<p>Catalog version: {catalogVersion}</p>
{preset.model && <p>Model: {preset.model}</p>}
<div className="flex flex-wrap gap-x-3 gap-y-1">
{preset.sources.map((source, index) => (
<a key={source} href={source} target="_blank" rel="noopener noreferrer" className="underline">
Source {index + 1}
</a>
))}
</div>
</>
) : (
<p>Preset preview unavailable. The saved reference is preserved</p>
)}
</div>
)}
</div>
);
}
export default function FuseProfilePresets({
value,
onChange,
@ -22,6 +164,9 @@ export default function FuseProfilePresets({
onChange: (value: FuseSettings) => void;
}) {
const id = React.useId();
const [customWithoutPreview, setCustomWithoutPreview] = React.useState<ReadonlySet<FuseProfileField>>(
() => new Set(),
);
const { data, isPending, isError } = $api.useQuery(
"get",
"/public/complexity_router/fuse_presets",
@ -32,7 +177,8 @@ export default function FuseProfilePresets({
<div className="space-y-4 min-w-0">
<p className="text-xs text-muted-foreground">
Choose profiles that match every deployment in each solver group and its actual settings. Profile selection is
independent of routing model names
independent of routing model names. Presets describe the solvers and runtime; they do not set a quality gap or
calibration. Validate those separately for your workload, judge, and exact profile versions.
</p>
{isPending && (
<p role="status" className="text-xs text-muted-foreground">
@ -44,74 +190,19 @@ export default function FuseProfilePresets({
Profile presets could not be loaded. Saved references are preserved and Custom editing is available
</p>
)}
{fuseProfileFields.map((field) => {
const label = {
efficient_profile: "Efficient solver profile",
capable_profile: "Capable solver profile",
harness: "Harness and budget",
}[field];
const presetId = value[`${field}_preset`];
const presets = field === "harness" ? data?.harnesses : data?.models;
const preset = presets?.find((entry) => entry.id === presetId);
const custom = value[field] != null || presetId == null;
const effectiveText = value[field] ?? preset?.text ?? "";
return (
<div key={field} className="space-y-2 min-w-0">
<Label htmlFor={`${id}-${field}-preset`}>{label} preset</Label>
<SearchSelect
inputId={`${id}-${field}-preset`}
aria-label={`${label} preset`}
value={custom ? "custom" : presetId}
allowClear={false}
options={[
{ value: "custom", label: "Custom" },
...(presets ?? []).map((entry) => ({ value: entry.id, label: entry.label, sublabel: entry.id })),
]}
onValueChange={(selected) => {
if (selected)
onChange(
selectFuseProfile(value, field, selected === "custom" ? undefined : selected, effectiveText),
);
}}
/>
<Textarea
aria-label={label}
value={effectiveText}
readOnly={!custom}
maxLength={4000}
rows={4}
placeholder={
field === "harness"
? "Tools, execution environment, verification, and budget available to each solver"
: "Describe this solver's strengths, limitations, and settings"
}
onChange={(event) => onChange({ ...value, [field]: event.target.value })}
/>
{presetId && (
<div className="space-y-1 text-xs text-muted-foreground break-words">
<p>
{custom ? "Custom text overrides preset" : "Preset"}: {presetId}
</p>
{preset ? (
<>
<p>Catalog version: {data?.version}</p>
{"model" in preset && typeof preset.model === "string" && <p>Model: {preset.model}</p>}
<div className="flex flex-wrap gap-x-3 gap-y-1">
{preset.sources.map((source, index) => (
<a key={source} href={source} target="_blank" rel="noopener noreferrer" className="underline">
Source {index + 1}
</a>
))}
</div>
</>
) : (
<p>Preset preview unavailable. The saved reference is preserved</p>
)}
</div>
)}
</div>
);
})}
{fuseProfileFields.map((field) => (
<FuseProfilePresetField
key={field}
id={id}
field={field}
value={value}
onChange={onChange}
presets={field === "harness" ? data?.harnesses : data?.models}
catalogVersion={data?.version}
customWithoutPreview={customWithoutPreview}
setCustomWithoutPreview={setCustomWithoutPreview}
/>
))}
</div>
);
}