diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d9159cea426..6505746bca1 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -179,51 +179,6 @@ Configure capability forecasting through YAML or the model-management API. The dashboard preserves its classifier and calibration on an untouched save; it does not provide a capability-card editor -### Fuse v2 profile presets - -Fuse v2 accepts maintained model and runtime descriptions instead of requiring -custom prose for both solvers and the harness. Select profiles explicitly for -all deployments behind your configured model groups and their actual settings. -Group names do not select profiles automatically - -```yaml -complexity_router_config: - classifier_type: llm_v2 - classifier_llm_config: - model: your-judge-group - tiers: - SIMPLE: your-efficient-group - REASONING: your-capable-group - llm_v2_config: - efficient_profile_preset: claude-sonnet-5-v1 - capable_profile_preset: claude-fable-5-1-v1 - harness_preset: claude-code-v1 - max_quality_gap: 0.05 -``` - -`GET /public/complexity_router/fuse_presets` returns the catalog version, model -profiles, and runtime descriptions, including source URLs. The bundled catalog -is loaded once per process without network requests. Sources are citations only - -Each of `efficient_profile`, `capable_profile`, and `harness` requires either -nonblank custom text or its corresponding preset reference. Custom text wins -when both are supplied, but an unknown or wrong-kind preset is still rejected. -Explicit blank text is invalid even with a valid preset. Custom text remains -limited to 4000 characters - -Saved configurations retain preset references and explicit text separately. -Preset text is resolved when building the classifier prompt, not copied into -stored custom fields. Existing all-custom configurations keep the same prompt. -Versioned preset IDs identify immutable content: revised wording receives a new -ID, and older referenced entries must remain available - -The runtime presets do not imply a repository, runnable tests, network access, -additional tools, or a step, time, or spending budget. mini-SWE-agent describes -an agent interface, not a SWE-bench task. Model descriptions summarize provider -positioning without solve rates or guaranteed rankings. Wording is an evaluation -input, not a calibrated quality claim. Existing Fuse licensing, policy, -calibration, and prompt version are unchanged - ### Heuristic v2 Set `classifier_type: heuristic_v2` to classify with the bundled calibrated diff --git a/tests/test_litellm/router_strategy/test_fuse_presets.py b/tests/test_litellm/router_strategy/test_fuse_presets.py index 0b8d936383b..0cd4b1f660b 100644 --- a/tests/test_litellm/router_strategy/test_fuse_presets.py +++ b/tests/test_litellm/router_strategy/test_fuse_presets.py @@ -1,6 +1,7 @@ import json +from hashlib import sha256 from importlib.resources import files -from typing import Final +from typing import Final, Literal import pytest from pydantic import ValidationError @@ -19,11 +20,36 @@ def test_catalog_is_loaded_once_and_preserves_bundled_content() -> None: assert first.model_dump(mode="json") == bundled entries: Final = (*first.models, *first.harnesses) assert len({entry.id for entry in entries}) == len(entries) - assert len(first.models) == 9 - assert len(first.harnesses) == 5 assert all(entry.sources and all(source.startswith("https://") for source in entry.sources) for entry in entries) +@pytest.mark.parametrize( + ("kind", "preset_id", "expected_digest"), + ( + ("model", "gpt-6-astra-v1", "a9403b0c00ea64081b7b08b5b968850670f3a047d219a7e0668f2169146ae96e"), + ("model", "gpt-5.6-sol-v1", "2b91a6c43e0e93183aaaf9c355e1bbb8ed2e9817aab6b0c2f50148f53a23247b"), + ("model", "gpt-5.6-luna-v1", "fff94a9e01bf4519798d5be4e76a3f9d57b75a2d9966a59dc92cbfeb5cd08d07"), + ("model", "gpt-5.6-terra-v1", "75de040f3bea841fa4764885738303893ee7ac0804aed1e932cd3959185ff893"), + ("model", "claude-haiku-4-5-v1", "91c1920953073462b6b70ef810596a5325f08286b5e62630cff47938fc4157db"), + ("model", "claude-sonnet-5-v1", "133f4414c644a707cd8cf565a486153856f4836ca4e4f75ee0553f2b7a1e3663"), + ("model", "claude-opus-5-v1", "9cbfcae45d2e3a2575e44ce5adf618f56614abff4b3221d35900c647200b99ef"), + ("model", "claude-fable-5-v1", "25c275d7403f1572ffb4fe899d5feecd9a434ebdc37b4dd9ef601a8ecf4850fc"), + ("model", "claude-fable-5-1-v1", "37693107c878ab6266530395bbdc2d2813d676d179bf281d05e5a5ec1b9d4c60"), + ("harness", "unspecified-v1", "d9eb30b61509456f0c71ca805b33d821cab6605578d567a29ab421d8f602ce7b"), + ("harness", "claude-code-v1", "7ee8e9d50f1cf44a8a58461efff66d6182f245d25499702c144d1c642c101ed9"), + ("harness", "codex-cli-v1", "0678047e34562ef05b5e2fba099c1f9e5876304f7eaf3b0d8c3809e707eb3311"), + ("harness", "opencode-v1", "8b6cc240d90091ac2ef9b374b535f981a55abb91e25d4c04fdb9fc206eeb907e"), + ("harness", "mini-swe-agent-v1", "21e2dc4a8a2320a5a554a498b30516326dc3592ebf20b0f4db20ddb33e879a39"), + ), +) +def test_existing_preset_text_is_unchanged( + kind: Literal["model", "harness"], preset_id: str, expected_digest: str +) -> None: + text: Final = resolve_fuse_profile(None, preset_id, kind) + assert text is not None + assert sha256(text.encode("utf-8")).hexdigest() == expected_digest + + def test_every_catalog_entry_resolves_without_changing_custom_ownership() -> None: catalog: Final = get_fuse_presets() for entry in catalog.models: diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx index f9b0edf9508..a03ccb11456 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx @@ -159,7 +159,7 @@ describe("forecast classifier form", () => { const effectiveText = override ?? catalog.models[0].text; await waitFor(() => expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText)); await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" })); - await user.click(screen.getByRole("option", { name: "Custom", exact: true })); + await user.click(screen.getByRole("option", { name: "Custom" })); expect(screen.getByLabelText("Efficient solver profile")).not.toHaveAttribute("readonly"); expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText); fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Custom budget" } }); @@ -219,6 +219,81 @@ describe("forecast classifier form", () => { }, ); + it.each([ + ["efficient_profile", "Efficient solver profile"], + ["capable_profile", "Capable solver profile"], + ["harness", "Harness and budget"], + ] as const)( + "preserves the saved %s reference during a catalog outage until Custom text replaces it", + async (field, label) => { + const user = userEvent.setup(); + vi.mocked(fetch).mockImplementation(async () => Response.json({ error: "unavailable" }, { status: 503 })); + renderWithProviders(
); + expect(await screen.findByText(/Profile presets could not be loaded/)).toBeInTheDocument(); + const save = screen.getByRole("button", { name: "Save configuration" }); + const output = screen.getByRole("status", { name: "Saved configuration" }); + expect(save).toBeEnabled(); + await user.click(save); + expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig); + + await user.click(screen.getByRole("combobox", { name: `${label} preset` })); + await user.click(screen.getByRole("option", { name: "Custom" })); + expect(screen.getByLabelText(label)).toHaveValue(""); + expect(screen.getByLabelText(label)).not.toHaveAttribute("readonly"); + expect(screen.getByRole("combobox", { name: `${label} preset` })).toHaveValue("Custom"); + expect(save).toBeEnabled(); + await user.click(save); + expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig); + + fireEvent.change(screen.getByLabelText(label), { target: { value: " " } }); + expect(save).toBeDisabled(); + await user.click(screen.getByRole("button", { name: `Keep saved ${label.toLowerCase()} preset` })); + expect(screen.getByLabelText(label)).toHaveAttribute("readonly"); + expect(save).toBeEnabled(); + await user.click(save); + expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig); + + await user.click(screen.getByRole("combobox", { name: `${label} preset` })); + await user.click(screen.getByRole("option", { name: "Custom" })); + const replacement = "Manually authored replacement"; + fireEvent.change(screen.getByLabelText(label), { target: { value: replacement } }); + expect(save).toBeEnabled(); + await user.click(save); + const referenceKey = `${field}_preset` as const; + const { [referenceKey]: _reference, ...remaining } = presetConfig; + expect(JSON.parse(output.textContent!).llm_v2_config).toEqual({ ...remaining, [field]: replacement }); + }, + ); + + it.each([true, false])( + "keeps a reference selected as Custom while the catalog settles, success=%s", + async (success) => { + const user = userEvent.setup(); + const response = Promise.withResolvers(); + vi.mocked(fetch).mockReturnValue(response.promise); + renderWithProviders(); + await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" })); + await user.click(screen.getByRole("option", { name: "Custom" })); + await act(async () => response.resolve(success ? Response.json(catalog) : Response.json({}, { status: 503 }))); + if (success) await screen.findAllByText(`Catalog version: ${catalog.version}`); + else await screen.findByText(/Profile presets could not be loaded/); + expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(success ? catalog.models[0].text : ""); + await user.click(screen.getByRole("button", { name: "Save configuration" })); + expect( + JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config, + ).toEqual(presetConfig); + fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Replacement" } }); + await user.click(screen.getByRole("button", { name: "Save configuration" })); + const { efficient_profile_preset: _reference, ...remaining } = presetConfig; + expect( + JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config, + ).toEqual({ + ...remaining, + efficient_profile: "Replacement", + }); + }, + ); + it("keeps unknown saved IDs visible with unavailable previews rather than replacing them", async () => { const settings = { ...presetConfig, efficient_profile_preset: "unavailable-v8" }; renderWithProviders(); @@ -330,7 +405,7 @@ describe("forecast classifier form", () => { fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) })); await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); - await user.click(screen.getByRole("option", { name: "judge", exact: true })); + await user.click(screen.getByRole("option", { name: "judge" })); fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); const output = screen.getByRole("status", { name: "Saved configuration" }); expect(output).toHaveTextContent('"classification_rubric":"agentic"'); diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx index 556ba0aa1ee..c336423fe39 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx @@ -245,7 +245,7 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions value={fuse.max_quality_gap} min={0} max={1} - help="Allowed difference between capable and efficient success probabilities, from 0 to 1. This is an estimate, not a measured quality guarantee" + help="Allowed difference between capable and efficient success probabilities, from 0 to 1. Tune on held-out tasks from your workload; this estimate is not a measured quality guarantee. A gap of 0 still selects efficient on tied or higher forecasts. Route directly to one model to avoid judging when you do not want model selection" onChange={(max_quality_gap) => updateFuse({ ...fuse, max_quality_gap })} /> diff --git a/ui/litellm-dashboard/src/components/add_model/FuseProfilePresets.tsx b/ui/litellm-dashboard/src/components/add_model/FuseProfilePresets.tsx index a8a5803b708..f9312996b0d 100644 --- a/ui/litellm-dashboard/src/components/add_model/FuseProfilePresets.tsx +++ b/ui/litellm-dashboard/src/components/add_model/FuseProfilePresets.tsx @@ -1,9 +1,15 @@ import React from "react"; import { $api } from "@/lib/http/api"; import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Button } from "@/components/ui/button"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; -import { fuseProfileFields, selectFuseProfile, type FuseSettings } from "./forecast_classifier_config"; +import { + fuseProfileFields, + selectFuseProfile, + type FuseProfileField, + type FuseSettings, +} from "./forecast_classifier_config"; const catalogQueryOptions = { staleTime: Infinity, @@ -14,6 +20,142 @@ const catalogQueryOptions = { refetchOnReconnect: false, }; +const profileLabels: Readonly> = { + efficient_profile: "Efficient solver profile", + capable_profile: "Capable solver profile", + harness: "Harness and budget", +}; + +type FusePresetEntry = { + id: string; + label: string; + text: string; + sources: readonly string[]; + model?: string; +}; + +type FieldProps = { + id: string; + field: FuseProfileField; + value: FuseSettings; + onChange: (value: FuseSettings) => void; + presets: readonly FusePresetEntry[] | undefined; + catalogVersion: string | undefined; + customWithoutPreview: ReadonlySet; + setCustomWithoutPreview: React.Dispatch>>; +}; + +const profileSelectionLabel = (awaitingCustomText: boolean, custom: boolean): string => { + if (awaitingCustomText) return "Saved preset remains active until replacement text is entered"; + if (custom) return "Custom text overrides preset"; + return "Preset"; +}; + +function FuseProfilePresetField({ + id, + field, + value, + onChange, + presets, + catalogVersion, + customWithoutPreview, + setCustomWithoutPreview, +}: FieldProps) { + const label = profileLabels[field]; + const presetId = value[`${field}_preset`]; + const preset = presets?.find((entry) => entry.id === presetId); + const awaitingCustomText = customWithoutPreview.has(field) && value[field] == null && presetId != null; + const custom = value[field] != null || presetId == null || awaitingCustomText; + const effectiveText = value[field] ?? preset?.text ?? ""; + const selectionLabel = profileSelectionLabel(awaitingCustomText, custom); + const chooseProfile = (selected: string | null) => { + if (!selected) return; + const missingPreview = presetId != null && preset == null && value[field] == null; + if (selected === "custom" && missingPreview) { + setCustomWithoutPreview((fields) => new Set([...fields, field])); + return; + } + setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field))); + onChange(selectFuseProfile(value, field, selected === "custom" ? undefined : selected, effectiveText)); + }; + const editProfile = (event: React.ChangeEvent) => { + if (customWithoutPreview.has(field) && event.target.value.trim().length > 0) { + setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field))); + onChange(selectFuseProfile(value, field, undefined, event.target.value)); + return; + } + onChange({ ...value, [field]: event.target.value }); + }; + const keepSavedPreset = () => { + if (presetId == null) return; + setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field))); + onChange(selectFuseProfile(value, field, presetId, "")); + }; + const placeholder = + field === "harness" + ? "Tools, execution environment, verification, and budget available to each solver" + : "Describe this solver's strengths, limitations, and settings"; + + return ( +
+ + ({ value: entry.id, label: entry.label, sublabel: entry.id })), + ]} + onValueChange={chooseProfile} + /> +