mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
Merge pull request #42252 from BerriAI/litellm_heuristic_v2_success_threshold
feat(auto-router): configure heuristic v2 success threshold
This commit is contained in:
tin-berri
GitHub
commit
fe34fe6047
20 changed files with 518 additions and 21 deletions
|
|
@ -191,6 +191,7 @@ model_list:
|
|||
model: auto_router/complexity_router
|
||||
complexity_router_config:
|
||||
classifier_type: heuristic_v2
|
||||
heuristic_v2_success_threshold: 0.9
|
||||
tiers:
|
||||
SIMPLE: luna
|
||||
MEDIUM: terra
|
||||
|
|
@ -201,9 +202,18 @@ model_list:
|
|||
No classifier model call or per-model training data is required. The classifier
|
||||
uses global tier quality, request-type quality, and similar-request cohorts from
|
||||
the bundled UltraFeedback artifact. It estimates success at every tier, enforces
|
||||
monotonic probabilities, and returns the first tier meeting the trained 0.75
|
||||
threshold. The existing complexity-router tier pool then selects and dispatches
|
||||
a model from that tier
|
||||
monotonic probabilities, and returns the first tier meeting the success threshold,
|
||||
or REASONING if no tier meets it. The existing complexity-router tier pool then
|
||||
selects and dispatches a model from that tier
|
||||
|
||||
Set `heuristic_v2_success_threshold` to a value from 0 to 1 to override the
|
||||
artifact's threshold. For example, `0.9` requires a predicted success probability
|
||||
of at least 90%. Higher thresholds favor more capable tiers. Omit the setting or
|
||||
set it to `null` to use the artifact's `routing_threshold`, which is `0.75` for
|
||||
the bundled artifact. The override leaves the predicted probabilities unchanged
|
||||
|
||||
In the dashboard, select Heuristic v2 under Advanced: Classification Method and
|
||||
set Success threshold. Clear the field to restore the artifact's default
|
||||
|
||||
Spend logs record `routing_decision.cause: heuristic_v2`, the detected request
|
||||
type, and all four predicted probabilities. Existing `classifier_type: heuristic`
|
||||
|
|
|
|||
|
|
@ -1429,7 +1429,10 @@ class ComplexityRouter(CustomLogger):
|
|||
_ClassifierCircuitBreaker(circuit_breaker_cooldown) if circuit_breaker_cooldown is not None else None
|
||||
)
|
||||
self._tier_success_predictor: TierSuccessPredictor | None = (
|
||||
TierSuccessPredictor(resolve_tier_artifact(self.config.heuristic_v2_artifact))
|
||||
TierSuccessPredictor(
|
||||
resolve_tier_artifact(self.config.heuristic_v2_artifact),
|
||||
routing_threshold=self.config.heuristic_v2_success_threshold,
|
||||
)
|
||||
if self.config.classifier_type == "heuristic_v2"
|
||||
else None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1036,6 +1036,18 @@ class ComplexityRouterConfig(BaseModel):
|
|||
"UltraFeedback artifact is selected by default; an inline trained artifact may replace it"
|
||||
),
|
||||
)
|
||||
heuristic_v2_success_threshold: float | None = Field(
|
||||
default=None,
|
||||
strict=True,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description=(
|
||||
"Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. "
|
||||
"The first tier meeting this threshold is selected, or REASONING if none meets it. "
|
||||
"When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). "
|
||||
"Other classifier types ignore this setting"
|
||||
),
|
||||
)
|
||||
classifier_llm_config: ClassifierLLMConfig | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
|
|
|||
|
|
@ -108,8 +108,9 @@ class TierPrediction:
|
|||
|
||||
|
||||
class TierSuccessPredictor:
|
||||
def __init__(self, artifact: TrainedTierArtifact) -> None:
|
||||
def __init__(self, artifact: TrainedTierArtifact, *, routing_threshold: float | None = None) -> None:
|
||||
self._artifact = artifact
|
||||
self._routing_threshold: Final = artifact.routing_threshold if routing_threshold is None else routing_threshold
|
||||
self._global: Mapping[int, TierGlobalStatistic] = MappingProxyType(
|
||||
{stat.tier: stat for stat in artifact.global_statistics}
|
||||
)
|
||||
|
|
@ -122,7 +123,7 @@ class TierSuccessPredictor:
|
|||
|
||||
@property
|
||||
def routing_threshold(self) -> float:
|
||||
return self._artifact.routing_threshold
|
||||
return self._routing_threshold
|
||||
|
||||
def predict(self, prompt: str, request_type: RequestType) -> TierPrediction:
|
||||
cohort: Final = similarity_cohort(prompt, request_type)
|
||||
|
|
@ -132,7 +133,7 @@ class TierSuccessPredictor:
|
|||
{int(tier): probability for tier, probability in zip(_TIERS, monotonic)}
|
||||
)
|
||||
required_tier: Final = next(
|
||||
(tier for tier in _TIERS if probabilities[tier] >= self._artifact.routing_threshold),
|
||||
(tier for tier in _TIERS if probabilities[tier] >= self.routing_threshold),
|
||||
4,
|
||||
)
|
||||
return TierPrediction(probabilities=probabilities, required_tier=required_tier)
|
||||
|
|
|
|||
|
|
@ -3722,16 +3722,37 @@ class TestLLMClassifier:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("redact", (False, True))
|
||||
@pytest.mark.parametrize(
|
||||
"override,threshold,tier,model",
|
||||
(
|
||||
({}, 0.8, "COMPLEX", "complex-model"),
|
||||
({"heuristic_v2_success_threshold": None}, 0.8, "COMPLEX", "complex-model"),
|
||||
({"heuristic_v2_success_threshold": 0.0}, 0.0, "SIMPLE", "simple-model"),
|
||||
({"heuristic_v2_success_threshold": 21 / 102}, 21 / 102, "MEDIUM", "medium-model"),
|
||||
({"heuristic_v2_success_threshold": 0.95}, 0.95, "REASONING", "reasoning-model"),
|
||||
({"heuristic_v2_success_threshold": 1.0}, 1.0, "REASONING", "reasoning-model"),
|
||||
),
|
||||
ids=("omitted", "null", "zero", "inclusive", "higher", "no-tier-passes"),
|
||||
)
|
||||
async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(
|
||||
self, mock_router_instance: MagicMock, redact: bool, monkeypatch: pytest.MonkeyPatch
|
||||
self,
|
||||
mock_router_instance: MagicMock,
|
||||
redact: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
override: Mapping[str, float | None],
|
||||
threshold: float,
|
||||
tier: str,
|
||||
model: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
|
||||
router = ComplexityRouter(
|
||||
artifact: Final = _heuristic_v2_artifact()
|
||||
router: Final = ComplexityRouter(
|
||||
model_name="tier-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={
|
||||
"classifier_type": "heuristic_v2",
|
||||
"heuristic_v2_artifact": _heuristic_v2_artifact(),
|
||||
"heuristic_v2_artifact": artifact,
|
||||
**override,
|
||||
"tiers": {
|
||||
"SIMPLE": "simple-model",
|
||||
"MEDIUM": "medium-model",
|
||||
|
|
@ -3741,15 +3762,15 @@ class TestLLMClassifier:
|
|||
},
|
||||
)
|
||||
|
||||
response = await router.async_pre_routing_hook(
|
||||
response: Final = await router.async_pre_routing_hook(
|
||||
model="tier-router",
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": "Handle this new request"}],
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "complex-model"
|
||||
assert response.routing_decision["tier"] == "COMPLEX"
|
||||
assert response.model == model
|
||||
assert response.routing_decision["tier"] == tier
|
||||
assert response.routing_decision["cause"] == "heuristic_v2"
|
||||
assert response.routing_decision["signals"] == [
|
||||
"request-type:general",
|
||||
|
|
@ -3769,10 +3790,62 @@ class TestLLMClassifier:
|
|||
"COMPLEX": 91 / 102,
|
||||
"REASONING": 100 / 102,
|
||||
},
|
||||
"threshold": 0.8,
|
||||
"predicted_tier": "COMPLEX",
|
||||
"threshold": threshold,
|
||||
"predicted_tier": tier,
|
||||
"request_type": "general",
|
||||
}
|
||||
assert artifact.routing_threshold == 0.8
|
||||
|
||||
@pytest.mark.parametrize("threshold", (-0.01, 1.01, math.nan, math.inf, -math.inf, True, "0.95"))
|
||||
def test_heuristic_v2_success_threshold_rejects_invalid_values(self, threshold: float | bool | str) -> None:
|
||||
with pytest.raises(ValidationError, match="heuristic_v2_success_threshold"):
|
||||
ComplexityRouterConfig.model_validate(
|
||||
{"classifier_type": "heuristic_v2", "heuristic_v2_success_threshold": threshold}
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_heuristic_v2_threshold_reload_and_rejected_update_keep_router_isolated(self) -> None:
|
||||
artifact: Final = _heuristic_v2_artifact()
|
||||
|
||||
def deployment(threshold: float, name: str = "editable") -> Deployment:
|
||||
return Deployment(
|
||||
model_name=name,
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="auto_router/complexity_router",
|
||||
complexity_router_config={
|
||||
"classifier_type": "heuristic_v2",
|
||||
"heuristic_v2_artifact": artifact.model_dump(),
|
||||
"heuristic_v2_success_threshold": threshold,
|
||||
"session_affinity": False,
|
||||
"tiers": {"SIMPLE": "simple-model", "REASONING": "reasoning-model"},
|
||||
},
|
||||
),
|
||||
model_info={"id": name},
|
||||
)
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
deployment(0.95).model_dump(exclude_none=True),
|
||||
deployment(0.95, "unchanged").model_dump(exclude_none=True),
|
||||
],
|
||||
ignore_invalid_deployments=True,
|
||||
)
|
||||
|
||||
async def routed_threshold(name: str) -> tuple[str, float]:
|
||||
response: Final = await router.async_pre_routing_hook(
|
||||
model=name,
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": "Handle this new request"}],
|
||||
)
|
||||
assert response is not None and response.routing_decision is not None
|
||||
return response.model, response.routing_decision["heuristic_v2_forecast"]["threshold"]
|
||||
|
||||
assert await routed_threshold("editable") == ("reasoning-model", 0.95)
|
||||
assert router.upsert_deployment(deployment(0.0)) is not None
|
||||
assert await routed_threshold("editable") == ("simple-model", 0.0)
|
||||
assert await routed_threshold("unchanged") == ("reasoning-model", 0.95)
|
||||
assert router.upsert_deployment(deployment(1.01)) is None
|
||||
assert await routed_threshold("editable") == ("simple-model", 0.0)
|
||||
|
||||
def test_heuristic_v2_needs_no_classifier_model(self):
|
||||
config = ComplexityRouterConfig(classifier_type="heuristic_v2")
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@ vi.mock("../networking", () => ({
|
|||
|
||||
const CONFIG = {
|
||||
tiers: { SIMPLE: ["cheap"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["o3"] },
|
||||
classifier_type: "heuristic",
|
||||
classifier_type: "heuristic_v2",
|
||||
heuristic_v2_success_threshold: 0,
|
||||
} as unknown as ComplexityRouterConfigPayload;
|
||||
|
||||
const Harness = () => (
|
||||
|
|
@ -62,6 +63,22 @@ describe("AutoRouterRoutingTest", () => {
|
|||
expect(screen.getByTestId("auto-router-routing-test-send")).toBeDisabled();
|
||||
});
|
||||
|
||||
it("blocks previewing an invalid success threshold instead of sending NaN as null", () => {
|
||||
renderWithProviders(
|
||||
<AutoRouterRoutingTest
|
||||
accessToken="token"
|
||||
config={{ ...CONFIG, heuristic_v2_success_threshold: Number.NaN }}
|
||||
defaultModel="mid"
|
||||
routerName="my-router"
|
||||
teamId={undefined}
|
||||
/>,
|
||||
);
|
||||
fireEvent.change(screen.getByTestId("auto-router-routing-test-prompt"), { target: { value: "hello" } });
|
||||
expect(screen.getByTestId("auto-router-routing-test-send")).toBeDisabled();
|
||||
expect(screen.getByText("Success threshold must be a number between 0 and 1")).toBeVisible();
|
||||
expect(testAutoRouterRouting).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes the typed prompt through the config being edited and shows where it landed", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(testAutoRouterRouting).mockResolvedValue(successResponse);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { Button } from "@/components/ui/button";
|
|||
import { Textarea } from "@/components/ui/textarea";
|
||||
import RoutingDecisionCard from "@/components/view_logs/LogDetailsDrawer/RoutingDecisionCard";
|
||||
import { AutoRouterRoutingTestResult, testAutoRouterRouting } from "../networking";
|
||||
import { ComplexityRouterConfigPayload } from "./build_complexity_router_config";
|
||||
import { ComplexityRouterConfigPayload, getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
|
||||
import { buildAutoRouterRoutingTestRequest } from "./build_auto_router_routing_test_request";
|
||||
|
||||
interface AutoRouterRoutingTestProps {
|
||||
|
|
@ -31,8 +31,10 @@ const AutoRouterRoutingTest: React.FC<AutoRouterRoutingTestProps> = ({
|
|||
}) => {
|
||||
const [prompt, setPrompt] = React.useState<string>("");
|
||||
const [state, setState] = React.useState<TestState>({ status: "idle" });
|
||||
const configError = getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold);
|
||||
|
||||
const send = async () => {
|
||||
if (configError) return;
|
||||
setState({ status: "running" });
|
||||
const params = { prompt, config, defaultModel, routerName, teamId };
|
||||
const request = buildAutoRouterRoutingTestRequest(params);
|
||||
|
|
@ -62,13 +64,15 @@ const AutoRouterRoutingTest: React.FC<AutoRouterRoutingTestProps> = ({
|
|||
<div className="flex justify-end">
|
||||
<Button
|
||||
onClick={send}
|
||||
disabled={prompt.trim().length === 0 || state.status === "running"}
|
||||
disabled={prompt.trim().length === 0 || state.status === "running" || Boolean(configError)}
|
||||
data-testid="auto-router-routing-test-send"
|
||||
>
|
||||
{state.status === "running" ? "Routing..." : "Send Test Prompt"}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{configError && <p className="text-sm text-destructive">{configError}</p>}
|
||||
|
||||
{state.status === "failed" && (
|
||||
<div
|
||||
className="rounded-md border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive"
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { MultiSelect } from "@/components/shared/MultiSelect";
|
|||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
|
||||
|
|
@ -17,6 +18,7 @@ import HeuristicScoringConfig from "./HeuristicScoringConfig";
|
|||
import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect";
|
||||
import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
|
||||
import ClassifierVisionConfig from "./ClassifierVisionConfig";
|
||||
import { getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
|
||||
import type { ReasoningEffort } from "./complexity_router_tiers";
|
||||
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
|
||||
import {
|
||||
|
|
@ -48,13 +50,15 @@ const DEFAULT_SCORING_EXPLANATION =
|
|||
"The weighted score determines the tier:";
|
||||
|
||||
const HEURISTIC_V2_EXPLANATION =
|
||||
"The router estimates success probability for all four tiers with the bundled calibrated model, then selects " +
|
||||
"the first tier that meets its trained threshold. It runs locally with no classifier API call.";
|
||||
"The router estimates success probability for all four tiers with its calibrated model, then selects " +
|
||||
"the first tier that meets the success threshold. If none qualify, it selects Reasoning. " +
|
||||
"It runs locally with no classifier API call.";
|
||||
|
||||
const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms";
|
||||
const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
|
||||
const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
|
||||
const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin";
|
||||
const HEURISTIC_V2_SUCCESS_THRESHOLD_ID = "heuristic-v2-success-threshold";
|
||||
|
||||
const CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK =
|
||||
"This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " +
|
||||
|
|
@ -165,6 +169,39 @@ interface ClassificationMethodConfigProps {
|
|||
defaultModel?: string;
|
||||
}
|
||||
|
||||
export const InactiveHeuristicV2Threshold: React.FC<Pick<ClassificationMethodConfigProps, "value" | "onChange">> = ({
|
||||
value,
|
||||
onChange,
|
||||
}) => {
|
||||
const threshold = value.heuristic_v2_success_threshold;
|
||||
if (effectiveClassifierType(value) === "heuristic_v2" || threshold === undefined) return null;
|
||||
const error = getHeuristicV2SuccessThresholdError(threshold);
|
||||
return (
|
||||
<section aria-label="Inactive Heuristic v2 threshold" className="mb-4 space-y-2 rounded-md border p-3">
|
||||
<p className="text-sm font-medium">
|
||||
Heuristic v2 success threshold (inactive):{" "}
|
||||
<output aria-label="Retained Heuristic v2 threshold">
|
||||
{Number.isFinite(threshold) ? threshold : "Invalid value"}
|
||||
</output>
|
||||
</p>
|
||||
<p className="text-sm text-muted-foreground">Only used when Heuristic v2 is selected</p>
|
||||
{error && (
|
||||
<p className="text-sm text-destructive" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => onChange({ ...value, heuristic_v2_success_threshold: undefined })}
|
||||
>
|
||||
Clear Heuristic v2 threshold
|
||||
</Button>
|
||||
</section>
|
||||
);
|
||||
};
|
||||
|
||||
const ClassifierTypeRadios: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
classifierType: ClassifierType;
|
||||
|
|
@ -259,6 +296,12 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
const classifierModel = value.classifier_llm_config?.model ?? "";
|
||||
const classifierReasoningEffort = value.classifier_llm_config?.reasoning_effort;
|
||||
const explicitlySupportedClassifierEfforts = effortOptionsByModel[classifierModel];
|
||||
const successThresholdError = getHeuristicV2SuccessThresholdError(value.heuristic_v2_success_threshold);
|
||||
const successThresholdDraft =
|
||||
draft?.id === HEURISTIC_V2_SUCCESS_THRESHOLD_ID &&
|
||||
Object.is(value.heuristic_v2_success_threshold, draft.raw.trim() === "" ? undefined : Number(draft.raw))
|
||||
? draft.raw
|
||||
: null;
|
||||
|
||||
const handleClassifierTypeChange = (classifierType: ClassifierType) => {
|
||||
onChange(transitionClassifierType(value, classifierType));
|
||||
|
|
@ -275,6 +318,14 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
onChange({ ...value, hybrid_boundary_margin: Math.min(1, Math.max(0, parsed)) });
|
||||
};
|
||||
|
||||
const handleSuccessThresholdChange = (raw: string) => {
|
||||
setDraft({ id: HEURISTIC_V2_SUCCESS_THRESHOLD_ID, raw });
|
||||
onChange({
|
||||
...value,
|
||||
heuristic_v2_success_threshold: raw.trim() === "" ? undefined : Number(raw),
|
||||
});
|
||||
};
|
||||
|
||||
// One write for everything the prompt dialog owns. The rubric arrives here rather than through the
|
||||
// rubric handler because two onChange calls in one tick would both spread this render's `value`,
|
||||
// so whichever landed second would drop the other's edit.
|
||||
|
|
@ -407,6 +458,36 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
<>
|
||||
<ClassifierTypeRadios value={value} classifierType={classifierType} onTypeChange={handleClassifierTypeChange} />
|
||||
|
||||
{classifierType === "heuristic_v2" && (
|
||||
<div className="mt-4 space-y-2">
|
||||
<Label htmlFor={HEURISTIC_V2_SUCCESS_THRESHOLD_ID} className="block font-semibold">
|
||||
Success threshold
|
||||
</Label>
|
||||
<Input
|
||||
id={HEURISTIC_V2_SUCCESS_THRESHOLD_ID}
|
||||
type="text"
|
||||
inputMode="decimal"
|
||||
placeholder="Artifact default"
|
||||
value={successThresholdDraft ?? value.heuristic_v2_success_threshold?.toString() ?? ""}
|
||||
onChange={(event) => handleSuccessThresholdChange(event.target.value)}
|
||||
onBlur={() => {
|
||||
if (!successThresholdError) setDraft(null);
|
||||
}}
|
||||
aria-invalid={Boolean(successThresholdError)}
|
||||
aria-describedby={`${HEURISTIC_V2_SUCCESS_THRESHOLD_ID}-help${successThresholdError ? ` ${HEURISTIC_V2_SUCCESS_THRESHOLD_ID}-error` : ""}`}
|
||||
/>
|
||||
<p id={`${HEURISTIC_V2_SUCCESS_THRESHOLD_ID}-help`} className="text-sm text-muted-foreground">
|
||||
Minimum predicted success probability, from 0 to 1. Higher values favor more capable tiers. Leave blank to
|
||||
use the artifact default
|
||||
</p>
|
||||
{successThresholdError && (
|
||||
<p id={`${HEURISTIC_V2_SUCCESS_THRESHOLD_ID}-error`} className="text-sm text-destructive" role="alert">
|
||||
{successThresholdError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{classifierType === "heuristic_first" && (
|
||||
<div className="mt-4 space-y-2">
|
||||
<strong className="block font-semibold">Decide locally up to</strong>
|
||||
|
|
|
|||
|
|
@ -153,6 +153,51 @@ describe("ComplexityRouterConfig", () => {
|
|||
expect(screen.queryByText(/Score < 0.15/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each<[string, Partial<ComplexityRouterConfigValue>]>([
|
||||
["heuristic", { classifier_type: "heuristic" }],
|
||||
["LLM", { classifier_type: "llm" }],
|
||||
["heuristic first", { classifier_type: "heuristic_first" }],
|
||||
["hybrid", { classifier_type: "hybrid" }],
|
||||
["Capability", { classifier_type: "capability" }],
|
||||
["Fuse v2", { classifier_type: "llm_v2" }],
|
||||
[
|
||||
"custom tiers",
|
||||
{
|
||||
classifier_type: "heuristic_v2",
|
||||
custom_tier_set: {
|
||||
tiers: [{ id: "review", name: "REVIEW", definition: "Review code", models: ["gpt-4"] }],
|
||||
fallback_tier_id: "review",
|
||||
},
|
||||
},
|
||||
],
|
||||
])("shows and clears an invalid inactive threshold under %s", (_label, overrides) => {
|
||||
const value = { ...defaultValue, ...overrides, heuristic_v2_success_threshold: Number.NaN };
|
||||
const onChange = vi.fn();
|
||||
renderWithProviders(<ComplexityRouterConfig {...baseProps} value={value} onChange={onChange} />);
|
||||
const retained = screen.getByRole("region", { name: "Inactive Heuristic v2 threshold" });
|
||||
expect(within(retained).getByRole("status", { name: "Retained Heuristic v2 threshold" })).toHaveTextContent(
|
||||
"Invalid value",
|
||||
);
|
||||
expect(within(retained).getByRole("alert")).toHaveTextContent("Success threshold must be a number between 0 and 1");
|
||||
fireEvent.click(within(retained).getByRole("button", { name: "Clear Heuristic v2 threshold" }));
|
||||
expect(onChange).toHaveBeenCalledWith({ ...value, heuristic_v2_success_threshold: undefined });
|
||||
});
|
||||
|
||||
it("shows an inactive zero threshold until explicitly cleared and hides the summary for active or absent values", () => {
|
||||
const onChange = vi.fn();
|
||||
const value = { ...defaultValue, heuristic_v2_success_threshold: 0 };
|
||||
const { rerender } = renderWithProviders(
|
||||
<ComplexityRouterConfig {...baseProps} value={value} onChange={onChange} />,
|
||||
);
|
||||
expect(screen.getByRole("status", { name: "Retained Heuristic v2 threshold" })).toHaveTextContent("0");
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
rerender(<ComplexityRouterConfig {...baseProps} value={{ ...value, classifier_type: "heuristic_v2" }} />);
|
||||
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
|
||||
rerender(<ComplexityRouterConfig {...baseProps} value={defaultValue} />);
|
||||
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show classifier fields and use the configured values when classifier_type is llm", () => {
|
||||
const llmValue: ComplexityRouterConfigValue = {
|
||||
...defaultValue,
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import {
|
|||
import React from "react";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
|
||||
import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
|
||||
import ResponseFormatControls from "./ResponseFormatControls";
|
||||
import StallEscalationConfig from "./StallEscalationConfig";
|
||||
|
|
@ -374,6 +374,7 @@ export interface ComplexityRouterConfigValue {
|
|||
/** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */
|
||||
default_model?: string;
|
||||
classifier_type: ClassifierType;
|
||||
heuristic_v2_success_threshold?: number;
|
||||
capability_classifier_config?: CapabilitySettings;
|
||||
llm_v2_config?: FuseSettings;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
|
|
@ -618,6 +619,8 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
)}
|
||||
</div>
|
||||
|
||||
<InactiveHeuristicV2Threshold value={value} onChange={onChange} />
|
||||
|
||||
{forecast ? (
|
||||
<>
|
||||
<ForecastSolverModels
|
||||
|
|
|
|||
|
|
@ -669,6 +669,81 @@ describe("AddAutoRouterTab", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("blocks invalid success thresholds and creates a heuristic v2 router with explicit zero", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
renderWithProviders(<Harness />);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "threshold-router" } });
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
|
||||
const threshold = screen.getByRole("textbox", { name: "Success threshold" });
|
||||
expect(threshold).toHaveValue("");
|
||||
fireEvent.change(threshold, { target: { value: "invalid" } });
|
||||
fireEvent.blur(threshold);
|
||||
expect(threshold).toHaveValue("invalid");
|
||||
expect(threshold).toHaveAttribute("aria-invalid", "true");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
expect(screen.getByTestId("auto-router-test-routing-btn")).toBeDisabled();
|
||||
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.01" } });
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0" } });
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
|
||||
classifier_type: "heuristic_v2",
|
||||
heuristic_v2_success_threshold: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("clears an invalid threshold draft when automatic setup replaces the configuration", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
const automaticSetup = await screen.findByRole("button", { name: "Configure automatically" });
|
||||
await waitFor(() => expect(automaticSetup).toBeEnabled());
|
||||
await user.click(automaticSetup);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "reset-threshold-router" } });
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } });
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
|
||||
await user.click(automaticSetup);
|
||||
expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveValue("");
|
||||
expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveAttribute("aria-invalid", "false");
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty(
|
||||
"heuristic_v2_success_threshold",
|
||||
);
|
||||
});
|
||||
|
||||
it("clears an invalid inactive threshold before creating the router", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
renderWithProviders(<Harness />);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "clear-threshold-router" } });
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "invalid" } });
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Clear Heuristic v2 threshold" }));
|
||||
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty(
|
||||
"heuristic_v2_success_threshold",
|
||||
);
|
||||
});
|
||||
|
||||
it("carries a context-window escalation opt-out through to the create payload", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import {
|
|||
buildComplexityRouterConfig,
|
||||
getKeywordTierRulesError,
|
||||
getClassifierModelError,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getClassifierReasoningEffortError,
|
||||
getMissingTiersError,
|
||||
getPlanModeTierError,
|
||||
|
|
@ -146,6 +147,7 @@ export const getSubmitBlockedReason = (
|
|||
getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
|
||||
getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
|
||||
getClassifierModelError(config) ??
|
||||
getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
|
||||
(heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ??
|
||||
getClassifierReasoningEffortError(config, modelInfo) ??
|
||||
getReferencedModelsError(referencedModelsParams, availability)
|
||||
|
|
@ -405,6 +407,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
classificationMode: complexityRouterConfig.classification_mode,
|
||||
tierLabels: complexityRouterConfig.tier_labels,
|
||||
classifierType: complexityRouterConfig.classifier_type,
|
||||
heuristicV2SuccessThreshold: complexityRouterConfig.heuristic_v2_success_threshold,
|
||||
capabilityClassifierConfig: complexityRouterConfig.capability_classifier_config,
|
||||
llmV2Config: complexityRouterConfig.llm_v2_config,
|
||||
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
normalizeClassifierLlmConfig,
|
||||
getKeywordTierRulesError,
|
||||
getClassifierModelError,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getClassifierReasoningEffortError,
|
||||
getMissingTiersError,
|
||||
hydrateCustomTierSet,
|
||||
|
|
@ -211,8 +212,28 @@ describe("buildComplexityRouterConfig", () => {
|
|||
expect(config.classifier_llm_config).toBeUndefined();
|
||||
expect(config.classifier_context_window_size).toBeUndefined();
|
||||
expect(config.classifier_fallback).toBeUndefined();
|
||||
expect(config).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
});
|
||||
|
||||
it.each([0, 0.95, 1])("serializes a heuristic v2 success threshold of %s", (heuristicV2SuccessThreshold) => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "heuristic_v2",
|
||||
heuristicV2SuccessThreshold,
|
||||
});
|
||||
expect(config.heuristic_v2_success_threshold).toBe(heuristicV2SuccessThreshold);
|
||||
});
|
||||
|
||||
it.each(["heuristic", "llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const)(
|
||||
"retains the inactive success threshold under %s",
|
||||
(classifierType) => {
|
||||
expect(
|
||||
buildComplexityRouterConfig({ ...baseParams, classifierType, heuristicV2SuccessThreshold: 0.91 })
|
||||
.heuristic_v2_success_threshold,
|
||||
).toBe(0.91);
|
||||
},
|
||||
);
|
||||
|
||||
it("includes classifier_context_window_size and classifier_context_budget_chars only when classifier_type is llm", () => {
|
||||
const params: BuildComplexityRouterConfigParams = {
|
||||
...baseParams,
|
||||
|
|
@ -884,6 +905,19 @@ describe("buildComplexityRouterConfig tier model params", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("getHeuristicV2SuccessThresholdError", () => {
|
||||
it.each([undefined, 0, 0.95, 1])("accepts the optional probability %s", (threshold) => {
|
||||
expect(getHeuristicV2SuccessThresholdError(threshold)).toBeNull();
|
||||
});
|
||||
|
||||
it.each([-0.01, 1.01, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])(
|
||||
"rejects invalid success threshold %s",
|
||||
(threshold) => {
|
||||
expect(getHeuristicV2SuccessThresholdError(threshold)).toBe("Success threshold must be a number between 0 and 1");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("getClassifierModelError", () => {
|
||||
it("stays quiet for a heuristic router, which needs no classifier model", () => {
|
||||
expect(getClassifierModelError({ classifier_type: "heuristic" })).toBeNull();
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ export interface StoredComplexityRouterConfig {
|
|||
hybrid_boundary_margin?: unknown;
|
||||
tier_labels?: unknown;
|
||||
classifier_type?: ClassifierType;
|
||||
heuristic_v2_success_threshold?: unknown;
|
||||
capability_classifier_config?: unknown;
|
||||
llm_v2_config?: unknown;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
|
|
@ -182,6 +183,7 @@ export interface BuildComplexityRouterConfigParams {
|
|||
planModeMinTier: string | undefined;
|
||||
tierLabels: ComplexityTierLabels | undefined;
|
||||
classifierType: ClassifierType;
|
||||
heuristicV2SuccessThreshold?: number;
|
||||
capabilityClassifierConfig?: CapabilitySettings;
|
||||
llmV2Config?: FuseSettings;
|
||||
classifierLlmConfig: ClassifierLLMConfigWire | undefined;
|
||||
|
|
@ -248,6 +250,7 @@ export interface ComplexityRouterConfigPayload {
|
|||
plan_mode_min_tier?: string;
|
||||
tier_labels?: ComplexityTierLabels;
|
||||
classifier_type: ClassifierType;
|
||||
heuristic_v2_success_threshold?: number;
|
||||
capability_classifier_config?: CapabilitySettings;
|
||||
llm_v2_config?: FuseSettings;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
|
|
@ -356,6 +359,12 @@ export const getKeywordTierRulesError = (
|
|||
return `Keyword rule(s) ${orphaned.join(", ")} route to a tier this router no longer has`;
|
||||
};
|
||||
|
||||
export const getHeuristicV2SuccessThresholdError = (threshold: number | undefined): string | null => {
|
||||
if (threshold === undefined) return null;
|
||||
const validProbability = Number.isFinite(threshold) && threshold >= 0 && threshold <= 1;
|
||||
return validProbability ? null : "Success threshold must be a number between 0 and 1";
|
||||
};
|
||||
|
||||
// An edited tier set forces the LLM classifier, so the model requirement follows the EFFECTIVE type.
|
||||
// Both forms' submit gates and their submit handlers read this one answer so they cannot drift.
|
||||
export const getClassifierModelError = (
|
||||
|
|
@ -557,6 +566,7 @@ export const buildComplexityRouterConfig = ({
|
|||
planModeMinTier,
|
||||
tierLabels,
|
||||
classifierType,
|
||||
heuristicV2SuccessThreshold,
|
||||
capabilityClassifierConfig,
|
||||
llmV2Config,
|
||||
classifierLlmConfig,
|
||||
|
|
@ -640,6 +650,9 @@ export const buildComplexityRouterConfig = ({
|
|||
...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }),
|
||||
...(cleanedTierLabels && { tier_labels: cleanedTierLabels }),
|
||||
classifier_type: classifierType,
|
||||
...(heuristicV2SuccessThreshold !== undefined && {
|
||||
heuristic_v2_success_threshold: heuristicV2SuccessThreshold,
|
||||
}),
|
||||
...classifierWireFields(effectiveType, classifierInputs),
|
||||
...(effectiveType === "capability" &&
|
||||
capabilityClassifierConfig && { capability_classifier_config: capabilityClassifierConfig }),
|
||||
|
|
|
|||
|
|
@ -46,6 +46,34 @@ const hydratedState: KeywordMatchingState = {
|
|||
};
|
||||
|
||||
describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
|
||||
it.each([0, 0.92, 1])("hydrates and saves a success threshold of %s without changing the artifact", (threshold) => {
|
||||
const stored = {
|
||||
...STORED,
|
||||
classifier_type: "heuristic_v2" as const,
|
||||
heuristic_v2_success_threshold: threshold,
|
||||
heuristic_v2_artifact: { routing_threshold: 0.82, custom_metadata: "retained" },
|
||||
};
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
expect(hydrated.heuristic_v2_success_threshold).toBe(threshold);
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
|
||||
expect(saved.heuristic_v2_success_threshold).toBe(threshold);
|
||||
expect(saved.heuristic_v2_artifact).toEqual(stored.heuristic_v2_artifact);
|
||||
|
||||
const cleared = buildUpdatedComplexityRouterConfig(stored, {
|
||||
...hydrated,
|
||||
heuristic_v2_success_threshold: undefined,
|
||||
});
|
||||
expect(cleared).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
expect(cleared.heuristic_v2_artifact).toEqual(stored.heuristic_v2_artifact);
|
||||
});
|
||||
|
||||
it.each([undefined, null])("keeps an inherited success threshold %s omitted after saving", (threshold) => {
|
||||
const stored = { ...STORED, heuristic_v2_success_threshold: threshold };
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
expect(hydrated.heuristic_v2_success_threshold).toBeUndefined();
|
||||
expect(buildUpdatedComplexityRouterConfig(stored, hydrated)).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
});
|
||||
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"handles enabled stored overrides when editing %s with or without keyword form state",
|
||||
(classifier_type) => {
|
||||
|
|
@ -669,6 +697,7 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
plan_mode_min_tier: "COMPLEX",
|
||||
tier_labels: { SIMPLE: "Cheap" },
|
||||
classifier_type: "heuristic_first",
|
||||
heuristic_v2_success_threshold: 0.89,
|
||||
heuristic_first_max_tier: "SIMPLE",
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
classifier_context_window_size: 5,
|
||||
|
|
|
|||
|
|
@ -132,6 +132,74 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
expect(await screen.findByText(/Keyword\/Semantic Matching/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each(["0", ""])("hydrates the saved threshold and saves an edit to '%s'", async (raw) => {
|
||||
const user = userEvent.setup();
|
||||
renderModal({
|
||||
modelData: {
|
||||
...MODEL_DATA,
|
||||
litellm_params: {
|
||||
...MODEL_DATA.litellm_params,
|
||||
complexity_router_config: {
|
||||
...STORED_CONFIG,
|
||||
classifier_type: "heuristic_v2",
|
||||
heuristic_v2_success_threshold: 0.91,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
const threshold = screen.getByRole("textbox", { name: "Success threshold" });
|
||||
expect(threshold).toHaveValue("0.91");
|
||||
fireEvent.change(threshold, { target: { value: raw } });
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
if (raw === "") expect(savedConfig()).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
else expect(savedConfig().heuristic_v2_success_threshold).toBe(0);
|
||||
});
|
||||
|
||||
it("blocks an invalid threshold edit and retains a corrected value when switching classifiers", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderModal({
|
||||
modelData: {
|
||||
...MODEL_DATA,
|
||||
litellm_params: {
|
||||
...MODEL_DATA.litellm_params,
|
||||
complexity_router_config: {
|
||||
...STORED_CONFIG,
|
||||
classifier_type: "heuristic_v2",
|
||||
heuristic_v2_success_threshold: 0.91,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "-0.1" } });
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
expect(modelPatchUpdateCall).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.88" } });
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
expect(screen.queryByRole("textbox", { name: "Success threshold" })).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
expect(savedConfig()).toMatchObject({ classifier_type: "heuristic", heuristic_v2_success_threshold: 0.88 });
|
||||
});
|
||||
|
||||
it("clears an invalid inactive threshold before saving the router", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } });
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Clear Heuristic v2 threshold" }));
|
||||
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
expect(savedConfig()).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
});
|
||||
|
||||
// These keys are rewritten from form state on save, so if the modal renders the controls
|
||||
// without hydrating them, an untouched save silently wipes the stored configuration. This
|
||||
// drives the real component; a test of the payload builder alone cannot see that bug.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import {
|
|||
type BuildComplexityRouterConfigParams,
|
||||
buildComplexityRouterConfig,
|
||||
getClassifierModelError,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getClassifierReasoningEffortError,
|
||||
getKeywordTierRulesError,
|
||||
getMissingTiersError,
|
||||
|
|
@ -127,6 +128,10 @@ export const hydrateComplexityRouterConfig = (
|
|||
plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set),
|
||||
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
|
||||
classifier_type: parsedConfig.classifier_type || "heuristic",
|
||||
heuristic_v2_success_threshold:
|
||||
typeof parsedConfig.heuristic_v2_success_threshold === "number"
|
||||
? parsedConfig.heuristic_v2_success_threshold
|
||||
: undefined,
|
||||
capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data,
|
||||
llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data,
|
||||
classifier_llm_config: parsedConfig.classifier_llm_config,
|
||||
|
|
@ -227,6 +232,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"classification_examples",
|
||||
"heuristic_first_max_tier",
|
||||
"hybrid_boundary_margin",
|
||||
"heuristic_v2_success_threshold",
|
||||
"classification_mode",
|
||||
"session_affinity",
|
||||
"session_affinity_ttl_seconds",
|
||||
|
|
@ -329,6 +335,7 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
classificationMode: value.classification_mode,
|
||||
tierLabels: value.tier_labels,
|
||||
classifierType: value.classifier_type,
|
||||
heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold,
|
||||
capabilityClassifierConfig: value.capability_classifier_config,
|
||||
llmV2Config: value.llm_v2_config,
|
||||
classifierLlmConfig: value.classifier_llm_config,
|
||||
|
|
@ -427,6 +434,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ??
|
||||
getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
|
||||
getClassifierModelError(complexityRouterConfig) ??
|
||||
getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
|
||||
getForecastConfigError(complexityRouterConfig) ??
|
||||
(heuristicScoringRole(complexityRouterConfig) === "decides"
|
||||
? customDimensionsError(complexityRouterConfig.custom_dimensions)
|
||||
|
|
@ -559,6 +567,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
}
|
||||
const classifierError =
|
||||
getClassifierModelError(complexityRouterConfig) ??
|
||||
getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
|
||||
getForecastConfigError(complexityRouterConfig) ??
|
||||
(heuristicScoringRole(complexityRouterConfig) === "decides"
|
||||
? customDimensionsError(complexityRouterConfig.custom_dimensions)
|
||||
|
|
|
|||
|
|
@ -680,6 +680,17 @@ describe("autorouter_presets", () => {
|
|||
});
|
||||
|
||||
describe("buildPresetPrefill", () => {
|
||||
it.each([undefined, 0, 0.95])("carries a preset's success threshold %s into the form", (threshold) => {
|
||||
const preset = getPresetByKey("anthropic_family")!;
|
||||
const config = {
|
||||
...preset.complexity_router_config,
|
||||
classifier_type: "heuristic_v2" as const,
|
||||
heuristic_v2_success_threshold: threshold,
|
||||
};
|
||||
const prefill = buildPresetPrefill(config, groupsOnly(getRequiredModelsInPreset(preset)));
|
||||
expect(prefill.complexityRouterConfig.heuristic_v2_success_threshold).toBe(threshold);
|
||||
});
|
||||
|
||||
it("prefills a real bundled preset's tiers into the config", () => {
|
||||
const preset = getPresetByKey("anthropic_family")!;
|
||||
const prefill = buildPresetPrefill(
|
||||
|
|
|
|||
|
|
@ -284,6 +284,7 @@ export const buildPresetPrefill = (
|
|||
tier_model_params: resolveParamKeys(hydrateTierModelParams(config.tiers, config.tier_model_configs)),
|
||||
tier_labels: hydrateTierLabels(config.tier_labels),
|
||||
classifier_type: config.classifier_type,
|
||||
heuristic_v2_success_threshold: config.heuristic_v2_success_threshold,
|
||||
classifier_llm_config: config.classifier_llm_config && {
|
||||
...config.classifier_llm_config,
|
||||
model: resolve(config.classifier_llm_config.model),
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -36684,6 +36684,11 @@ export interface components {
|
|||
* @default ultrafeedback
|
||||
*/
|
||||
heuristic_v2_artifact: components["schemas"]["TrainedTierArtifact"] | "ultrafeedback";
|
||||
/**
|
||||
* Heuristic V2 Success Threshold
|
||||
* @description Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. The first tier meeting this threshold is selected, or REASONING if none meets it. When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). Other classifier types ignore this setting
|
||||
*/
|
||||
heuristic_v2_success_threshold?: number | null;
|
||||
/**
|
||||
* Housekeeping Patterns
|
||||
* @description Additional case-sensitive literal sentinels that mark a request as client housekeeping, on top of the built-in conversation-title ones. For clients whose wording the built-ins don't cover, or after a client release changes its strings.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue