feat(auto-router): add per-model Fast mode toggle

This commit is contained in:
Tin Chi Lo 2026-09-15 12:29:19 -07:00
parent d3929287fe
commit 864f4a7a0e
17 changed files with 400 additions and 56 deletions

View file

@ -539,6 +539,13 @@ class AnthropicModelInfo(BaseLLMModelInfo):
value: Final = litellm.model_cost.get(model, {}).get(key) value: Final = litellm.model_cost.get(model, {}).get(key)
return value if isinstance(value, bool) else None return value if isinstance(value, bool) else None
@staticmethod
def supports_fast_mode(model: str, custom_llm_provider: str) -> bool:
return (
custom_llm_provider == "anthropic"
and AnthropicModelInfo._get_exact_model_capability(model, "supports_fast_mode") is True
)
@staticmethod @staticmethod
def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> bool | None: def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> bool | None:
"""Resolve boolean capability ``key`` for ``model`` under the caller's provider. """Resolve boolean capability ``key`` for ``model`` under the caller's provider.

View file

@ -14062,6 +14062,7 @@
}, },
"supports_output_config": true, "supports_output_config": true,
"supports_speed": true, "supports_speed": true,
"supports_fast_mode": true,
"prompt_cache_min_tokens": 512, "prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/about-claude/pricing" "source": "https://platform.claude.com/docs/en/about-claude/pricing"
}, },
@ -14103,6 +14104,7 @@
}, },
"supports_output_config": true, "supports_output_config": true,
"supports_speed": true, "supports_speed": true,
"supports_fast_mode": true,
"prompt_cache_min_tokens": 1024, "prompt_cache_min_tokens": 1024,
"source": "https://platform.claude.com/docs/en/about-claude/pricing" "source": "https://platform.claude.com/docs/en/about-claude/pricing"
}, },

View file

@ -99,6 +99,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
mask_sensitive_structure, mask_sensitive_structure,
) )
from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
from litellm.llms.base_llm.vector_store.transformation import ( from litellm.llms.base_llm.vector_store.transformation import (
RouterVectorStoreEmbeddingExecutor, RouterVectorStoreEmbeddingExecutor,
@ -10794,6 +10795,7 @@ class Router:
"model_group": user_facing_model_group_name, "model_group": user_facing_model_group_name,
"providers": [llm_provider], "providers": [llm_provider],
**model_info, **model_info,
"supports_fast_mode": True,
"supported_reasoning_efforts": None, "supported_reasoning_efforts": None,
} }
) )
@ -10872,6 +10874,9 @@ class Router:
if model_info.get("rpm", None) is not None and _deployment_rpm is None: if model_info.get("rpm", None) is not None and _deployment_rpm is None:
_deployment_rpm = model_info.get("rpm") _deployment_rpm = model_info.get("rpm")
model_group_info.supports_fast_mode = model_group_info.supports_fast_mode and (
AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider)
)
deployment_reasoning_efforts = ( deployment_reasoning_efforts = (
resolve_supported_reasoning_efforts( # rebind-ok: recalculated per deployment resolve_supported_reasoning_efforts( # rebind-ok: recalculated per deployment
model_info, deployment_is_mapped=deployment_is_mapped model_info, deployment_is_mapped=deployment_is_mapped

View file

@ -722,6 +722,7 @@ class ModelGroupInfo(BaseModel):
supports_url_context: bool = Field(default=False) supports_url_context: bool = Field(default=False)
supports_reasoning: bool = Field(default=False) supports_reasoning: bool = Field(default=False)
supports_function_calling: bool = Field(default=False) supports_function_calling: bool = Field(default=False)
supports_fast_mode: bool = Field(default=False)
supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None) supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None)
supported_openai_params: list[str] | None = Field(default=[]) supported_openai_params: list[str] | None = Field(default=[])
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None

View file

@ -14062,6 +14062,7 @@
}, },
"supports_output_config": true, "supports_output_config": true,
"supports_speed": true, "supports_speed": true,
"supports_fast_mode": true,
"prompt_cache_min_tokens": 512, "prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/about-claude/pricing" "source": "https://platform.claude.com/docs/en/about-claude/pricing"
}, },
@ -14103,6 +14104,7 @@
}, },
"supports_output_config": true, "supports_output_config": true,
"supports_speed": true, "supports_speed": true,
"supports_fast_mode": true,
"prompt_cache_min_tokens": 1024, "prompt_cache_min_tokens": 1024,
"source": "https://platform.claude.com/docs/en/about-claude/pricing" "source": "https://platform.claude.com/docs/en/about-claude/pricing"
}, },

View file

@ -716,6 +716,9 @@
"supports_embedding_image_input": { "supports_embedding_image_input": {
"type": "boolean" "type": "boolean"
}, },
"supports_fast_mode": {
"type": "boolean"
},
"supports_forced_tool_use": { "supports_forced_tool_use": {
"type": "boolean" "type": "boolean"
}, },

View file

@ -12056,6 +12056,83 @@ def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_o
@pytest.mark.parametrize(
"model,provider,expected",
[
("anthropic/claude-opus-5", None, True),
("claude-opus-4-8", None, True),
("anthropic/claude-opus-4-7", None, False),
("anthropic/claude-opus-4-6", None, False),
("anthropic/claude-sonnet-5", None, False),
("anthropic/off-map-opus", None, False),
("vertex_ai/claude-opus-5", None, False),
("bedrock/claude-opus-5", None, False),
("claude-opus-5", "vertex_ai", False),
("claude-opus-5", "bedrock", False),
],
)
@pytest.mark.parametrize("operator_flag", [True, False])
def test_model_group_info_fast_mode_uses_exact_provider_catalog(
local_model_cost_map: None, model: str, provider: str | None, expected: bool, operator_flag: bool
) -> None:
router: Final = Router(model_list=[{
"model_name": "fast-group",
"litellm_params": {"model": model, "custom_llm_provider": provider, "api_key": "fake-key"},
"model_info": {"supports_fast_mode": operator_flag},
}])
result: Final = router.get_model_group_info("fast-group")
assert result is not None
assert result.supports_fast_mode is expected
@pytest.mark.parametrize("flag", [None, False, "true", 1])
def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean(
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, flag: object
) -> None:
entry: Final = {key: value for key, value in litellm.model_cost["claude-opus-5"].items()
if key != "supports_fast_mode"}
if flag is not None:
entry["supports_fast_mode"] = flag
monkeypatch.setitem(litellm.model_cost, "claude-opus-5", entry)
router: Final = Router(model_list=[{
"model_name": "fast-group",
"litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "fake-key"},
"model_info": {"supports_fast_mode": True},
}])
result: Final = router.get_model_group_info("fast-group")
assert result is not None
assert result.supports_fast_mode is False
@pytest.mark.parametrize("other_model,expected", [
("anthropic/claude-opus-4-8", True),
("anthropic/claude-opus-4-7", False),
("anthropic/off-map-opus", False),
("vertex_ai/claude-opus-5", False),
("bedrock/claude-opus-5", False),
])
@pytest.mark.parametrize("reverse", [True, False])
def test_model_group_info_fast_mode_requires_every_deployment(
local_model_cost_map: None, other_model: str, expected: bool, reverse: bool
) -> None:
models: Final = (other_model, "anthropic/claude-opus-5") if reverse else (
"anthropic/claude-opus-5", other_model
)
router: Final = Router(model_list=[{
"model_name": "fast-group",
"litellm_params": {"model": model, "api_key": "fake-key"},
} for model in models])
result: Final = router.get_model_group_info("fast-group")
assert result is not None
assert result.supports_fast_mode is expected
def test_model_group_info_surfaces_supports_parallel_function_calling(local_model_cost_map): def test_model_group_info_surfaces_supports_parallel_function_calling(local_model_cost_map):
"""``/model_group/info`` folds each deployment's registry flags into the group; a deployment whose """``/model_group/info`` folds each deployment's registry flags into the group; a deployment whose
registry entry declares parallel function calling must flip the group to True instead of False.""" registry entry declares parallel function calling must flip the group to True instead of False."""

View file

@ -1165,6 +1165,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_sampling_params": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"},
"supports_output_config": {"type": "boolean"}, "supports_output_config": {"type": "boolean"},
"supports_speed": {"type": "boolean"}, "supports_speed": {"type": "boolean"},
"supports_fast_mode": {"type": "boolean"},
"supported_audio_formats": { "supported_audio_formats": {
"type": "array", "type": "array",
"items": { "items": {

View file

@ -42,9 +42,10 @@ import { Restricted, restrictedBy } from "./TierRestrictions";
import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
import { import {
ReasoningEffort, ReasoningEffort,
TierModelParamChange,
TierModelParamsByTier, TierModelParamsByTier,
classifierEffortOptionsForModels, classifierEffortOptionsForModels,
setTierModelReasoningEffort, setTierModelParam,
tierEffortOptionsForModels, tierEffortOptionsForModels,
tierRowLabel, tierRowLabel,
} from "./complexity_router_tiers"; } from "./complexity_router_tiers";
@ -613,6 +614,9 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
const exitToBuiltInTiers = () => dispatch({ kind: "restore" }); const exitToBuiltInTiers = () => dispatch({ kind: "restore" });
const tierEffortOptionsByModel = tierEffortOptionsForModels(modelInfo); const tierEffortOptionsByModel = tierEffortOptionsForModels(modelInfo);
const fastModeByModel = Object.fromEntries(
modelInfo.map((model) => [model.model_group, model.supports_fast_mode === true]),
);
const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo); const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo);
// Embedding models can't serve a chat-completion role, so they're excluded here. // Embedding models can't serve a chat-completion role, so they're excluded here.
@ -623,12 +627,11 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
label: model.model_group, label: model.model_group,
})); }));
const handleTierModelEffortChange = (tier: string, model: string, effort: ReasoningEffort | undefined) => { const handleTierModelParamChange = (tier: string, model: string, change: TierModelParamChange) =>
onChange({ onChange({
...value, ...value,
tier_model_params: setTierModelReasoningEffort(value.tier_model_params, tier, model, effort), tier_model_params: setTierModelParam(value.tier_model_params, tier, model, change),
}); });
};
// Clearing the select drops the key entirely rather than storing "", so an emptied pin reads as // Clearing the select drops the key entirely rather than storing "", so an emptied pin reads as
// "track the tiers" everywhere downstream instead of as a blank model name. // "track the tiers" everywhere downstream instead of as a blank model name.
@ -726,7 +729,13 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
models={row.models} models={row.models}
effortOptionsByModel={tierEffortOptionsByModel} effortOptionsByModel={tierEffortOptionsByModel}
paramsByModel={row.params} paramsByModel={row.params}
onEffortChange={(model, effort) => handleTierModelEffortChange(row.id, model, effort)} fastModeByModel={fastModeByModel}
onEffortChange={(model, effort) =>
handleTierModelParamChange(row.id, model, ["reasoning_effort", effort])
}
onFastModeChange={(model, enabled) =>
handleTierModelParamChange(row.id, model, ["speed", enabled ? "fast" : undefined])
}
/> />
{row.models.length > 1 && ( {row.models.length > 1 && (
<span className="text-xs text-muted-foreground"> <span className="text-xs text-muted-foreground">

View file

@ -0,0 +1,143 @@
import userEvent from "@testing-library/user-event";
import React from "react";
import { describe, expect, it, vi } from "vitest";
import { renderWithProviders, screen } from "../../../tests/test-utils";
import {
buildUpdatedComplexityRouterConfig,
hydrateComplexityRouterConfig,
} from "../edit_auto_router/edit_auto_router_modal";
import type { ModelGroup } from "../llm_calls/fetch_models";
import ComplexityRouterConfig, { type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
const modelInfo: ModelGroup[] = [
{ model_group: "primary", supported_reasoning_efforts: ["low", "high"], supports_fast_mode: true },
{ model_group: "secondary", supports_fast_mode: true },
{ model_group: "blocked", supported_reasoning_efforts: ["low"], supports_fast_mode: false },
{ model_group: "missing", supported_reasoning_efforts: ["low"] },
];
it.each([false, true])("edits and round-trips independent model settings with custom tiers=%s", async (custom) => {
const user = userEvent.setup();
const tier = custom ? "custom-a" : "COMPLEX";
const otherTier = custom ? "custom-b" : "REASONING";
const label = custom ? "Interactive" : "Complex";
const models = ["primary", "secondary", "blocked", "missing"];
const initial: ComplexityRouterConfigValue = {
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: models, REASONING: ["primary"] },
classifier_type: "heuristic",
...(custom && {
custom_tier_set: {
tiers: [
{ id: tier, name: label, definition: "Interactive requests", models },
{ id: otherTier, name: "Deliberate", definition: "Careful requests", models: ["primary"] },
],
fallback_tier_id: tier,
},
}),
tier_model_params: {
[tier]: {
primary: { reasoning_effort: "high", max_tokens: 1024 },
secondary: { speed: "fast" },
blocked: { speed: "fast" },
},
[otherTier]: { primary: { speed: "fast", reasoning_effort: "low" } },
},
};
const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>();
const editor = (value: ComplexityRouterConfigValue) => (
<ComplexityRouterConfig modelInfo={modelInfo} value={value} onChange={onChange} />
);
const view = renderWithProviders(editor(initial));
const fast = () => screen.getByRole("switch", { name: `Fast mode for primary in the ${label} tier` });
expect(screen.getAllByRole("switch", { name: /^Fast mode for/ })).toHaveLength(3);
expect(screen.queryByRole("switch", { name: /^Fast mode for (blocked|missing)/ })).not.toBeInTheDocument();
expect(screen.queryByRole("combobox", { name: /^Reasoning effort for secondary/ })).not.toBeInTheDocument();
expect(screen.getByRole("switch", { name: `Fast mode for secondary in the ${label} tier` })).toBeChecked();
expect(fast()).not.toBeChecked();
expect(onChange).not.toHaveBeenCalled();
await user.click(fast());
const enabled = onChange.mock.lastCall![0];
expect(enabled.tier_model_params).toEqual({
...initial.tier_model_params,
[tier]: {
...initial.tier_model_params![tier],
primary: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" },
},
});
const saved = buildUpdatedComplexityRouterConfig({}, enabled);
expect(saved.tier_model_configs).toEqual({
[custom ? label : tier]: [
{ model_name: "primary", litellm_params: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" } },
{ model_name: "secondary", litellm_params: { speed: "fast" } },
{ model_name: "blocked", litellm_params: { speed: "fast" } },
],
[custom ? "Deliberate" : otherTier]: [
{ model_name: "primary", litellm_params: { speed: "fast", reasoning_effort: "low" } },
],
});
const reopened = hydrateComplexityRouterConfig(saved, undefined);
const reopenedTier = custom ? reopened.custom_tier_set!.tiers[0].id : tier;
view.rerender(editor(reopened));
expect(fast()).toBeChecked();
await user.click(screen.getByRole("combobox", { name: `Reasoning effort for primary in the ${label} tier` }));
await user.click(await screen.findByRole("option", { name: "low" }));
const effortChanged = onChange.mock.lastCall![0];
expect(effortChanged.tier_model_params?.[reopenedTier].primary).toEqual({
reasoning_effort: "low",
max_tokens: 1024,
speed: "fast",
});
view.rerender(editor(effortChanged));
await user.click(fast());
const disabled = onChange.mock.lastCall![0];
expect(disabled.tier_model_params).toEqual({
...effortChanged.tier_model_params,
[reopenedTier]: {
...effortChanged.tier_model_params![reopenedTier],
primary: { reasoning_effort: "low", max_tokens: 1024 },
},
});
view.rerender(editor(disabled));
expect(fast()).not.toBeChecked();
const picker = () => screen.getByRole("combobox", { name: `Select model(s) for ${label.toLowerCase()} queries` });
await user.click(picker());
await user.click(await screen.findByRole("option", { name: "primary" }));
await user.keyboard("{Escape}");
const deselected = onChange.mock.lastCall![0];
expect(deselected.tier_model_params?.[reopenedTier]).toEqual({
secondary: { speed: "fast" },
blocked: { speed: "fast" },
});
view.rerender(editor(deselected));
expect(screen.queryByRole("switch", { name: `Fast mode for primary in the ${label} tier` })).not.toBeInTheDocument();
await user.click(picker());
await user.click(await screen.findByRole("option", { name: "primary" }));
await user.keyboard("{Escape}");
const reselected = onChange.mock.lastCall![0];
view.rerender(editor(reselected));
expect(fast()).not.toBeChecked();
expect(screen.getByRole("combobox", { name: `Reasoning effort for primary in the ${label} tier` })).toHaveTextContent(
"Default",
);
});
describe("Fast mode metadata", () => {
it("offers nothing before model capabilities load and leaves stored speed untouched", () => {
const value: ComplexityRouterConfigValue = {
tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic",
tier_model_params: { SIMPLE: { primary: { speed: "fast" } } },
};
const onChange = vi.fn();
renderWithProviders(<ComplexityRouterConfig modelInfo={[]} value={value} onChange={onChange} />);
expect(screen.queryByRole("switch", { name: /^Fast mode for/ })).not.toBeInTheDocument();
expect(onChange).not.toHaveBeenCalled();
expect(buildUpdatedComplexityRouterConfig({}, value).tier_model_configs).toEqual({
SIMPLE: [{ model_name: "primary", litellm_params: { speed: "fast" } }],
});
});
});

View file

@ -1,5 +1,6 @@
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { SimpleTooltip } from "@/components/ui/tooltip"; import { SimpleTooltip } from "@/components/ui/tooltip";
import { Switch } from "@/components/ui/switch";
import { Info } from "lucide-react"; import { Info } from "lucide-react";
import React from "react"; import React from "react";
import { ReasoningEffort, TierModelParams } from "./complexity_router_tiers"; import { ReasoningEffort, TierModelParams } from "./complexity_router_tiers";
@ -18,6 +19,8 @@ interface TierModelEffortRowsProps {
effortOptionsByModel: Record<string, string[]>; effortOptionsByModel: Record<string, string[]>;
paramsByModel: Record<string, TierModelParams> | undefined; paramsByModel: Record<string, TierModelParams> | undefined;
onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void; onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void;
fastModeByModel?: Record<string, boolean>;
onFastModeChange: (model: string, enabled: boolean) => void;
} }
export interface TierEffortRow { export interface TierEffortRow {
@ -29,13 +32,16 @@ export interface TierEffortRow {
/** /**
* A stored effort outside the model's supported set (hand-authored, or capabilities changed since * A stored effort outside the model's supported set (hand-authored, or capabilities changed since
* it was saved) is listed anyway, so the row renders with its value selected and can be cleared. * it was saved) is listed anyway, so the row renders with its value selected and can be cleared.
* Only a model with no supported level and nothing stored drops out.
*/ */
export const tierEffortRows = ({ export const tierEffortRows = ({
models, models,
effortOptionsByModel, effortOptionsByModel,
paramsByModel, paramsByModel,
}: Pick<TierModelEffortRowsProps, "models" | "effortOptionsByModel" | "paramsByModel">): TierEffortRow[] => fastModeByModel,
}: Pick<
TierModelEffortRowsProps,
"models" | "effortOptionsByModel" | "paramsByModel" | "fastModeByModel"
>): TierEffortRow[] =>
models models
.map((model) => { .map((model) => {
const effort = storedEffort(paramsByModel?.[model]); const effort = storedEffort(paramsByModel?.[model]);
@ -43,56 +49,74 @@ export const tierEffortRows = ({
const listed = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported; const listed = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported;
return { model, effort, options: Array.from(new Set(listed)) }; return { model, effort, options: Array.from(new Set(listed)) };
}) })
.filter(({ options }) => options.length > 0); .filter(({ model, options }) => options.length > 0 || fastModeByModel?.[model] === true);
const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({ const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = (props) => {
tierLabel, const { tierLabel, paramsByModel, onEffortChange, fastModeByModel, onFastModeChange } = props;
models, const rows = tierEffortRows(props);
effortOptionsByModel,
paramsByModel,
onEffortChange,
}) => {
const rows = tierEffortRows({ models, effortOptionsByModel, paramsByModel });
if (rows.length === 0) return null; if (rows.length === 0) return null;
return ( return (
<div className="mt-2 space-y-1"> <div className="mt-2 space-y-1">
<div className="flex items-center gap-1"> {rows.some(({ options }) => options.length > 0) && (
<span className="text-xs font-medium text-muted-foreground">Reasoning effort</span> <div className="flex items-center gap-1">
<SimpleTooltip <span className="text-xs font-medium text-muted-foreground">Reasoning effort</span>
content={`Sent as reasoning_effort on requests this tier routes to the model, overriding the caller's value. Default leaves the request untouched.`} <SimpleTooltip
> content={`Sent as reasoning_effort on requests this tier routes to the model, overriding the caller's value. Default leaves the request untouched.`}
<Info className="size-3 text-muted-foreground/70" />
</SimpleTooltip>
</div>
{rows.map(({ model, effort, options }) => (
<div key={model} className="flex items-center justify-between gap-2">
<span className="truncate text-xs">{model}</span>
<Select
items={[
{ value: PROVIDER_DEFAULT, label: "Default" },
...options.map((option) => ({ value: option, label: option })),
]}
value={effort ?? PROVIDER_DEFAULT}
onValueChange={(selected: string | null) =>
selected !== null && onEffortChange(model, selected === PROVIDER_DEFAULT ? undefined : selected)
}
> >
<SelectTrigger <Info className="size-3 text-muted-foreground/70" />
size="sm" </SimpleTooltip>
className="w-36" </div>
aria-label={`Reasoning effort for ${model} in the ${tierLabel} tier`} )}
> {rows.map(({ model, effort, options }) => (
<SelectValue /> <div key={model} className="flex flex-wrap items-center justify-between gap-2">
</SelectTrigger> <span className="min-w-0 flex-1 basis-32 truncate text-xs" title={model}>
<SelectContent> {model}
<SelectItem value={PROVIDER_DEFAULT}>Default</SelectItem> </span>
{options.map((option) => ( <div className="flex flex-wrap items-center gap-3">
<SelectItem key={option} value={option}> {options.length > 0 && (
{option} <Select
</SelectItem> items={[
))} { value: PROVIDER_DEFAULT, label: "Default" },
</SelectContent> ...options.map((option) => ({ value: option, label: option })),
</Select> ]}
value={effort ?? PROVIDER_DEFAULT}
onValueChange={(selected: string | null) =>
selected !== null && onEffortChange(model, selected === PROVIDER_DEFAULT ? undefined : selected)
}
>
<SelectTrigger
size="sm"
className="w-36"
aria-label={`Reasoning effort for ${model} in the ${tierLabel} tier`}
>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value={PROVIDER_DEFAULT}>Default</SelectItem>
{options.map((option) => (
<SelectItem key={option} value={option}>
{option}
</SelectItem>
))}
</SelectContent>
</Select>
)}
{fastModeByModel?.[model] === true && (
<SimpleTooltip content="Fast mode has higher pricing and requires an eligible provider account. Off removes this tier's speed override and inherits the request or provider default">
<label
className="flex items-center gap-2 text-xs"
aria-label={`Fast mode for ${model} in the ${tierLabel} tier`}
>
<Switch
size="sm"
checked={paramsByModel?.[model]?.speed === "fast"}
onCheckedChange={(enabled) => onFastModeChange(model, enabled)}
/>
Fast mode
</label>
</SimpleTooltip>
)}
</div>
</div> </div>
))} ))}
</div> </div>

View file

@ -48,6 +48,19 @@ const baseParams: BuildComplexityRouterConfigParams = {
}; };
describe("buildComplexityRouterConfig", () => { describe("buildComplexityRouterConfig", () => {
it("carries Fast and reasoning overrides independently into a new router payload", () => {
const params = { speed: "fast", reasoning_effort: "high", max_tokens: 1024 };
const config = buildComplexityRouterConfig({
...baseParams,
tiers: { ...tiers, COMPLEX: ["primary"], REASONING: ["secondary"] },
tierModelParams: { COMPLEX: { primary: params }, REASONING: { secondary: { speed: "fast" } } },
});
expect(config.tier_model_configs).toEqual({
COMPLEX: [{ model_name: "primary", litellm_params: params }],
REASONING: [{ model_name: "secondary", litellm_params: { speed: "fast" } }],
});
});
it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => { it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => {
const config = buildComplexityRouterConfig(baseParams); const config = buildComplexityRouterConfig(baseParams);
const expected = { const expected = {

View file

@ -7,6 +7,7 @@ import {
serializeTierModelConfigs, serializeTierModelConfigs,
tierRowLabel, tierRowLabel,
setTierModelReasoningEffort, setTierModelReasoningEffort,
setTierModelParam,
} from "./complexity_router_tiers"; } from "./complexity_router_tiers";
import { resolveComplexityDefaultModel } from "./tier_rows"; import { resolveComplexityDefaultModel } from "./tier_rows";
@ -218,6 +219,28 @@ describe("setTierModelReasoningEffort", () => {
}); });
}); });
describe("setTierModelParam", () => {
it.each(["reasoning_effort", "speed"] as const)("clears only %s and preserves the input", (key) => {
const params = { reasoning_effort: "high", speed: "fast", max_tokens: 512 };
const current = { COMPLEX: { primary: params, secondary: { speed: "fast" } }, REASONING: { primary: params } };
const cleared = setTierModelParam(current, "COMPLEX", "primary", [key, undefined]);
expect(cleared).toEqual({
...current,
COMPLEX: {
...current.COMPLEX,
primary: key === "speed" ? { reasoning_effort: "high", max_tokens: 512 } : { speed: "fast", max_tokens: 512 },
},
});
expect(current.COMPLEX.primary).toEqual({ reasoning_effort: "high", speed: "fast", max_tokens: 512 });
});
it("removes empty records when the only override is Fast", () => {
const enabled = setTierModelParam(undefined, "COMPLEX", "primary", ["speed", "fast"]);
expect(enabled).toEqual({ COMPLEX: { primary: { speed: "fast" } } });
expect(setTierModelParam(enabled, "COMPLEX", "primary", ["speed", undefined])).toBeUndefined();
});
});
describe("pruneTierModelParams", () => { describe("pruneTierModelParams", () => {
it("drops params for models deselected from the tier", () => { it("drops params for models deselected from the tier", () => {
expect( expect(

View file

@ -114,14 +114,16 @@ export const serializeTierModelConfigs = (
return serialized.length > 0 ? Object.fromEntries(serialized) : undefined; return serialized.length > 0 ? Object.fromEntries(serialized) : undefined;
}; };
export const setTierModelReasoningEffort = ( export type TierModelParamChange = ["reasoning_effort", ReasoningEffort | undefined] | ["speed", "fast" | undefined];
export const setTierModelParam = (
current: TierModelParamsByTier | undefined, current: TierModelParamsByTier | undefined,
tier: string, tier: string,
model: string, model: string,
effort: ReasoningEffort | undefined, [key, value]: TierModelParamChange,
): TierModelParamsByTier | undefined => { ): TierModelParamsByTier | undefined => {
const { reasoning_effort: _dropped, ...rest } = current?.[tier]?.[model] ?? {}; const { [key]: _dropped, ...rest } = current?.[tier]?.[model] ?? {};
const params = effort === undefined ? rest : { ...rest, reasoning_effort: effort }; const params = value === undefined ? rest : { ...rest, [key]: value };
const byModel = Object.fromEntries( const byModel = Object.fromEntries(
Object.entries({ ...current?.[tier], [model]: params }).filter(([, value]) => Object.keys(value).length > 0), Object.entries({ ...current?.[tier], [model]: params }).filter(([, value]) => Object.keys(value).length > 0),
); );
@ -131,6 +133,13 @@ export const setTierModelReasoningEffort = (
return Object.keys(next).length > 0 ? next : undefined; return Object.keys(next).length > 0 ? next : undefined;
}; };
export const setTierModelReasoningEffort = (
current: TierModelParamsByTier | undefined,
tier: string,
model: string,
effort: ReasoningEffort | undefined,
): TierModelParamsByTier | undefined => setTierModelParam(current, tier, model, ["reasoning_effort", effort]);
export const pruneTierModelParams = ( export const pruneTierModelParams = (
current: TierModelParamsByTier | undefined, current: TierModelParamsByTier | undefined,
tier: string, tier: string,

View file

@ -52,6 +52,23 @@ describe("fetchAvailableModels", () => {
]); ]);
}); });
it("carries only explicitly supported Fast capabilities, not accepted speed parameters", async () => {
modelHubCallMock.mockResolvedValue({
data: [
{ model_group: "fast", supports_fast_mode: true },
{ model_group: "blocked", supports_fast_mode: false },
{ model_group: "missing", supports_speed: true },
{ model_group: "unknown", supports_fast_mode: null },
],
});
expect(await fetchAvailableModels("token")).toEqual([
{ model_group: "blocked" },
{ model_group: "fast", supports_fast_mode: true },
{ model_group: "missing" },
{ model_group: "unknown" },
]);
});
it("preserves absent, unknown, empty, and explicit effort capability states", async () => { it("preserves absent, unknown, empty, and explicit effort capability states", async () => {
modelHubCallMock.mockResolvedValue({ modelHubCallMock.mockResolvedValue({
data: [ data: [

View file

@ -7,6 +7,7 @@ export interface ModelGroup {
model_group: string; model_group: string;
mode?: string; mode?: string;
supports_reasoning?: boolean; supports_reasoning?: boolean;
supports_fast_mode?: boolean;
supported_reasoning_efforts?: string[] | null; supported_reasoning_efforts?: string[] | null;
} }
@ -16,6 +17,7 @@ interface AvailableModel {
id?: string | null; id?: string | null;
mode?: string | null; mode?: string | null;
supports_reasoning?: boolean | null; supports_reasoning?: boolean | null;
supports_fast_mode?: boolean | null;
supported_reasoning_efforts?: string[] | null; supported_reasoning_efforts?: string[] | null;
} }
@ -25,6 +27,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
model_group: groupName, model_group: groupName,
...(item.mode && { mode: item.mode }), ...(item.mode && { mode: item.mode }),
...(item.supports_reasoning === true && { supports_reasoning: true }), ...(item.supports_reasoning === true && { supports_reasoning: true }),
...(item.supports_fast_mode === true && { supports_fast_mode: true }),
...(item.supported_reasoning_efforts !== undefined && { ...(item.supported_reasoning_efforts !== undefined && {
supported_reasoning_efforts: item.supported_reasoning_efforts, supported_reasoning_efforts: item.supported_reasoning_efforts,
}), }),

View file

@ -32565,6 +32565,11 @@ export interface components {
supported_openai_params: string[] | null; supported_openai_params: string[] | null;
/** Supported Reasoning Efforts */ /** Supported Reasoning Efforts */
supported_reasoning_efforts?: string[] | null; supported_reasoning_efforts?: string[] | null;
/**
* Supports Fast Mode
* @default false
*/
supports_fast_mode: boolean;
/** /**
* Supports Function Calling * Supports Function Calling
* @default false * @default false