From 864f4a7a0e70ad19b4c251e1a2447ea1aa7965bf Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 15 Sep 2026 12:29:19 -0700 Subject: [PATCH] feat(auto-router): add per-model Fast mode toggle --- litellm/llms/anthropic/common_utils.py | 7 + ...odel_prices_and_context_window_backup.json | 2 + litellm/router.py | 5 + litellm/types/router.py | 1 + model_prices_and_context_window.json | 2 + model_prices_and_context_window.schema.json | 3 + tests/test_litellm/test_router.py | 77 ++++++++++ tests/test_litellm/test_utils.py | 1 + .../add_model/ComplexityRouterConfig.tsx | 19 ++- ...plexityRouterFastMode.integration.test.tsx | 143 ++++++++++++++++++ .../add_model/TierModelEffortRows.tsx | 118 +++++++++------ .../build_complexity_router_config.test.ts | 13 ++ .../add_model/complexity_router_tiers.test.ts | 23 +++ .../add_model/complexity_router_tiers.ts | 17 ++- .../llm_calls/fetch_models.test.tsx | 17 +++ .../src/components/llm_calls/fetch_models.tsx | 3 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 17 files changed, 400 insertions(+), 56 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 87c4ec8938e..d35a9372058 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -539,6 +539,13 @@ class AnthropicModelInfo(BaseLLMModelInfo): value: Final = litellm.model_cost.get(model, {}).get(key) return value if isinstance(value, bool) else None + @staticmethod + def supports_fast_mode(model: str, custom_llm_provider: str) -> bool: + return ( + custom_llm_provider == "anthropic" + and AnthropicModelInfo._get_exact_model_capability(model, "supports_fast_mode") is True + ) + @staticmethod def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> bool | None: """Resolve boolean capability ``key`` for ``model`` under the caller's provider. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9f91cf82f41..19c438ef52d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14062,6 +14062,7 @@ }, "supports_output_config": true, "supports_speed": true, + "supports_fast_mode": true, "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, @@ -14103,6 +14104,7 @@ }, "supports_output_config": true, "supports_speed": true, + "supports_fast_mode": true, "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, diff --git a/litellm/router.py b/litellm/router.py index fcdcf91c2cf..34e7a78f17b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -99,6 +99,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_sensitive_structure, ) from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.passthrough.transformation import replace_path_segment from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, @@ -10794,6 +10795,7 @@ class Router: "model_group": user_facing_model_group_name, "providers": [llm_provider], **model_info, + "supports_fast_mode": True, "supported_reasoning_efforts": None, } ) @@ -10872,6 +10874,9 @@ class Router: if model_info.get("rpm", None) is not None and _deployment_rpm is None: _deployment_rpm = model_info.get("rpm") + model_group_info.supports_fast_mode = model_group_info.supports_fast_mode and ( + AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider) + ) deployment_reasoning_efforts = ( resolve_supported_reasoning_efforts( # rebind-ok: recalculated per deployment model_info, deployment_is_mapped=deployment_is_mapped diff --git a/litellm/types/router.py b/litellm/types/router.py index 7732413b593..7c3e4d6943f 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -722,6 +722,7 @@ class ModelGroupInfo(BaseModel): supports_url_context: bool = Field(default=False) supports_reasoning: bool = Field(default=False) supports_function_calling: bool = Field(default=False) + supports_fast_mode: bool = Field(default=False) supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None) supported_openai_params: list[str] | None = Field(default=[]) configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9f91cf82f41..19c438ef52d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14062,6 +14062,7 @@ }, "supports_output_config": true, "supports_speed": true, + "supports_fast_mode": true, "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, @@ -14103,6 +14104,7 @@ }, "supports_output_config": true, "supports_speed": true, + "supports_fast_mode": true, "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index c2490041cf7..130cc6873fa 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -716,6 +716,9 @@ "supports_embedding_image_input": { "type": "boolean" }, + "supports_fast_mode": { + "type": "boolean" + }, "supports_forced_tool_use": { "type": "boolean" }, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9c29b9d829a..ea90c167845 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -12056,6 +12056,83 @@ def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_o +@pytest.mark.parametrize( + "model,provider,expected", + [ + ("anthropic/claude-opus-5", None, True), + ("claude-opus-4-8", None, True), + ("anthropic/claude-opus-4-7", None, False), + ("anthropic/claude-opus-4-6", None, False), + ("anthropic/claude-sonnet-5", None, False), + ("anthropic/off-map-opus", None, False), + ("vertex_ai/claude-opus-5", None, False), + ("bedrock/claude-opus-5", None, False), + ("claude-opus-5", "vertex_ai", False), + ("claude-opus-5", "bedrock", False), + ], +) +@pytest.mark.parametrize("operator_flag", [True, False]) +def test_model_group_info_fast_mode_uses_exact_provider_catalog( + local_model_cost_map: None, model: str, provider: str | None, expected: bool, operator_flag: bool +) -> None: + router: Final = Router(model_list=[{ + "model_name": "fast-group", + "litellm_params": {"model": model, "custom_llm_provider": provider, "api_key": "fake-key"}, + "model_info": {"supports_fast_mode": operator_flag}, + }]) + + result: Final = router.get_model_group_info("fast-group") + + assert result is not None + assert result.supports_fast_mode is expected + + +@pytest.mark.parametrize("flag", [None, False, "true", 1]) +def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, flag: object +) -> None: + entry: Final = {key: value for key, value in litellm.model_cost["claude-opus-5"].items() + if key != "supports_fast_mode"} + if flag is not None: + entry["supports_fast_mode"] = flag + monkeypatch.setitem(litellm.model_cost, "claude-opus-5", entry) + router: Final = Router(model_list=[{ + "model_name": "fast-group", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "fake-key"}, + "model_info": {"supports_fast_mode": True}, + }]) + + result: Final = router.get_model_group_info("fast-group") + + assert result is not None + assert result.supports_fast_mode is False + + +@pytest.mark.parametrize("other_model,expected", [ + ("anthropic/claude-opus-4-8", True), + ("anthropic/claude-opus-4-7", False), + ("anthropic/off-map-opus", False), + ("vertex_ai/claude-opus-5", False), + ("bedrock/claude-opus-5", False), +]) +@pytest.mark.parametrize("reverse", [True, False]) +def test_model_group_info_fast_mode_requires_every_deployment( + local_model_cost_map: None, other_model: str, expected: bool, reverse: bool +) -> None: + models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ( + "anthropic/claude-opus-5", other_model + ) + router: Final = Router(model_list=[{ + "model_name": "fast-group", + "litellm_params": {"model": model, "api_key": "fake-key"}, + } for model in models]) + + result: Final = router.get_model_group_info("fast-group") + + assert result is not None + assert result.supports_fast_mode is expected + + def test_model_group_info_surfaces_supports_parallel_function_calling(local_model_cost_map): """``/model_group/info`` folds each deployment's registry flags into the group; a deployment whose registry entry declares parallel function calling must flip the group to True instead of False.""" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 8bf8489fc52..6e6acd5aabf 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1165,6 +1165,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, "supports_speed": {"type": "boolean"}, + "supports_fast_mode": {"type": "boolean"}, "supported_audio_formats": { "type": "array", "items": { diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index febcde269f7..64b17751e95 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -42,9 +42,10 @@ import { Restricted, restrictedBy } from "./TierRestrictions"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { ReasoningEffort, + TierModelParamChange, TierModelParamsByTier, classifierEffortOptionsForModels, - setTierModelReasoningEffort, + setTierModelParam, tierEffortOptionsForModels, tierRowLabel, } from "./complexity_router_tiers"; @@ -613,6 +614,9 @@ const ComplexityRouterConfig: React.FC = ({ const exitToBuiltInTiers = () => dispatch({ kind: "restore" }); const tierEffortOptionsByModel = tierEffortOptionsForModels(modelInfo); + const fastModeByModel = Object.fromEntries( + modelInfo.map((model) => [model.model_group, model.supports_fast_mode === true]), + ); const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo); // Embedding models can't serve a chat-completion role, so they're excluded here. @@ -623,12 +627,11 @@ const ComplexityRouterConfig: React.FC = ({ label: model.model_group, })); - const handleTierModelEffortChange = (tier: string, model: string, effort: ReasoningEffort | undefined) => { + const handleTierModelParamChange = (tier: string, model: string, change: TierModelParamChange) => onChange({ ...value, - tier_model_params: setTierModelReasoningEffort(value.tier_model_params, tier, model, effort), + tier_model_params: setTierModelParam(value.tier_model_params, tier, model, change), }); - }; // Clearing the select drops the key entirely rather than storing "", so an emptied pin reads as // "track the tiers" everywhere downstream instead of as a blank model name. @@ -726,7 +729,13 @@ const ComplexityRouterConfig: React.FC = ({ models={row.models} effortOptionsByModel={tierEffortOptionsByModel} paramsByModel={row.params} - onEffortChange={(model, effort) => handleTierModelEffortChange(row.id, model, effort)} + fastModeByModel={fastModeByModel} + onEffortChange={(model, effort) => + handleTierModelParamChange(row.id, model, ["reasoning_effort", effort]) + } + onFastModeChange={(model, enabled) => + handleTierModelParamChange(row.id, model, ["speed", enabled ? "fast" : undefined]) + } /> {row.models.length > 1 && ( diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx new file mode 100644 index 00000000000..34d14091bde --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx @@ -0,0 +1,143 @@ +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { + buildUpdatedComplexityRouterConfig, + hydrateComplexityRouterConfig, +} from "../edit_auto_router/edit_auto_router_modal"; +import type { ModelGroup } from "../llm_calls/fetch_models"; +import ComplexityRouterConfig, { type ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const modelInfo: ModelGroup[] = [ + { model_group: "primary", supported_reasoning_efforts: ["low", "high"], supports_fast_mode: true }, + { model_group: "secondary", supports_fast_mode: true }, + { model_group: "blocked", supported_reasoning_efforts: ["low"], supports_fast_mode: false }, + { model_group: "missing", supported_reasoning_efforts: ["low"] }, +]; + +it.each([false, true])("edits and round-trips independent model settings with custom tiers=%s", async (custom) => { + const user = userEvent.setup(); + const tier = custom ? "custom-a" : "COMPLEX"; + const otherTier = custom ? "custom-b" : "REASONING"; + const label = custom ? "Interactive" : "Complex"; + const models = ["primary", "secondary", "blocked", "missing"]; + const initial: ComplexityRouterConfigValue = { + tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: models, REASONING: ["primary"] }, + classifier_type: "heuristic", + ...(custom && { + custom_tier_set: { + tiers: [ + { id: tier, name: label, definition: "Interactive requests", models }, + { id: otherTier, name: "Deliberate", definition: "Careful requests", models: ["primary"] }, + ], + fallback_tier_id: tier, + }, + }), + tier_model_params: { + [tier]: { + primary: { reasoning_effort: "high", max_tokens: 1024 }, + secondary: { speed: "fast" }, + blocked: { speed: "fast" }, + }, + [otherTier]: { primary: { speed: "fast", reasoning_effort: "low" } }, + }, + }; + const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>(); + const editor = (value: ComplexityRouterConfigValue) => ( + + ); + const view = renderWithProviders(editor(initial)); + const fast = () => screen.getByRole("switch", { name: `Fast mode for primary in the ${label} tier` }); + + expect(screen.getAllByRole("switch", { name: /^Fast mode for/ })).toHaveLength(3); + expect(screen.queryByRole("switch", { name: /^Fast mode for (blocked|missing)/ })).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: /^Reasoning effort for secondary/ })).not.toBeInTheDocument(); + expect(screen.getByRole("switch", { name: `Fast mode for secondary in the ${label} tier` })).toBeChecked(); + expect(fast()).not.toBeChecked(); + expect(onChange).not.toHaveBeenCalled(); + + await user.click(fast()); + const enabled = onChange.mock.lastCall![0]; + expect(enabled.tier_model_params).toEqual({ + ...initial.tier_model_params, + [tier]: { + ...initial.tier_model_params![tier], + primary: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" }, + }, + }); + const saved = buildUpdatedComplexityRouterConfig({}, enabled); + expect(saved.tier_model_configs).toEqual({ + [custom ? label : tier]: [ + { model_name: "primary", litellm_params: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" } }, + { model_name: "secondary", litellm_params: { speed: "fast" } }, + { model_name: "blocked", litellm_params: { speed: "fast" } }, + ], + [custom ? "Deliberate" : otherTier]: [ + { model_name: "primary", litellm_params: { speed: "fast", reasoning_effort: "low" } }, + ], + }); + const reopened = hydrateComplexityRouterConfig(saved, undefined); + const reopenedTier = custom ? reopened.custom_tier_set!.tiers[0].id : tier; + view.rerender(editor(reopened)); + expect(fast()).toBeChecked(); + + await user.click(screen.getByRole("combobox", { name: `Reasoning effort for primary in the ${label} tier` })); + await user.click(await screen.findByRole("option", { name: "low" })); + const effortChanged = onChange.mock.lastCall![0]; + expect(effortChanged.tier_model_params?.[reopenedTier].primary).toEqual({ + reasoning_effort: "low", + max_tokens: 1024, + speed: "fast", + }); + view.rerender(editor(effortChanged)); + await user.click(fast()); + const disabled = onChange.mock.lastCall![0]; + expect(disabled.tier_model_params).toEqual({ + ...effortChanged.tier_model_params, + [reopenedTier]: { + ...effortChanged.tier_model_params![reopenedTier], + primary: { reasoning_effort: "low", max_tokens: 1024 }, + }, + }); + view.rerender(editor(disabled)); + expect(fast()).not.toBeChecked(); + + const picker = () => screen.getByRole("combobox", { name: `Select model(s) for ${label.toLowerCase()} queries` }); + await user.click(picker()); + await user.click(await screen.findByRole("option", { name: "primary" })); + await user.keyboard("{Escape}"); + const deselected = onChange.mock.lastCall![0]; + expect(deselected.tier_model_params?.[reopenedTier]).toEqual({ + secondary: { speed: "fast" }, + blocked: { speed: "fast" }, + }); + view.rerender(editor(deselected)); + expect(screen.queryByRole("switch", { name: `Fast mode for primary in the ${label} tier` })).not.toBeInTheDocument(); + await user.click(picker()); + await user.click(await screen.findByRole("option", { name: "primary" })); + await user.keyboard("{Escape}"); + const reselected = onChange.mock.lastCall![0]; + view.rerender(editor(reselected)); + expect(fast()).not.toBeChecked(); + expect(screen.getByRole("combobox", { name: `Reasoning effort for primary in the ${label} tier` })).toHaveTextContent( + "Default", + ); +}); + +describe("Fast mode metadata", () => { + it("offers nothing before model capabilities load and leaves stored speed untouched", () => { + const value: ComplexityRouterConfigValue = { + tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic", + tier_model_params: { SIMPLE: { primary: { speed: "fast" } } }, + }; + const onChange = vi.fn(); + renderWithProviders(); + expect(screen.queryByRole("switch", { name: /^Fast mode for/ })).not.toBeInTheDocument(); + expect(onChange).not.toHaveBeenCalled(); + expect(buildUpdatedComplexityRouterConfig({}, value).tier_model_configs).toEqual({ + SIMPLE: [{ model_name: "primary", litellm_params: { speed: "fast" } }], + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx index ec9705b9451..54afa28fb6e 100644 --- a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx +++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx @@ -1,5 +1,6 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { SimpleTooltip } from "@/components/ui/tooltip"; +import { Switch } from "@/components/ui/switch"; import { Info } from "lucide-react"; import React from "react"; import { ReasoningEffort, TierModelParams } from "./complexity_router_tiers"; @@ -18,6 +19,8 @@ interface TierModelEffortRowsProps { effortOptionsByModel: Record; paramsByModel: Record | undefined; onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void; + fastModeByModel?: Record; + onFastModeChange: (model: string, enabled: boolean) => void; } export interface TierEffortRow { @@ -29,13 +32,16 @@ export interface TierEffortRow { /** * A stored effort outside the model's supported set (hand-authored, or capabilities changed since * it was saved) is listed anyway, so the row renders with its value selected and can be cleared. - * Only a model with no supported level and nothing stored drops out. */ export const tierEffortRows = ({ models, effortOptionsByModel, paramsByModel, -}: Pick): TierEffortRow[] => + fastModeByModel, +}: Pick< + TierModelEffortRowsProps, + "models" | "effortOptionsByModel" | "paramsByModel" | "fastModeByModel" +>): TierEffortRow[] => models .map((model) => { const effort = storedEffort(paramsByModel?.[model]); @@ -43,56 +49,74 @@ export const tierEffortRows = ({ const listed = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported; return { model, effort, options: Array.from(new Set(listed)) }; }) - .filter(({ options }) => options.length > 0); + .filter(({ model, options }) => options.length > 0 || fastModeByModel?.[model] === true); -const TierModelEffortRows: React.FC = ({ - tierLabel, - models, - effortOptionsByModel, - paramsByModel, - onEffortChange, -}) => { - const rows = tierEffortRows({ models, effortOptionsByModel, paramsByModel }); +const TierModelEffortRows: React.FC = (props) => { + const { tierLabel, paramsByModel, onEffortChange, fastModeByModel, onFastModeChange } = props; + const rows = tierEffortRows(props); if (rows.length === 0) return null; return (
-
- Reasoning effort - - - -
- {rows.map(({ model, effort, options }) => ( -
- {model} - + + +
+ )} + {rows.map(({ model, effort, options }) => ( +
+ + {model} + +
+ {options.length > 0 && ( + + )} + {fastModeByModel?.[model] === true && ( + + + + )} +
))}
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 9973aec7616..bc30b591ea9 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -48,6 +48,19 @@ const baseParams: BuildComplexityRouterConfigParams = { }; describe("buildComplexityRouterConfig", () => { + it("carries Fast and reasoning overrides independently into a new router payload", () => { + const params = { speed: "fast", reasoning_effort: "high", max_tokens: 1024 }; + const config = buildComplexityRouterConfig({ + ...baseParams, + tiers: { ...tiers, COMPLEX: ["primary"], REASONING: ["secondary"] }, + tierModelParams: { COMPLEX: { primary: params }, REASONING: { secondary: { speed: "fast" } } }, + }); + expect(config.tier_model_configs).toEqual({ + COMPLEX: [{ model_name: "primary", litellm_params: params }], + REASONING: [{ model_name: "secondary", litellm_params: { speed: "fast" } }], + }); + }); + it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => { const config = buildComplexityRouterConfig(baseParams); const expected = { diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts index b0fab49e80a..98a3fe792bd 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts @@ -7,6 +7,7 @@ import { serializeTierModelConfigs, tierRowLabel, setTierModelReasoningEffort, + setTierModelParam, } from "./complexity_router_tiers"; import { resolveComplexityDefaultModel } from "./tier_rows"; @@ -218,6 +219,28 @@ describe("setTierModelReasoningEffort", () => { }); }); +describe("setTierModelParam", () => { + it.each(["reasoning_effort", "speed"] as const)("clears only %s and preserves the input", (key) => { + const params = { reasoning_effort: "high", speed: "fast", max_tokens: 512 }; + const current = { COMPLEX: { primary: params, secondary: { speed: "fast" } }, REASONING: { primary: params } }; + const cleared = setTierModelParam(current, "COMPLEX", "primary", [key, undefined]); + expect(cleared).toEqual({ + ...current, + COMPLEX: { + ...current.COMPLEX, + primary: key === "speed" ? { reasoning_effort: "high", max_tokens: 512 } : { speed: "fast", max_tokens: 512 }, + }, + }); + expect(current.COMPLEX.primary).toEqual({ reasoning_effort: "high", speed: "fast", max_tokens: 512 }); + }); + + it("removes empty records when the only override is Fast", () => { + const enabled = setTierModelParam(undefined, "COMPLEX", "primary", ["speed", "fast"]); + expect(enabled).toEqual({ COMPLEX: { primary: { speed: "fast" } } }); + expect(setTierModelParam(enabled, "COMPLEX", "primary", ["speed", undefined])).toBeUndefined(); + }); +}); + describe("pruneTierModelParams", () => { it("drops params for models deselected from the tier", () => { expect( diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts index 3fec63518e5..916feaf26c0 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts @@ -114,14 +114,16 @@ export const serializeTierModelConfigs = ( return serialized.length > 0 ? Object.fromEntries(serialized) : undefined; }; -export const setTierModelReasoningEffort = ( +export type TierModelParamChange = ["reasoning_effort", ReasoningEffort | undefined] | ["speed", "fast" | undefined]; + +export const setTierModelParam = ( current: TierModelParamsByTier | undefined, tier: string, model: string, - effort: ReasoningEffort | undefined, + [key, value]: TierModelParamChange, ): TierModelParamsByTier | undefined => { - const { reasoning_effort: _dropped, ...rest } = current?.[tier]?.[model] ?? {}; - const params = effort === undefined ? rest : { ...rest, reasoning_effort: effort }; + const { [key]: _dropped, ...rest } = current?.[tier]?.[model] ?? {}; + const params = value === undefined ? rest : { ...rest, [key]: value }; const byModel = Object.fromEntries( Object.entries({ ...current?.[tier], [model]: params }).filter(([, value]) => Object.keys(value).length > 0), ); @@ -131,6 +133,13 @@ export const setTierModelReasoningEffort = ( return Object.keys(next).length > 0 ? next : undefined; }; +export const setTierModelReasoningEffort = ( + current: TierModelParamsByTier | undefined, + tier: string, + model: string, + effort: ReasoningEffort | undefined, +): TierModelParamsByTier | undefined => setTierModelParam(current, tier, model, ["reasoning_effort", effort]); + export const pruneTierModelParams = ( current: TierModelParamsByTier | undefined, tier: string, diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx index a3a6c77930b..c65ece4ca1c 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx @@ -52,6 +52,23 @@ describe("fetchAvailableModels", () => { ]); }); + it("carries only explicitly supported Fast capabilities, not accepted speed parameters", async () => { + modelHubCallMock.mockResolvedValue({ + data: [ + { model_group: "fast", supports_fast_mode: true }, + { model_group: "blocked", supports_fast_mode: false }, + { model_group: "missing", supports_speed: true }, + { model_group: "unknown", supports_fast_mode: null }, + ], + }); + expect(await fetchAvailableModels("token")).toEqual([ + { model_group: "blocked" }, + { model_group: "fast", supports_fast_mode: true }, + { model_group: "missing" }, + { model_group: "unknown" }, + ]); + }); + it("preserves absent, unknown, empty, and explicit effort capability states", async () => { modelHubCallMock.mockResolvedValue({ data: [ diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index 1d21b5e43ba..b3df5c9bf65 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -7,6 +7,7 @@ export interface ModelGroup { model_group: string; mode?: string; supports_reasoning?: boolean; + supports_fast_mode?: boolean; supported_reasoning_efforts?: string[] | null; } @@ -16,6 +17,7 @@ interface AvailableModel { id?: string | null; mode?: string | null; supports_reasoning?: boolean | null; + supports_fast_mode?: boolean | null; supported_reasoning_efforts?: string[] | null; } @@ -25,6 +27,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => { model_group: groupName, ...(item.mode && { mode: item.mode }), ...(item.supports_reasoning === true && { supports_reasoning: true }), + ...(item.supports_fast_mode === true && { supports_fast_mode: true }), ...(item.supported_reasoning_efforts !== undefined && { supported_reasoning_efforts: item.supported_reasoning_efforts, }), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 371fbbe67bd..a8801f643f4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32565,6 +32565,11 @@ export interface components { supported_openai_params: string[] | null; /** Supported Reasoning Efforts */ supported_reasoning_efforts?: string[] | null; + /** + * Supports Fast Mode + * @default false + */ + supports_fast_mode: boolean; /** * Supports Function Calling * @default false