diff --git a/litellm/router.py b/litellm/router.py index ef04423e3ab..0ef20ce2e2b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9463,6 +9463,11 @@ class Router: if model_info.get("supports_reasoning", None) is not None and model_info["supports_reasoning"] is True: model_group_info.supports_reasoning = True + if ( + model_info.get("supports_xhigh_reasoning_effort", None) is not None + and model_info["supports_xhigh_reasoning_effort"] is True + ): + model_group_info.supports_xhigh_reasoning_effort = True if ( model_info.get("supported_openai_params", None) is not None and model_info["supported_openai_params"] is not None diff --git a/litellm/types/router.py b/litellm/types/router.py index 99a4603ae49..1f05f707036 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -634,6 +634,7 @@ class ModelGroupInfo(BaseModel): supports_web_search: bool = Field(default=False) supports_url_context: bool = Field(default=False) supports_reasoning: bool = Field(default=False) + supports_xhigh_reasoning_effort: bool = Field(default=False) supports_function_calling: bool = Field(default=False) supported_openai_params: list[str] | None = Field(default=[]) configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 65debae9a16..994c65199b2 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1751,6 +1751,40 @@ def test_model_group_info_db_fallback_with_stringified_cost_values(): assert isinstance(result.output_cost_per_token, float) +def test_model_group_info_aggregates_xhigh_reasoning_support(): + router = litellm.Router( + model_list=[ + { + "model_name": "reasoning-model", + "litellm_params": {"model": "openai/reasoning-one"}, + "model_info": {"id": "deployment-one"}, + }, + { + "model_name": "reasoning-model", + "litellm_params": {"model": "openai/reasoning-two"}, + "model_info": {"id": "deployment-two"}, + }, + ] + ) + + def _model_info(model_id: str, model_name: str): + return { + "key": model_name, + "litellm_provider": "openai", + "mode": "chat", + "supports_xhigh_reasoning_effort": model_id == "deployment-two", + } + + with patch.object(router, "get_deployment_model_info", side_effect=_model_info): + result = router._set_model_group_info( + model_group="reasoning-model", + user_facing_model_group_name="reasoning-model", + ) + + assert result is not None + assert result.supports_xhigh_reasoning_effort is True + + def test_get_model_access_groups_caching(): """ Test that get_model_access_groups caches the no-args result diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 5f5ae703b0e..133a32a7951 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -60,6 +60,48 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText(/Think step by step/)).toBeInTheDocument(); }); + it("shows reasoning controls only for capable models and gates xhigh", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + const xhighControl = screen.getByRole("combobox", { name: "Reasoning effort for reasoning-model" }); + const basicControl = screen.getByRole("combobox", { name: "Reasoning effort for reasoning-basic" }); + expect(xhighControl).toBeInTheDocument(); + expect(basicControl).toBeInTheDocument(); + expect(screen.queryByLabelText("Reasoning effort for chat-model")).not.toBeInTheDocument(); + await user.click(xhighControl); + expect(screen.getByText("xhigh")).toBeInTheDocument(); + await user.click(screen.getByText("xhigh")); + await user.click(basicControl); + expect(screen.queryByRole("option", { name: "xhigh" })).not.toBeInTheDocument(); + }); + it("should display the how classification works section", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index ab6b1d401ce..9435877b039 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -17,6 +17,7 @@ import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; +import { TierModelParamsByTier } from "./complexity_router_tiers"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; @@ -27,6 +28,8 @@ export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200; export const DEFAULT_SESSION_AFFINITY = false; export const DEFAULT_DEPLOYMENT_AFFINITY = true; +const REASONING_EFFORT_UNSET = "__model_default__"; + export interface ComplexityTiers { SIMPLE: string[]; MEDIUM: string[]; @@ -116,6 +119,7 @@ export type ComplexityTierLabels = Partial export interface ComplexityRouterConfigValue { tiers: ComplexityTiers; + tier_model_params?: TierModelParamsByTier; tier_labels?: ComplexityTierLabels; /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ default_model?: string; @@ -238,9 +242,41 @@ const ComplexityRouterConfig: React.FC = ({ })); const handleTierChange = (tier: keyof ComplexityTiers, models: string[]) => { + const selectedModels = new Set(models); + const existingParams = value.tier_model_params?.[tier]; + const tierModelParams = existingParams + ? Object.fromEntries(Object.entries(existingParams).filter(([model]) => selectedModels.has(model))) + : undefined; onChange({ ...value, tiers: { ...value.tiers, [tier]: models }, + tier_model_params: + tierModelParams && Object.keys(tierModelParams).length > 0 + ? { ...value.tier_model_params, [tier]: tierModelParams } + : Object.fromEntries(Object.entries(value.tier_model_params ?? {}).filter(([key]) => key !== tier)), + }); + }; + + const handleReasoningEffortChange = ( + tier: keyof ComplexityTiers, + model: string, + reasoningEffort: string | undefined, + ) => { + const tierModelParams = { ...(value.tier_model_params?.[tier] ?? {}) }; + const existingParams = tierModelParams[model] ?? {}; + const updatedParams = reasoningEffort + ? { ...existingParams, reasoning_effort: reasoningEffort } + : Object.fromEntries(Object.entries(existingParams).filter(([key]) => key !== "reasoning_effort")); + const updatedTierModelParams = + updatedParams && Object.keys(updatedParams).length > 0 + ? { ...tierModelParams, [model]: updatedParams } + : Object.fromEntries(Object.entries(tierModelParams).filter(([key]) => key !== model)); + onChange({ + ...value, + tier_model_params: + Object.keys(updatedTierModelParams).length > 0 + ? { ...value.tier_model_params, [tier]: updatedTierModelParams } + : Object.fromEntries(Object.entries(value.tier_model_params ?? {}).filter(([key]) => key !== tier)), }); }; @@ -325,6 +361,46 @@ const ComplexityRouterConfig: React.FC = ({ emptyText="No models found" className={tierMissing ? "w-full border-destructive" : "w-full"} /> + {value.tiers[tier].map((model) => { + const modelDetails = modelInfo.find((candidate) => candidate.model_group === model); + if (!modelDetails?.supports_reasoning) return null; + const currentEffort = value.tier_model_params?.[tier]?.[model]?.reasoning_effort; + const effortOptions = [ + "minimal", + "low", + "medium", + "high", + ...(modelDetails.supports_xhigh_reasoning_effort ? ["xhigh"] : []), + ]; + return ( +
+ Reasoning effort for {model} + +
+ ); + })} {value.tiers[tier].length > 1 && ( Multiple models selected — the router randomly picks among them per request (or Thompson-samples diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 64d1519f915..a8f72dc7db1 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -348,6 +348,7 @@ const AddAutoRouterTab: React.FC = ({ tiers: complexityRouterConfig.tiers, defaultModel: complexityRouterConfig.default_model, planModeMinTier: complexityRouterConfig.plan_mode_min_tier, + tierModelParams: complexityRouterConfig.tier_model_params, tierLabels: complexityRouterConfig.tier_labels, classifierType: complexityRouterConfig.classifier_type, classifierLlmConfig: complexityRouterConfig.classifier_llm_config, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index bddf8321ad2..5554c615927 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -15,6 +15,7 @@ import { effectiveTierLabel, heuristicScoringRoleFor, } from "./ComplexityRouterConfig"; +import { serializeTierConfig, type TierModelParamsByTier } from "./complexity_router_tiers"; /** * Drop an empty system_prompt so the payload carries an override only when there is one. The @@ -75,6 +76,7 @@ export interface BuildComplexityRouterConfigParams { tiers: ComplexityTiers; defaultModel: string | undefined; planModeMinTier: string | undefined; + tierModelParams?: TierModelParamsByTier; tierLabels: ComplexityTierLabels | undefined; classifierType: ClassifierType; classifierLlmConfig: ClassifierLLMConfig | undefined; @@ -102,7 +104,7 @@ export interface BuildComplexityRouterConfigParams { } export interface ComplexityRouterConfigPayload { - tiers: ComplexityTiers; + tiers: Record; default_model?: string; plan_mode_min_tier?: string; tier_labels?: ComplexityTierLabels; @@ -212,6 +214,7 @@ export const buildComplexityRouterConfig = ({ tiers, defaultModel, planModeMinTier, + tierModelParams, tierLabels, classifierType, classifierLlmConfig, @@ -251,7 +254,7 @@ export const buildComplexityRouterConfig = ({ const scorerKnobs = scorerKnobPayload(scorerInputs); return { - tiers, + tiers: serializeTierConfig(tiers, tierModelParams), ...(defaultModel?.trim() && { default_model: defaultModel }), ...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }), ...(cleanedTierLabels && { tier_labels: cleanedTierLabels }), diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts index 8f75b26f250..06ad0e72e4b 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; -import { normalizeTierModels, resolveComplexityDefaultModel } from "./complexity_router_tiers"; +import { + extractTierModelParams, + normalizeTierModels, + resolveComplexityDefaultModel, + serializeTierConfig, +} from "./complexity_router_tiers"; import type { ComplexityTiers } from "./ComplexityRouterConfig"; @@ -29,6 +34,48 @@ describe("normalizeTierModels", () => { it.each([[undefined], [null], [{}], [42]])("returns no models for %s", (value) => { expect(normalizeTierModels(value)).toEqual([]); }); + + it("widens a single object entry", () => { + expect(normalizeTierModels({ model_name: "o3", litellm_params: { reasoning_effort: "high" } })).toEqual(["o3"]); + }); + + it("widens mixed string and object entries", () => { + expect( + normalizeTierModels(["gpt-4o-mini", { model_name: "o3", litellm_params: { reasoning_effort: "high" } }]), + ).toEqual(["gpt-4o-mini", "o3"]); + }); + + it("extracts per-model params while preserving unrelated request params", () => { + expect( + extractTierModelParams([ + "gpt-4o-mini", + { model_name: "o3", litellm_params: { reasoning_effort: "high", max_tokens: 1000 } }, + ]), + ).toEqual({ o3: { reasoning_effort: "high", max_tokens: 1000 } }); + }); + + it("serializes unset entries as strings and configured entries as objects", () => { + expect( + serializeTierConfig( + { SIMPLE: ["gpt-4o-mini", "o3"], REASONING: ["o3"] }, + { SIMPLE: { o3: { reasoning_effort: "high" } } }, + ), + ).toEqual({ + SIMPLE: ["gpt-4o-mini", { model_name: "o3", litellm_params: { reasoning_effort: "high" } }], + REASONING: ["o3"], + }); + }); + + it("round-trips a stored single object entry with unknown params", () => { + const stored = { + model_name: "o3", + litellm_params: { reasoning_effort: "xhigh", custom_request_param: "preserve-me" }, + }; + const tiers = { REASONING: normalizeTierModels(stored) }; + const params = { REASONING: extractTierModelParams(stored) }; + + expect(serializeTierConfig(tiers, params)).toEqual({ REASONING: stored }); + }); }); // router.py derives the default as `MEDIUM or SIMPLE` and raises when neither holds a model, so diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts index 6f5eb7f877b..7e415504fd2 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts @@ -2,7 +2,7 @@ import type { ComplexityTiers } from "./ComplexityRouterConfig"; import type { ComplexityTier } from "./KeywordTierRules"; /** - * A complexity tier maps to `str | list[str]` on the backend + * A complexity tier maps to `str | list[str] | object | list[object]` on the backend * (litellm/router_strategy/complexity_router/config.py: "string = pin; list = random pick"), * and the router widens the bare string with `models if isinstance(models, list) else [models]`. * @@ -11,9 +11,19 @@ import type { ComplexityTier } from "./KeywordTierRules"; * and do not need it. */ export const normalizeTierModels = (value: unknown): string[] => { - if (Array.isArray(value)) return value.filter((model): model is string => typeof model === "string"); - if (typeof value === "string" && value) return [value]; - return []; + const entries = Array.isArray(value) ? value : [value]; + return entries.flatMap((entry) => { + if (typeof entry === "string" && entry) return [entry]; + if ( + typeof entry === "object" && + entry !== null && + !Array.isArray(entry) && + typeof (entry as { model_name?: unknown }).model_name === "string" + ) { + return [(entry as { model_name: string }).model_name]; + } + return []; + }); }; /** @@ -37,3 +47,47 @@ export const tierOptions = ( tierLabels: Partial> | undefined, ): { value: ComplexityTier; label: string }[] => TIER_ORDER.map((tier) => ({ value: tier, label: tierLabels?.[tier]?.trim() || DEFAULT_TIER_LABELS[tier] })); + +export type TierModelParams = Record; +export type SerializedTierModel = string | { model_name: string; litellm_params: TierModelParams }; + +export type TierModelParamsByTier = Partial>>; + +export const extractTierModelParams = (value: unknown): Record => { + const entries = Array.isArray(value) ? value : [value]; + return Object.fromEntries( + entries.flatMap((entry) => { + if ( + typeof entry !== "object" || + entry === null || + Array.isArray(entry) || + typeof (entry as { model_name?: unknown }).model_name !== "string" + ) { + return []; + } + const params = (entry as { litellm_params?: unknown }).litellm_params; + if (typeof params !== "object" || params === null || Array.isArray(params)) return []; + return [[(entry as { model_name: string }).model_name, params as TierModelParams] as const]; + }), + ); +}; + +export const serializeTierModels = ( + models: string[], + paramsByModel: Record | undefined, +): SerializedTierModel | SerializedTierModel[] => { + const entries = models.map((model) => { + const params = paramsByModel?.[model]; + return params && Object.keys(params).length > 0 ? { model_name: model, litellm_params: params } : model; + }); + if (entries.length === 1 && typeof entries[0] !== "string") return entries[0]; + return entries; +}; + +export const serializeTierConfig = ( + tiers: Partial, + paramsByTier: TierModelParamsByTier | undefined, +): Record => + Object.fromEntries( + Object.entries(tiers).map(([tier, models]) => [tier, serializeTierModels(models ?? [], paramsByTier?.[tier])]), + ); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 0ae782dc5fc..2be18d19e83 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -45,6 +45,32 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { expect(result.some_future_backend_key).toEqual({ nested: true }); }); + it("preserves stored tier request params when saving an edit", () => { + const stored = { + ...STORED, + tiers: { + SIMPLE: { + model_name: "gpt-4o-mini", + litellm_params: { reasoning_effort: "high", custom_request_param: "preserve-me" }, + }, + MEDIUM: [], + COMPLEX: [], + REASONING: [], + }, + }; + const value = { + ...FORM_VALUE, + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tier_model_params: { + SIMPLE: { + "gpt-4o-mini": { reasoning_effort: "high", custom_request_param: "preserve-me" }, + }, + }, + }; + + expect(buildUpdatedComplexityRouterConfig(stored, value, undefined, hydratedState).tiers).toEqual(stored.tiers); + }); + it("persists an edited keyword rule", () => { const result = buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE, undefined, { ...hydratedState, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 012624454d3..60be7c71c1d 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -14,7 +14,12 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC import { modelAvailableCall, modelPatchUpdateCall } from "../networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder from "../add_model/RouterConfigBuilder"; -import { normalizeTierModels, resolveComplexityDefaultModel } from "../add_model/complexity_router_tiers"; +import { + extractTierModelParams, + normalizeTierModels, + resolveComplexityDefaultModel, + serializeTierConfig, +} from "../add_model/complexity_router_tiers"; import { isComplexityRouter } from "../add_model/auto_router_strategies"; import { getKeywordTierRulesError, @@ -151,7 +156,7 @@ export const buildUpdatedComplexityRouterConfig = ( return { ...preservedConfig, - tiers: value.tiers, + tiers: serializeTierConfig(value.tiers, value.tier_model_params), ...(value.default_model?.trim() && { default_model: value.default_model }), ...(value.plan_mode_min_tier?.trim() && { plan_mode_min_tier: value.plan_mode_min_tier }), ...(serializedTierLabels && { tier_labels: serializedTierLabels }), @@ -342,6 +347,12 @@ const EditAutoRouterModal: React.FC = ({ const hydratedComplexityRouterConfig: ComplexityRouterConfigValue = { tiers: hydratedTiers, + tier_model_params: Object.fromEntries( + ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((tier) => [ + tier, + extractTierModelParams(parsedConfig.tiers?.[tier]), + ]), + ), default_model: hydratePinnedDefaultModel( parsedConfig.default_model, modelData.litellm_params?.complexity_router_default_model, diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index 9fbaa868998..c24eda20f06 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -6,6 +6,8 @@ import { modelAvailableCall, modelHubCall } from "@/components/networking"; export interface ModelGroup { model_group: string; mode?: string; + supports_reasoning?: boolean; + supports_xhigh_reasoning_effort?: boolean; } interface AvailableModel { @@ -13,6 +15,8 @@ interface AvailableModel { model_name?: string | null; id?: string | null; mode?: string | null; + supports_reasoning?: boolean | null; + supports_xhigh_reasoning_effort?: boolean | null; } export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: string): Promise => { @@ -36,6 +40,8 @@ export const fetchAvailableModels = async (accessToken: string): Promise ({ model_group: item.model_group || item.id || item.model_name || "", mode: item.mode || undefined, + supports_reasoning: item.supports_reasoning ?? undefined, + supports_xhigh_reasoning_effort: item.supports_xhigh_reasoning_effort ?? undefined, })) .filter((model: ModelGroup) => model.model_group !== ""); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index fed15a2ba41..5e042ee74a1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29470,6 +29470,11 @@ export interface components { * @default false */ supports_web_search: boolean; + /** + * Supports Xhigh Reasoning Effort + * @default false + */ + supports_xhigh_reasoning_effort: boolean; /** Tpm */ tpm?: number | null; };