scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(dashboard): add reasoning effort controls to complexity tiers

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-08-16 02:56:54 +00:00
parent f5cfa84220
commit 5f12fdfca2
13 changed files with 320 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/router.py
View file

@ -9463,6 +9463,11 @@ class Router:
if model_info.get("supports_reasoning", None) is not None and model_info["supports_reasoning"] is True:
model_group_info.supports_reasoning = True
if (
model_info.get("supports_xhigh_reasoning_effort", None) is not None
and model_info["supports_xhigh_reasoning_effort"] is True
):
model_group_info.supports_xhigh_reasoning_effort = True
if (
model_info.get("supported_openai_params", None) is not None
and model_info["supported_openai_params"] is not None

1
litellm/types/router.py
View file

@ -634,6 +634,7 @@ class ModelGroupInfo(BaseModel):
supports_web_search: bool = Field(default=False)
supports_url_context: bool = Field(default=False)
supports_reasoning: bool = Field(default=False)
supports_xhigh_reasoning_effort: bool = Field(default=False)
supports_function_calling: bool = Field(default=False)
supported_openai_params: list[str] | None = Field(default=[])
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None

34
tests/test_litellm/test_router.py
View file

@ -1751,6 +1751,40 @@ def test_model_group_info_db_fallback_with_stringified_cost_values():
assert isinstance(result.output_cost_per_token, float)
def test_model_group_info_aggregates_xhigh_reasoning_support():
router = litellm.Router(
model_list=[
{
"model_name": "reasoning-model",
"litellm_params": {"model": "openai/reasoning-one"},
"model_info": {"id": "deployment-one"},
},
{
"model_name": "reasoning-model",
"litellm_params": {"model": "openai/reasoning-two"},
"model_info": {"id": "deployment-two"},
},
]
)
def _model_info(model_id: str, model_name: str):
return {
"key": model_name,
"litellm_provider": "openai",
"mode": "chat",
"supports_xhigh_reasoning_effort": model_id == "deployment-two",
}
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
result = router._set_model_group_info(
model_group="reasoning-model",
user_facing_model_group_name="reasoning-model",
)
assert result is not None
assert result.supports_xhigh_reasoning_effort is True
def test_get_model_access_groups_caching():
"""
Test that get_model_access_groups caches the no-args result

42
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
View file

@ -60,6 +60,48 @@ describe("ComplexityRouterConfig", () => {
expect(screen.getByText(/Think step by step/)).toBeInTheDocument();
});
it("shows reasoning controls only for capable models and gates xhigh", async () => {
const user = userEvent.setup();
renderWithProviders(
<ComplexityRouterConfig
{...baseProps}
modelInfo={[
{
model_group: "reasoning-model",
mode: "chat",
supports_reasoning: true,
supports_xhigh_reasoning_effort: true,
},
{
model_group: "reasoning-basic",
mode: "chat",
supports_reasoning: true,
supports_xhigh_reasoning_effort: false,
},
{ model_group: "chat-model", mode: "chat", supports_reasoning: false },
]}
value={{
...defaultValue,
tiers: {
...defaultValue.tiers,
REASONING: ["reasoning-model", "reasoning-basic", "chat-model"],
},
}}
/>,
);
const xhighControl = screen.getByRole("combobox", { name: "Reasoning effort for reasoning-model" });
const basicControl = screen.getByRole("combobox", { name: "Reasoning effort for reasoning-basic" });
expect(xhighControl).toBeInTheDocument();
expect(basicControl).toBeInTheDocument();
expect(screen.queryByLabelText("Reasoning effort for chat-model")).not.toBeInTheDocument();
await user.click(xhighControl);
expect(screen.getByText("xhigh")).toBeInTheDocument();
await user.click(screen.getByText("xhigh"));
await user.click(basicControl);
expect(screen.queryByRole("option", { name: "xhigh" })).not.toBeInTheDocument();
});
it("should display the how classification works section", () => {
renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
fireEvent.click(screen.getByText("Advanced: Classification Method"));

76
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
View file

@ -17,6 +17,7 @@ import EscalationKeywords from "./EscalationKeywords";
import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
import SemanticKeywordMatching from "./SemanticKeywordMatching";
import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
import { TierModelParamsByTier } from "./complexity_router_tiers";
export type { DimensionWeights, TierBoundaries, TokenThresholds };
@ -27,6 +28,8 @@ export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200;
export const DEFAULT_SESSION_AFFINITY = false;
export const DEFAULT_DEPLOYMENT_AFFINITY = true;
const REASONING_EFFORT_UNSET = "__model_default__";
export interface ComplexityTiers {
SIMPLE: string[];
MEDIUM: string[];
@ -116,6 +119,7 @@ export type ComplexityTierLabels = Partial<Record<keyof ComplexityTiers, string>
export interface ComplexityRouterConfigValue {
tiers: ComplexityTiers;
tier_model_params?: TierModelParamsByTier;
tier_labels?: ComplexityTierLabels;
/** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */
default_model?: string;
@ -238,9 +242,41 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
}));
const handleTierChange = (tier: keyof ComplexityTiers, models: string[]) => {
const selectedModels = new Set(models);
const existingParams = value.tier_model_params?.[tier];
const tierModelParams = existingParams
? Object.fromEntries(Object.entries(existingParams).filter(([model]) => selectedModels.has(model)))
: undefined;
onChange({
...value,
tiers: { ...value.tiers, [tier]: models },
tier_model_params:
tierModelParams && Object.keys(tierModelParams).length > 0
? { ...value.tier_model_params, [tier]: tierModelParams }
: Object.fromEntries(Object.entries(value.tier_model_params ?? {}).filter(([key]) => key !== tier)),
});
};
const handleReasoningEffortChange = (
tier: keyof ComplexityTiers,
model: string,
reasoningEffort: string | undefined,
) => {
const tierModelParams = { ...(value.tier_model_params?.[tier] ?? {}) };
const existingParams = tierModelParams[model] ?? {};
const updatedParams = reasoningEffort
? { ...existingParams, reasoning_effort: reasoningEffort }
: Object.fromEntries(Object.entries(existingParams).filter(([key]) => key !== "reasoning_effort"));
const updatedTierModelParams =
updatedParams && Object.keys(updatedParams).length > 0
? { ...tierModelParams, [model]: updatedParams }
: Object.fromEntries(Object.entries(tierModelParams).filter(([key]) => key !== model));
onChange({
...value,
tier_model_params:
Object.keys(updatedTierModelParams).length > 0
? { ...value.tier_model_params, [tier]: updatedTierModelParams }
: Object.fromEntries(Object.entries(value.tier_model_params ?? {}).filter(([key]) => key !== tier)),
});
};
@ -325,6 +361,46 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
emptyText="No models found"
className={tierMissing ? "w-full border-destructive" : "w-full"}
/>
{value.tiers[tier].map((model) => {
const modelDetails = modelInfo.find((candidate) => candidate.model_group === model);
if (!modelDetails?.supports_reasoning) return null;
const currentEffort = value.tier_model_params?.[tier]?.[model]?.reasoning_effort;
const effortOptions = [
"minimal",
"low",
"medium",
"high",
...(modelDetails.supports_xhigh_reasoning_effort ? ["xhigh"] : []),
];
return (
<div key={`${tier}-${model}`} className="mt-2">
<span className="block mb-1 text-xs text-muted-foreground">Reasoning effort for {model}</span>
<Select
items={effortOptions.map((effort) => ({ value: effort, label: effort }))}
value={typeof currentEffort === "string" ? currentEffort : REASONING_EFFORT_UNSET}
onValueChange={(effort: string | null) =>
handleReasoningEffortChange(
tier,
model,
!effort || effort === REASONING_EFFORT_UNSET ? undefined : effort,
)
}
>
<SelectTrigger aria-label={`Reasoning effort for ${model}`} className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value={REASONING_EFFORT_UNSET}>Model default</SelectItem>
{effortOptions.map((effort) => (
<SelectItem key={effort} value={effort}>
{effort}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
);
})}
{value.tiers[tier].length > 1 && (
<span className="text-xs text-muted-foreground">
Multiple models selected — the router randomly picks among them per request (or Thompson-samples

1
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
View file

@ -348,6 +348,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
tiers: complexityRouterConfig.tiers,
defaultModel: complexityRouterConfig.default_model,
planModeMinTier: complexityRouterConfig.plan_mode_min_tier,
tierModelParams: complexityRouterConfig.tier_model_params,
tierLabels: complexityRouterConfig.tier_labels,
classifierType: complexityRouterConfig.classifier_type,
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,

7
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -15,6 +15,7 @@ import {
effectiveTierLabel,
heuristicScoringRoleFor,
} from "./ComplexityRouterConfig";
import { serializeTierConfig, type TierModelParamsByTier } from "./complexity_router_tiers";
/**
* Drop an empty system_prompt so the payload carries an override only when there is one. The
@ -75,6 +76,7 @@ export interface BuildComplexityRouterConfigParams {
tiers: ComplexityTiers;
defaultModel: string | undefined;
planModeMinTier: string | undefined;
tierModelParams?: TierModelParamsByTier;
tierLabels: ComplexityTierLabels | undefined;
classifierType: ClassifierType;
classifierLlmConfig: ClassifierLLMConfig | undefined;
@ -102,7 +104,7 @@ export interface BuildComplexityRouterConfigParams {
}
export interface ComplexityRouterConfigPayload {
tiers: ComplexityTiers;
tiers: Record<string, unknown>;
default_model?: string;
plan_mode_min_tier?: string;
tier_labels?: ComplexityTierLabels;
@ -212,6 +214,7 @@ export const buildComplexityRouterConfig = ({
tiers,
defaultModel,
planModeMinTier,
tierModelParams,
tierLabels,
classifierType,
classifierLlmConfig,
@ -251,7 +254,7 @@ export const buildComplexityRouterConfig = ({
const scorerKnobs = scorerKnobPayload(scorerInputs);
return {
tiers,
tiers: serializeTierConfig(tiers, tierModelParams),
...(defaultModel?.trim() && { default_model: defaultModel }),
...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }),
...(cleanedTierLabels && { tier_labels: cleanedTierLabels }),

49
ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts
View file

@ -1,6 +1,11 @@
import { describe, expect, it } from "vitest";
import { normalizeTierModels, resolveComplexityDefaultModel } from "./complexity_router_tiers";
import {
extractTierModelParams,
normalizeTierModels,
resolveComplexityDefaultModel,
serializeTierConfig,
} from "./complexity_router_tiers";
import type { ComplexityTiers } from "./ComplexityRouterConfig";
@ -29,6 +34,48 @@ describe("normalizeTierModels", () => {
it.each([[undefined], [null], [{}], [42]])("returns no models for %s", (value) => {
expect(normalizeTierModels(value)).toEqual([]);
});
it("widens a single object entry", () => {
expect(normalizeTierModels({ model_name: "o3", litellm_params: { reasoning_effort: "high" } })).toEqual(["o3"]);
});
it("widens mixed string and object entries", () => {
expect(
normalizeTierModels(["gpt-4o-mini", { model_name: "o3", litellm_params: { reasoning_effort: "high" } }]),
).toEqual(["gpt-4o-mini", "o3"]);
});
it("extracts per-model params while preserving unrelated request params", () => {
expect(
extractTierModelParams([
"gpt-4o-mini",
{ model_name: "o3", litellm_params: { reasoning_effort: "high", max_tokens: 1000 } },
]),
).toEqual({ o3: { reasoning_effort: "high", max_tokens: 1000 } });
});
it("serializes unset entries as strings and configured entries as objects", () => {
expect(
serializeTierConfig(
{ SIMPLE: ["gpt-4o-mini", "o3"], REASONING: ["o3"] },
{ SIMPLE: { o3: { reasoning_effort: "high" } } },
),
).toEqual({
SIMPLE: ["gpt-4o-mini", { model_name: "o3", litellm_params: { reasoning_effort: "high" } }],
REASONING: ["o3"],
});
});
it("round-trips a stored single object entry with unknown params", () => {
const stored = {
model_name: "o3",
litellm_params: { reasoning_effort: "xhigh", custom_request_param: "preserve-me" },
};
const tiers = { REASONING: normalizeTierModels(stored) };
const params = { REASONING: extractTierModelParams(stored) };
expect(serializeTierConfig(tiers, params)).toEqual({ REASONING: stored });
});
});
// router.py derives the default as `MEDIUM or SIMPLE` and raises when neither holds a model, so

62
ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
View file

@ -2,7 +2,7 @@ import type { ComplexityTiers } from "./ComplexityRouterConfig";
import type { ComplexityTier } from "./KeywordTierRules";
/**
* A complexity tier maps to `str | list[str]` on the backend
* A complexity tier maps to `str | list[str] | object | list[object]` on the backend
* (litellm/router_strategy/complexity_router/config.py: "string = pin; list = random pick"),
* and the router widens the bare string with `models if isinstance(models, list) else [models]`.
*
@ -11,9 +11,19 @@ import type { ComplexityTier } from "./KeywordTierRules";
* and do not need it.
*/
export const normalizeTierModels = (value: unknown): string[] => {
if (Array.isArray(value)) return value.filter((model): model is string => typeof model === "string");
if (typeof value === "string" && value) return [value];
return [];
const entries = Array.isArray(value) ? value : [value];
return entries.flatMap((entry) => {
if (typeof entry === "string" && entry) return [entry];
if (
typeof entry === "object" &&
entry !== null &&
!Array.isArray(entry) &&
typeof (entry as { model_name?: unknown }).model_name === "string"
) {
return [(entry as { model_name: string }).model_name];
}
return [];
});
};
/**
@ -37,3 +47,47 @@ export const tierOptions = (
tierLabels: Partial<Record<ComplexityTier, string>> | undefined,
): { value: ComplexityTier; label: string }[] =>
TIER_ORDER.map((tier) => ({ value: tier, label: tierLabels?.[tier]?.trim() || DEFAULT_TIER_LABELS[tier] }));
export type TierModelParams = Record<string, unknown>;
export type SerializedTierModel = string | { model_name: string; litellm_params: TierModelParams };
export type TierModelParamsByTier = Partial<Record<string, Record<string, TierModelParams>>>;
export const extractTierModelParams = (value: unknown): Record<string, TierModelParams> => {
const entries = Array.isArray(value) ? value : [value];
return Object.fromEntries(
entries.flatMap((entry) => {
if (
typeof entry !== "object" ||
entry === null ||
Array.isArray(entry) ||
typeof (entry as { model_name?: unknown }).model_name !== "string"
) {
return [];
}
const params = (entry as { litellm_params?: unknown }).litellm_params;
if (typeof params !== "object" || params === null || Array.isArray(params)) return [];
return [[(entry as { model_name: string }).model_name, params as TierModelParams] as const];
}),
);
};
export const serializeTierModels = (
models: string[],
paramsByModel: Record<string, TierModelParams> | undefined,
): SerializedTierModel | SerializedTierModel[] => {
const entries = models.map((model) => {
const params = paramsByModel?.[model];
return params && Object.keys(params).length > 0 ? { model_name: model, litellm_params: params } : model;
});
if (entries.length === 1 && typeof entries[0] !== "string") return entries[0];
return entries;
};
export const serializeTierConfig = (
tiers: Partial<ComplexityTiers>,
paramsByTier: TierModelParamsByTier | undefined,
): Record<string, SerializedTierModel | SerializedTierModel[]> =>
Object.fromEntries(
Object.entries(tiers).map(([tier, models]) => [tier, serializeTierModels(models ?? [], paramsByTier?.[tier])]),
);

26
ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
View file

@ -45,6 +45,32 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
expect(result.some_future_backend_key).toEqual({ nested: true });
});
it("preserves stored tier request params when saving an edit", () => {
const stored = {
...STORED,
tiers: {
SIMPLE: {
model_name: "gpt-4o-mini",
litellm_params: { reasoning_effort: "high", custom_request_param: "preserve-me" },
},
MEDIUM: [],
COMPLEX: [],
REASONING: [],
},
};
const value = {
...FORM_VALUE,
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
tier_model_params: {
SIMPLE: {
"gpt-4o-mini": { reasoning_effort: "high", custom_request_param: "preserve-me" },
},
},
};
expect(buildUpdatedComplexityRouterConfig(stored, value, undefined, hydratedState).tiers).toEqual(stored.tiers);
});
it("persists an edited keyword rule", () => {
const result = buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE, undefined, {
...hydratedState,

15
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
View file

@ -14,7 +14,12 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC
import { modelAvailableCall, modelPatchUpdateCall } from "../networking";
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
import RouterConfigBuilder from "../add_model/RouterConfigBuilder";
import { normalizeTierModels, resolveComplexityDefaultModel } from "../add_model/complexity_router_tiers";
import {
extractTierModelParams,
normalizeTierModels,
resolveComplexityDefaultModel,
serializeTierConfig,
} from "../add_model/complexity_router_tiers";
import { isComplexityRouter } from "../add_model/auto_router_strategies";
import {
getKeywordTierRulesError,
@ -151,7 +156,7 @@ export const buildUpdatedComplexityRouterConfig = (
return {
...preservedConfig,
tiers: value.tiers,
tiers: serializeTierConfig(value.tiers, value.tier_model_params),
...(value.default_model?.trim() && { default_model: value.default_model }),
...(value.plan_mode_min_tier?.trim() && { plan_mode_min_tier: value.plan_mode_min_tier }),
...(serializedTierLabels && { tier_labels: serializedTierLabels }),
@ -342,6 +347,12 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
const hydratedComplexityRouterConfig: ComplexityRouterConfigValue = {
tiers: hydratedTiers,
tier_model_params: Object.fromEntries(
["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((tier) => [
tier,
extractTierModelParams(parsedConfig.tiers?.[tier]),
]),
),
default_model: hydratePinnedDefaultModel(
parsedConfig.default_model,
modelData.litellm_params?.complexity_router_default_model,

6
ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
View file

@ -6,6 +6,8 @@ import { modelAvailableCall, modelHubCall } from "@/components/networking";
export interface ModelGroup {
model_group: string;
mode?: string;
supports_reasoning?: boolean;
supports_xhigh_reasoning_effort?: boolean;
}
interface AvailableModel {
@ -13,6 +15,8 @@ interface AvailableModel {
model_name?: string | null;
id?: string | null;
mode?: string | null;
supports_reasoning?: boolean | null;
supports_xhigh_reasoning_effort?: boolean | null;
}
export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: string): Promise<ModelGroup[]> => {
@ -36,6 +40,8 @@ export const fetchAvailableModels = async (accessToken: string): Promise<ModelGr
.map((item: AvailableModel) => ({
model_group: item.model_group || item.id || item.model_name || "",
mode: item.mode || undefined,
supports_reasoning: item.supports_reasoning ?? undefined,
supports_xhigh_reasoning_effort: item.supports_xhigh_reasoning_effort ?? undefined,
}))
.filter((model: ModelGroup) => model.model_group !== "");

5
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -29470,6 +29470,11 @@ export interface components {
* @default false
*/
supports_web_search: boolean;
/**
* Supports Xhigh Reasoning Effort
* @default false
*/
supports_xhigh_reasoning_effort: boolean;
/** Tpm */
tpm?: number | null;
};