diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 1537e3a540c..c4dbe68ebe2 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1321,7 +1321,7 @@ class ComplexityRouterConfig(BaseModel): ) enable_context_window_escalation: bool = Field( - default=True, + default=False, description=( "Escalate a request off a tier whose models provably cannot hold its prompt, before " "dispatch. The classifier scores complexity and never prompt size, so a long agentic " @@ -1331,7 +1331,8 @@ class ComplexityRouterConfig(BaseModel): "moves to the lowest configured tier with a model whose declared window fits; when " "only some of the tier's models fit, the pick is restricted to those and the tier " "keeps the request. Models with no resolvable window are never escalated away from " - "and never escalated onto. Set false to dispatch on complexity alone, as before." + "and never escalated onto. Disabled by default: omit or set false to dispatch on " + "complexity alone; set true to enable context-window escalation." ), ) context_window_escalation_buffer: float = Field( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 83f30dc52a4..97e7a799053 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -13,6 +13,7 @@ import time from collections.abc import AsyncIterator, Mapping, Sequence from copy import deepcopy from functools import partial +from types import MappingProxyType from typing import Dict, Final, List, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -90,6 +91,7 @@ from litellm.types.router import ( TaggedPreRoutingStrategy, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -6681,6 +6683,7 @@ class TestTierModelAffinity: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config={ "tiers": {"SIMPLE": ["small-model", "big-model"]}, + "enable_context_window_escalation": True, "adaptive": adaptive, "deployment_affinity": True, "session_affinity": False, @@ -13878,8 +13881,12 @@ _CJK_TURNS = [ ] -def _tier_config(**overrides) -> Dict: - return {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, **overrides} +def _tier_config(**overrides: object) -> dict[str, object]: + return { + "tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, + "enable_context_window_escalation": True, + **overrides, + } class TestContextWindowEscalation: @@ -13938,7 +13945,7 @@ class TestContextWindowEscalation: router = ComplexityRouter( model_name="test-router", litellm_router_instance=_windowed_router(_SMALL, ("mid-model", "openai/gpt-4o-mini", 200000), _BIG), - complexity_router_config={"tiers": {"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}}, + complexity_router_config=_tier_config(tiers={"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -13975,7 +13982,7 @@ class TestContextWindowEscalation: }, ] ), - complexity_router_config={"tiers": {"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}}, + complexity_router_config=_tier_config(tiers={"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -14026,7 +14033,7 @@ class TestContextWindowEscalation: router = ComplexityRouter( model_name="test-router", litellm_router_instance=_windowed_router(*deployments), - complexity_router_config={"tiers": tiers}, + complexity_router_config=_tier_config(tiers=tiers), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -14035,19 +14042,37 @@ class TestContextWindowEscalation: assert result.model == expected_model @pytest.mark.asyncio - async def test_the_disabled_gate_dispatches_on_complexity_alone(self): - """The escape hatch: enable_context_window_escalation false restores today's behavior.""" - router = ComplexityRouter( + @pytest.mark.parametrize("enabled", (None, False, True), ids=("omitted", "disabled", "enabled")) + @pytest.mark.parametrize("serialized", (False, True), ids=("config", "http-json")) + async def test_context_window_escalation_requires_opt_in(self, enabled: bool | None, serialized: bool) -> None: + setting: Final = ( + MappingProxyType({"enable_context_window_escalation": enabled}) + if enabled is not None + else MappingProxyType({}) + ) + raw_config: Final = RequestComplexityRouterConfig.model_validate( + MappingProxyType( + {"tiers": MappingProxyType({"SIMPLE": "small-model", "COMPLEX": "big-model"}), **setting} + ) + ) + config: Final = ( + RequestComplexityRouterConfig.model_validate_json(raw_config.model_dump_json()) + if serialized + else raw_config + ) + router: Final = ComplexityRouter( model_name="test-router", litellm_router_instance=_windowed_router(_SMALL, _BIG), - complexity_router_config=_tier_config(enable_context_window_escalation=False), + complexity_router_config=config.model_dump(exclude_unset=not serialized, exclude_none=True), ) - result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS + ) assert result is not None - assert result.model == "small-model" - assert "context_escalated" not in result.routing_decision + assert result.model == ("big-model" if enabled else "small-model") + assert result.routing_decision.get("context_escalated", False) is (enabled is True) @pytest.mark.asyncio async def test_out_of_band_system_and_tools_count_against_the_window(self): @@ -14156,7 +14181,7 @@ class TestContextWindowEscalation: }, ] ), - complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["small-model", "mid-model"]}}, + complexity_router_config=_tier_config(adaptive=True, tiers={"SIMPLE": ["small-model", "mid-model"]}), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -14186,7 +14211,7 @@ class TestContextWindowEscalation: }, ] ), - complexity_router_config={"tiers": {"SIMPLE": "cop-pool", "COMPLEX": "big-model"}}, + complexity_router_config=_tier_config(tiers={"SIMPLE": "cop-pool", "COMPLEX": "big-model"}), ) real_get_llm_provider = litellm.get_llm_provider copilot_resolutions: List = [] @@ -14219,7 +14244,7 @@ class TestContextWindowEscalation: "model_name": "smart-router", "litellm_params": { "model": "auto_router/complexity_router", - "complexity_router_config": {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}}, + "complexity_router_config": _tier_config(), }, }, { @@ -15139,7 +15164,12 @@ class TestHealthFallbackDispatch: ) -> None: from litellm.types.router import RouterRateLimitError - router: Final = self._router(config={"tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}}) + router: Final = self._router( + config={ + "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}, + "enable_context_window_escalation": True, + } + ) router.add_deployment( Deployment( model_name="large", @@ -15216,7 +15246,13 @@ class TestHealthFallbackDispatch: @pytest.mark.asyncio @pytest.mark.parametrize("default_fits", [True, False]) async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None: - router: Final = self._router(config={"modality_routing": True, "tiers": {"SIMPLE": "primary"}}) + router: Final = self._router( + config={ + "modality_routing": True, + "tiers": {"SIMPLE": "primary"}, + "enable_context_window_escalation": True, + } + ) for deployment in router.model_list: deployment["model_info"]["supports_vision"] = deployment["model_name"] == "fallback" deployment["model_info"]["max_input_tokens"] = 10000 if default_fits else 10 diff --git a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx index c0a65076d20..ad09efd8059 100644 --- a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx @@ -7,7 +7,7 @@ const ContextWindowEscalationConfig: React.FC<{ value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; }> = ({ value, onChange }) => { - const enabled = value.enable_context_window_escalation ?? true; + const enabled = value.enable_context_window_escalation ?? false; // A number input renders Number("0.") as "0", so a decimal cannot be typed without a local draft. const [bufferDraft, setBufferDraft] = React.useState(null); const commitBuffer = (raw: string) => { @@ -32,7 +32,8 @@ const ContextWindowEscalationConfig: React.FC<{ When a prompt provably cannot fit the decided tier's context windows, route it to the lowest tier whose - window holds it instead of letting the provider reject it. Off means requests dispatch on complexity alone. + window holds it instead of letting the provider reject it. Disabled by default. Off means requests dispatch on + complexity alone. {enabled && (
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index fdff7ad001c..242893e2ae8 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -744,7 +744,7 @@ describe("AddAutoRouterTab", () => { ); }); - it("carries a context-window escalation opt-out through to the create payload", async () => { + it("starts context-window escalation disabled and carries an explicit opt-in to the create payload", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); @@ -754,14 +754,15 @@ describe("AddAutoRouterTab", () => { expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Context Window Escalation")); const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); - expect(toggle).toBeChecked(); + expect(toggle).not.toBeChecked(); + expect(screen.queryByLabelText("Window fit buffer")).not.toBeInTheDocument(); await user.click(toggle); await user.click(screen.getByRole("button", { name: /add auto router/i })); await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ - enable_context_window_escalation: false, + enable_context_window_escalation: true, }); }); @@ -774,6 +775,7 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Context Window Escalation")); + await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); const buffer = await screen.findByLabelText("Window fit buffer"); fireEvent.change(buffer, { target: { value: "1.5" } }); fireEvent.blur(buffer, { target: { value: "1.5" } }); @@ -783,7 +785,7 @@ describe("AddAutoRouterTab", () => { await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; expect(config).toMatchObject({ context_window_escalation_buffer: 1 }); - expect(config).not.toHaveProperty("enable_context_window_escalation"); + expect(config).toHaveProperty("enable_context_window_escalation", true); }); it("clearing the buffer removes it from the payload so the router tracks the backend default", async () => { @@ -795,6 +797,7 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Context Window Escalation")); + await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); const buffer = await screen.findByLabelText("Window fit buffer"); fireEvent.change(buffer, { target: { value: "0.8" } }); fireEvent.blur(buffer, { target: { value: "0.8" } }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 6db2b8213d1..6d458d61797 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -168,7 +168,7 @@ describe("buildComplexityRouterConfig", () => { }); it.each(["capability", "llm_v2", "heuristic"] as const)( - "disables the removed overrides only for forecast creates: %s", + "preserves explicit context-window opt-in beside forecast restrictions: %s", (classifierType) => { const forecast = classifierType !== "heuristic"; const params = { @@ -180,14 +180,10 @@ describe("buildComplexityRouterConfig", () => { }; const config = buildComplexityRouterConfig(params); expect(config.adaptive).toBe(!forecast); - expect(config.enable_context_window_escalation).toBe(!forecast); + expect(config.enable_context_window_escalation).toBe(true); + expect(config.context_window_escalation_buffer).toBe(0.9); expect(config.escalation_keywords).toEqual(forecast ? [] : ["LITELLM ESCALATE"]); - for (const key of [ - "adaptive_weights", - "adaptive_eligible", - "tier_distance_penalty", - "context_window_escalation_buffer", - ]) { + for (const key of ["adaptive_weights", "adaptive_eligible", "tier_distance_penalty"]) { expect(Object.hasOwn(config, key)).toBe(!forecast); } if (forecast) { @@ -226,13 +222,14 @@ describe("buildComplexityRouterConfig", () => { expect(config).toEqual(expected); }); - it("carries an explicit context-window escalation opt-out and buffer, false included", () => { + it.each([undefined, false, true])("preserves the context-window escalation setting: %s", (enabled) => { const config = buildComplexityRouterConfig({ ...baseParams, - enableContextWindowEscalation: false, + enableContextWindowEscalation: enabled, contextWindowEscalationBuffer: 0.9, }); - expect(config.enable_context_window_escalation).toBe(false); + expect(config.enable_context_window_escalation).toBe(enabled); + expect(Object.hasOwn(config, "enable_context_window_escalation")).toBe(enabled !== undefined); expect(config.context_window_escalation_buffer).toBe(0.9); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index ca46e171970..9b4f1980f62 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -736,6 +736,7 @@ export const buildComplexityRouterConfig = ({ }; const effectiveType = effectiveClassifierType({ custom_tier_set: customTierSet, classifier_type: classifierType }); const forecast = isForecastClassifier(effectiveType); + const preserveContextWindowBuffer = !forecast || enableContextWindowEscalation === true; const cleanList = (items: string[] | undefined): string[] | undefined => { const cleaned = (items ?? []).map((item) => item.trim()).filter(Boolean); return cleaned.length > 0 ? cleaned : undefined; @@ -815,11 +816,10 @@ export const buildComplexityRouterConfig = ({ adaptive_eligible: adaptiveEligible, }), ...(returnRawModelName && { return_raw_model_name: true }), - // Omission enables the backend default, so hidden forecast controls need an explicit opt-out. ...((forecast || enableContextWindowEscalation !== undefined) && { - enable_context_window_escalation: forecast ? false : enableContextWindowEscalation, + enable_context_window_escalation: enableContextWindowEscalation ?? false, }), - ...(!forecast && + ...(preserveContextWindowBuffer && contextWindowEscalationBuffer !== undefined && { context_window_escalation_buffer: contextWindowEscalationBuffer, }), diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index eaa6595d8a3..162414b7814 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -189,14 +189,10 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, keywordState); const forecast = classifier_type !== "heuristic"; expect(saved.adaptive).toBe(!forecast); - expect(saved.enable_context_window_escalation).toBe(!forecast); + expect(saved.enable_context_window_escalation).toBe(true); + expect(saved.context_window_escalation_buffer).toBe(0.9); expect(saved.escalation_keywords).toEqual(forecast ? [] : stored.escalation_keywords); - for (const key of [ - "adaptive_weights", - "adaptive_eligible", - "tier_distance_penalty", - "context_window_escalation_buffer", - ]) { + for (const key of ["adaptive_weights", "adaptive_eligible", "tier_distance_penalty"]) { expect(Object.hasOwn(saved, key)).toBe(!forecast); } expect(saved.keyword_tier_rules).toEqual(STORED.keyword_tier_rules); @@ -208,6 +204,19 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { }, ); + it.each([undefined, false, true])("preserves stored context-window escalation on save: %s", (enabled) => { + const stored = { + ...STORED, + ...(enabled !== undefined && { enable_context_window_escalation: enabled }), + }; + const value = hydrateComplexityRouterConfig(stored, undefined); + const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, hydratedState); + const serialized: typeof saved = JSON.parse(JSON.stringify(saved)); + expect(value.enable_context_window_escalation).toBe(enabled); + expect(serialized.enable_context_window_escalation).toBe(enabled); + expect(Object.hasOwn(serialized, "enable_context_window_escalation")).toBe(enabled !== undefined); + }); + it("round-trips an untouched edit without changing any keyword-matching value", () => { // Opening the modal hydrates state from STORED; saving with nothing changed must be a // no-op. These keys are now MANAGED, so a hydration bug silently wipes them. diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index e1d9e6ab5ac..87efb2b1d5e 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -746,7 +746,7 @@ describe("autorouter_presets", () => { expect(prefill.escalationKeywords).toEqual([]); }); - it("carries a preset's context-window escalation opt-out and buffer through the prefill", () => { + it.each([undefined, false, true])("preserves a preset's context-window escalation setting: %s", (enabled) => { const prefill = buildPresetPrefill( { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -754,12 +754,12 @@ describe("autorouter_presets", () => { classification_mode: "every_request", session_affinity: false, deployment_affinity: true, - enable_context_window_escalation: false, + enable_context_window_escalation: enabled, context_window_escalation_buffer: 0.9, }, groupsOnly(["gpt-5-nano"]), ); - expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(false); + expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(enabled); expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 500cf35542e..58b5cedc2cf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36885,8 +36885,8 @@ export interface components { embedding_model?: string | null; /** * Enable Context Window Escalation - * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Set false to dispatch on complexity alone, as before. - * @default true + * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Disabled by default: omit or set false to dispatch on complexity alone; set true to enable context-window escalation. + * @default false */ enable_context_window_escalation: boolean; /**