diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index cf7bde93360..67237f0a898 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -29,10 +29,12 @@ The weighted sum is mapped to tiers using configurable boundaries: | Tier | Score Range | Boundary key below it | Typical Use | |------|-------------|-----------------------|-------------| -| SIMPLE | < 0.15 | - | Basic questions, greetings | -| MEDIUM | 0.15 - 0.35 | `simple_medium` | Standard queries | -| COMPLEX | 0.35 - 0.60 | `medium_complex` | Technical, multi-part requests | -| REASONING | > 0.60 | `complex_reasoning` | Chain-of-thought, analysis | +| SIMPLE | < 0.10 | - | Basic questions, greetings | +| MEDIUM | 0.10 - 0.25 | `simple_medium` | Standard queries | +| COMPLEX | 0.25 - 0.50 | `medium_complex` | Technical, multi-part requests | +| REASONING | >= 0.50 | `complex_reasoning` | Chain-of-thought, analysis | + +The three boundaries must ascend, and any key you leave out is filled from the shipped default, so setting one boundary without the others can put them out of order. A set that decreases would strand the tier between the inverted pair and silently route its traffic to a costlier one, so it is rejected at config load with a message naming the resolved values. Set every boundary you need to move, not just one. Tier names are defaults you can rename with [`tier_labels`](#renaming-the-tiers). The three `tier_boundaries` keys are named after those defaults but they are scorer knobs, not tiers: each one names the gap between two rungs and is persisted by name on every routing decision, so they stay `simple_medium` / `medium_complex` / `complex_reasoning` no matter what you call the tiers. The column above tells a renamed deployment which knob it is turning. @@ -120,9 +122,9 @@ model_list: # Tier boundaries (normalized scores) tier_boundaries: - simple_medium: 0.15 - medium_complex: 0.35 - complex_reasoning: 0.60 + simple_medium: 0.10 + medium_complex: 0.25 + complex_reasoning: 0.50 # Token count thresholds token_thresholds: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 087c1f7278d..d5aa989f763 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -55,6 +55,7 @@ from .config import ( ClassificationRubric, ComplexityRouterConfig, ComplexityTier, + resolve_tier_boundaries, ) if TYPE_CHECKING: @@ -1149,11 +1150,11 @@ class ComplexityRouter(CustomLogger): Shared by score-to-tier mapping and the per-request routing decision snapshot, so a logged decision always reflects the boundaries that actually applied. """ - boundaries: Final = self.config.tier_boundaries + resolved: Final = resolve_tier_boundaries(self.config.tier_boundaries) return StandardLoggingRoutingDecisionTierBoundaries( - simple_medium=boundaries.get("simple_medium", 0.15), - medium_complex=boundaries.get("medium_complex", 0.35), - complex_reasoning=boundaries.get("complex_reasoning", 0.60), + simple_medium=resolved["simple_medium"], + medium_complex=resolved["medium_complex"], + complex_reasoning=resolved["complex_reasoning"], ) def _build_routing_decision( diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 9907407d84d..92b0581a504 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -366,12 +366,21 @@ DEFAULT_DIMENSION_WEIGHTS: Final[dict[str, float]] = { # ─── Default Tier Boundaries ─── DEFAULT_TIER_BOUNDARIES: Final[dict[str, float]] = { - "simple_medium": 0.15, # Lower threshold to catch more MEDIUM cases - "medium_complex": 0.35, # Lower threshold to catch technical COMPLEX cases - "complex_reasoning": 0.60, # Reasoning tier reserved for explicit reasoning markers + "simple_medium": 0.10, + "medium_complex": 0.25, + "complex_reasoning": 0.50, } +def resolve_tier_boundaries(boundaries: Mapping[str, float]) -> Mapping[str, float]: + """The three boundaries in effect, with the shipped default filling in each key the config omits. + + The single place omitted keys are filled, so a validator and the scorer cannot disagree about + what a partially specified tier_boundaries actually means. + """ + return MappingProxyType({key: boundaries.get(key, default) for key, default in DEFAULT_TIER_BOUNDARIES.items()}) + + # ─── Default Token Thresholds ─── DEFAULT_TOKEN_THRESHOLDS: Final[dict[str, int]] = { @@ -1122,6 +1131,36 @@ class ComplexityRouterConfig(BaseModel): self.tiers = normalized return self + @model_validator(mode="after") + def _validate_tier_boundaries_ascend(self) -> "ComplexityRouterConfig": + # The score-to-tier mapping is a sequential comparison chain, so a boundary below the one under it + # asks for a tier starting above the tier above it, which no score satisfies, and the stranded tier's + # traffic silently lands on a costlier one. Equal boundaries pass: an empty band is coherent. + # Resolved, not raw, because filling an omitted key from a shipped default is the usual way an + # operator arrives here without having written anything out of order. + resolved: Final = resolve_tier_boundaries(self.tier_boundaries) + simple_medium, medium_complex, complex_reasoning = ( + resolved["simple_medium"], + resolved["medium_complex"], + resolved["complex_reasoning"], + ) + if simple_medium <= medium_complex <= complex_reasoning: + return self + stranded: Final = tuple( + tier + for tier, inverted in ( + ("MEDIUM", simple_medium > medium_complex), + ("COMPLEX", medium_complex > complex_reasoning), + ) + if inverted + ) + raise ValueError( + f"tier_boundaries must ascend, but resolve to simple_medium={simple_medium}, " + f"medium_complex={medium_complex}, complex_reasoning={complex_reasoning}, leaving " + f"{' and '.join(stranded)} unreachable. Boundaries you omit are filled from the shipped " + f"defaults {DEFAULT_TIER_BOUNDARIES}, so set every boundary you need to move, not just one." + ) + @model_validator(mode="after") def _validate_semantic_matching(self) -> "ComplexityRouterConfig": if not self.semantic_keyword_matching: diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index a29b4d03bc5..b185f1e47d4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -34,6 +34,7 @@ from litellm.router_strategy.complexity_router.config import ( DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, DEFAULT_COMPLEXITY_CONFIG, DEFAULT_TECHNICAL_KEYWORDS, + DEFAULT_TIER_BOUNDARIES, ClassifierLLMConfig, ComplexityRouterConfig, ComplexityTier, @@ -293,19 +294,25 @@ class TestReasoningMarkerScoring: assert tier == ComplexityTier.REASONING def test_floor_defaults_to_simple_medium_and_follows_it(self, mock_router_instance, basic_config): - """Unset tracks simple_medium, so moving that boundary moves the floor with it.""" + """Unset tracks simple_medium, so moving that boundary moves the floor with it. + + Both arms carry the fixture's other two boundaries unchanged: simple_medium is the only + variable, and boundaries must ascend, so the pair above it cannot be left to fill from + shipped defaults that sit below the value under test. + """ prompt = ( "Give me the pros and cons, step by step, of moving our checkout service to an event-driven architecture." ) + boundaries = basic_config["tier_boundaries"] low = ComplexityRouter( model_name="test-complexity-router", litellm_router_instance=mock_router_instance, - complexity_router_config={**basic_config, "tier_boundaries": {"simple_medium": 0.20}}, + complexity_router_config={**basic_config, "tier_boundaries": {**boundaries, "simple_medium": 0.20}}, ) high = ComplexityRouter( model_name="test-complexity-router", litellm_router_instance=mock_router_instance, - complexity_router_config={**basic_config, "tier_boundaries": {"simple_medium": 0.30}}, + complexity_router_config={**basic_config, "tier_boundaries": {**boundaries, "simple_medium": 0.30}}, ) assert low._effective_reasoning_override_min_score() == 0.20 assert high._effective_reasoning_override_min_score() == 0.30 @@ -599,6 +606,91 @@ class TestPreRoutingHook: assert result.model == "o1-preview" # REASONING tier model +class TestEffectiveTierBoundaries: + """Test how configured boundaries resolve against the shipped defaults.""" + + def test_unconfigured_boundaries_resolve_to_the_shipped_defaults(self, mock_router_instance): + """A router with no tier_boundaries runs on DEFAULT_TIER_BOUNDARIES, not on stale literals.""" + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={"tiers": {"SIMPLE": "a", "MEDIUM": "b", "COMPLEX": "c", "REASONING": "d"}}, + ) + assert dict(router._effective_tier_boundaries()) == DEFAULT_TIER_BOUNDARIES + + def test_partial_boundaries_fill_missing_keys_from_the_defaults(self, mock_router_instance): + """Overriding one boundary must not strand the other two on a second, drifting copy of the defaults.""" + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "a", "MEDIUM": "b", "COMPLEX": "c", "REASONING": "d"}, + "tier_boundaries": {"simple_medium": 0.2}, + }, + ) + assert dict(router._effective_tier_boundaries()) == { + "simple_medium": 0.2, + "medium_complex": DEFAULT_TIER_BOUNDARIES["medium_complex"], + "complex_reasoning": DEFAULT_TIER_BOUNDARIES["complex_reasoning"], + } + + def test_omitting_a_boundary_below_one_that_is_set_is_rejected(self, mock_router_instance): + """The trap this guards: one boundary set high, the rest filled from lower shipped defaults.""" + with pytest.raises(ValidationError, match="MEDIUM unreachable"): + ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "a", "MEDIUM": "b", "COMPLEX": "c", "REASONING": "d"}, + "tier_boundaries": {"simple_medium": 0.30}, + }, + ) + + def test_fully_specified_decreasing_boundaries_are_rejected(self, mock_router_instance): + """An operator can also strand a tier without omitting anything.""" + with pytest.raises(ValidationError, match="COMPLEX unreachable"): + ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "a", "MEDIUM": "b", "COMPLEX": "c", "REASONING": "d"}, + "tier_boundaries": {"simple_medium": 0.10, "medium_complex": 0.60, "complex_reasoning": 0.50}, + }, + ) + + def test_equal_boundaries_are_accepted_and_close_the_band(self, mock_router_instance): + """Equal boundaries are accepted, and they leave the tier between them unreachable. + + With simple_medium == medium_complex, MEDIUM's band is zero width: the strict `<` below it + already claims every lower score for SIMPLE, and every score from that value up falls + through to COMPLEX. The validator still allows it because a non-decreasing set is coherent, + it just describes an empty band, where a decreasing pair asks for a tier that starts above + the tier above it and no score can satisfy that. + """ + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "a", "MEDIUM": "b", "COMPLEX": "c", "REASONING": "d"}, + "tier_boundaries": {"simple_medium": 0.25, "medium_complex": 0.25, "complex_reasoning": 0.50}, + }, + ) + assert dict(router._effective_tier_boundaries()) == { + "simple_medium": 0.25, + "medium_complex": 0.25, + "complex_reasoning": 0.50, + } + + def test_defaults_stay_ordered_and_within_the_scoring_range(self): + """The tiers only all remain reachable while the boundaries ascend.""" + simple_medium, medium_complex, complex_reasoning = ( + DEFAULT_TIER_BOUNDARIES["simple_medium"], + DEFAULT_TIER_BOUNDARIES["medium_complex"], + DEFAULT_TIER_BOUNDARIES["complex_reasoning"], + ) + assert 0 < simple_medium < medium_complex < complex_reasoning < 1 + + class TestConfigOverrides: """Test configuration override functionality.""" @@ -5116,7 +5208,7 @@ class TestRoutingDecisionContents: assert isinstance(decision["score"], float) assert any("short" in signal for signal in decision["signals"]) # The snapshot must reflect the CONFIGURED boundaries (the fixture overrides the - # 0.15/0.35/0.60 defaults), so a logged row stays truthful after config edits. + # shipped defaults), so a logged row stays truthful after config edits. assert decision["tier_boundaries"] == { "simple_medium": 0.25, "medium_complex": 0.50, diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index 4e87652f8b3..1f3b7975a92 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -405,7 +405,7 @@ class TestPreRoutingHook: request in the session, and carries no signal about how requests differ. Before the fix this system prompt alone supplied 5 codePresence + 2 technicalTerms keyword matches, saturating both dimensions and crossing the default - simple_medium boundary (0.15) purely from harness text, independent of the ask.""" + simple_medium boundary purely from harness text, independent of the ask.""" agent_system_prompt = ( "You are Claude Code, Anthropic's official CLI for Claude.\n" "You are an interactive agent that helps users with software engineering tasks.\n\n" diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 45a967c0537..f75aff65bd8 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -74,10 +74,10 @@ describe("ComplexityRouterConfig", () => { it("should show score thresholds in the classification section", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByText(/Score < 0.15/)).toBeInTheDocument(); - expect(screen.getByText(/Score 0.15 - 0.35/)).toBeInTheDocument(); - expect(screen.getByText(/Score 0.35 - 0.60/)).toBeInTheDocument(); - expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); + expect(screen.getByText(/Score < 0.10/)).toBeInTheDocument(); + expect(screen.getByText(/Score 0.10 - 0.25/)).toBeInTheDocument(); + expect(screen.getByText(/Score 0.25 - 0.50/)).toBeInTheDocument(); + expect(screen.getByText(/Score > 0.50/)).toBeInTheDocument(); }); it("should default to heuristic and hide classifier model/timeout fields", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx index 9dccd767e49..bc2c155711b 100644 --- a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx @@ -41,7 +41,7 @@ describe("HeuristicScoringConfig", () => { it("prefills the shipped defaults", async () => { await render(BASE); - expect(screen.getByLabelText("Simple to Medium")).toHaveValue("0.15"); + expect(screen.getByLabelText("Simple to Medium")).toHaveValue("0.1"); expect(screen.getByLabelText("Long above")).toHaveValue("400"); expect(screen.getByTestId("dimension-weight-total")).toHaveTextContent("total 1.00"); }); @@ -57,8 +57,8 @@ describe("HeuristicScoringConfig", () => { expect((onChange.mock.calls.at(-1)?.[0] as ComplexityRouterConfigValue).tier_boundaries).toEqual({ simple_medium: 0.22, - medium_complex: 0.35, - complex_reasoning: 0.6, + medium_complex: 0.25, + complex_reasoning: 0.5, }); }); @@ -184,7 +184,7 @@ describe("ClassificationMethodConfig scorer gating", () => { expect(screen.getByText(/Score < 0.22/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.44 - 0.66/)).toBeInTheDocument(); - expect(screen.queryByText(/0.15/)).not.toBeInTheDocument(); + expect(screen.queryByText(/0.10/)).not.toBeInTheDocument(); }); it("states the configured override floor in the reasoning-marker aside, not the boundary", () => { @@ -196,7 +196,7 @@ describe("ClassificationMethodConfig scorer gating", () => { it("falls back to the Simple to Medium boundary when no override floor is set", () => { renderWithProviders(); - expect(screen.getByText(/2\+ reasoning markers with a score of at least 0\.15/)).toBeInTheDocument(); + expect(screen.getByText(/2\+ reasoning markers with a score of at least 0\.10/)).toBeInTheDocument(); }); it("renders a row for every scored dimension", async () => { diff --git a/ui/litellm-dashboard/tests/mocks/complexityScorerDefaults.ts b/ui/litellm-dashboard/tests/mocks/complexityScorerDefaults.ts index 3eb2d781213..507d8d65f72 100644 --- a/ui/litellm-dashboard/tests/mocks/complexityScorerDefaults.ts +++ b/ui/litellm-dashboard/tests/mocks/complexityScorerDefaults.ts @@ -10,7 +10,7 @@ import type { ComplexityScorerDefaults } from "@/components/networking"; * which is how the failure path is covered. */ export const SHIPPED_SCORER_DEFAULTS: ComplexityScorerDefaults = { - tier_boundaries: { simple_medium: 0.15, medium_complex: 0.35, complex_reasoning: 0.6 }, + tier_boundaries: { simple_medium: 0.1, medium_complex: 0.25, complex_reasoning: 0.5 }, token_thresholds: { simple: 15, complex: 400 }, dimension_weights: { codePresence: 0.3,