diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a3d6ccbd437..fb158372791 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1776,7 +1776,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, context_escalation_original_tier: ComplexityTier | str | None = None, - heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None, + previous_decision: StandardLoggingRoutingDecision | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1836,8 +1836,15 @@ class ComplexityRouter(CustomLogger): masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) if isinstance(masked_tier_litellm_params, Mapping): decision["tier_litellm_params"] = masked_tier_litellm_params - return ( - decision if heuristic_v2_forecast is None else {**decision, "heuristic_v2_forecast": heuristic_v2_forecast} + if previous_decision is None: + return decision + forecast_fields: Final = { + field: value + for field, value in previous_decision.items() + if field.startswith("classifier_") or field == "heuristic_v2_forecast" + } + return cast( # cast-ok: retaining optional keys from a typed decision preserves their declared values + StandardLoggingRoutingDecision, {**forecast_fields, **decision} ) async def aclassify( @@ -3569,7 +3576,7 @@ class ComplexityRouter(CustomLogger): context_escalation_original_tier=( decision.get("context_escalation_original_tier") if decision is not None else None ), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast") if decision is not None else None, + previous_decision=decision, ) from litellm.types.router import PreRoutingHookResponse as HookResponse @@ -3749,7 +3756,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), + previous_decision=decision, ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict @@ -3794,7 +3801,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(None, default_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), + previous_decision=decision, ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index ecd25ff654f..5516dcb0162 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -78,6 +78,7 @@ from litellm.router_strategy.complexity_router.jev_classifier import ( JevSystemOneResponse, JevUsage, ) +from litellm.router_strategy.complexity_router.llm_v2 import LLM_V2_PROMPT_VERSION from litellm.router_strategy.complexity_router.tier_predictor import ( TierGlobalStatistic, TrainedTierArtifact, @@ -3207,6 +3208,41 @@ class TestCapabilityClassifier: ) assert response.model == "capable-model" assert response.routing_decision["cause"] == "capability_classifier_fallback" + assert "classifier_p_solve" not in response.routing_decision + assert "classifier_threshold" not in response.routing_decision + + @pytest.mark.asyncio + @pytest.mark.parametrize("bypass", ("literal_keyword_match", "session_affinity_pin", "housekeeping")) + async def test_bypasses_do_not_reuse_the_previous_capability_forecast( + self, + mock_router_instance: MagicMock, + bypass: Literal["literal_keyword_match", "session_affinity_pin", "housekeeping"], + ) -> None: + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response(_capability_reply(p_solve=0.8))) + mock_router_instance.cache = DualCache() + router: Final = self._router( + mock_router_instance, + session_affinity=bypass == "session_affinity_pin", + keyword_tier_rules=[{"keywords": ["quick lookup"], "tier": "SIMPLE"}], + ) + original: Final = await router.async_pre_routing_hook( + model="capability-router", + request_kwargs={"metadata": {"session_id": "forecast-bypass"}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + result: Final = await router.async_pre_routing_hook( + model="capability-router", + request_kwargs={"metadata": {"session_id": "forecast-bypass"}}, + messages=[{"role": "user", "content": TITLE_ASK if bypass == "housekeeping" else "quick lookup"}], + ) + + assert original is not None and original.routing_decision is not None + assert original.routing_decision["classifier_p_solve"] == 0.8 + assert result is not None and result.routing_decision is not None + assert result.routing_decision["cause"] == bypass + assert "classifier_p_solve" not in result.routing_decision + assert "classifier_threshold" not in result.routing_decision + mock_router_instance.acompletion.assert_awaited_once() CUSTOM_TIER_LABELS: Dict[str, str] = { @@ -14557,6 +14593,121 @@ class TestModalityRouting: @pytest.mark.usefixtures("local_model_cost_map") class TestHealthFallbackDispatch: + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier", ("capability", "llm_v2")) + @pytest.mark.parametrize("calibrated", (False, True), ids=("raw", "calibrated")) + @pytest.mark.parametrize("rewrite", ("modality_escalation", "health_failover", "health_default_fallback")) + async def test_classifier_forecasts_survive_placement_rewrites( + self, + classifier: Literal["capability", "llm_v2"], + calibrated: bool, + rewrite: Literal["modality_escalation", "health_failover", "health_default_fallback"], + ) -> None: + calibration: Final = {"slope": 0.8, "intercept": 0.1} + classifier_config: Final = ( + { + "capability_classifier_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "base_threshold": 0.0, + "threshold_step": 0.1, + **({"calibration": {"version": "test-v1", **calibration}} if calibrated else {}), + } + } + if classifier == "capability" + else { + "llm_v2_config": { + "efficient_profile": "Small coding solver", + "capable_profile": "Large coding solver", + "harness": "Repository tools", + "max_quality_gap": 0.0, + **( + { + "calibration": { + "version": "test-v1", + "prompt_version": LLM_V2_PROMPT_VERSION, + "efficient": calibration, + "capable": calibration, + } + } + if calibrated + else {} + ), + } + } + ) + router: Final = self._router( + config={ + "classifier_type": classifier, + "classifier_llm_config": {"model": "fallback", "timeout_ms": 10000}, + "tiers": {"SIMPLE": "primary", "REASONING": "peer"}, + "tier_labels": {"SIMPLE": "Entry", "REASONING": "Advanced"}, + "modality_routing": True, + **classifier_config, + } + ) + verdict: Final = ( + _capability_reply(p_solve=0.0) + if classifier == "capability" + else json.dumps( + { + "crux": "Preserve existing behavior", + "demands": {"reasoning": "routine", "scope": "localized", "specification": "clear"}, + "verification": "relevant", + "forecasts": { + "efficient": {"likely_failure": "Miss an edge case", "p_solve": 0.0}, + "capable": {"likely_failure": "Miss an edge case", "p_solve": 0.0}, + }, + } + ) + ) + judge_response: Final = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": verdict}}], + usage={"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + ) + with respx.mock(assert_all_mocked=True) as upstream: + upstream.post(host="fallback.test").respond(json=judge_response.model_dump()) + original: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + for deployment in router.model_list: + deployment["model_info"]["supports_vision"] = ( + rewrite != "modality_escalation" or deployment["model_name"] != "primary" + ) + if rewrite != "modality_escalation": + self._unavailable(router, "primary-id", "cooldown") + if rewrite == "health_default_fallback": + self._unavailable(router, "peer-id", "cooldown") + result: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=TestModalityRouting.IMAGE_MESSAGE + ) + + assert original is not None and original.routing_decision is not None + assert original.model == "primary" + assert result is not None and result.routing_decision is not None + decision: Final = result.routing_decision + assert decision["cause"] == rewrite + assert result.model == ("fallback" if rewrite == "health_default_fallback" else "peer") + expected: Final = { + field: value for field, value in original.routing_decision.items() if field.startswith("classifier_") + } + assert expected["classifier_p_solve" if classifier == "capability" else "classifier_efficient_p_solve"] == 0.0 + assert ("classifier_calibration_version" in expected) is calibrated + assert {field: value for field, value in decision.items() if field.startswith("classifier_")} == expected + if rewrite == "health_default_fallback": + assert "tier" not in decision and "tier_label" not in decision + else: + assert decision["tier"] == "REASONING" + assert decision["tier_label"] == "Advanced" + redacted: Final = Router._redact_prompt_text_if_needed( + request_kwargs={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}, + routing_decision=decision, + ) + assert "classifier_crux" not in redacted and "signals" not in redacted + assert {field: value for field, value in redacted.items() if field.startswith("classifier_")} == { + field: value for field, value in expected.items() if field != "classifier_crux" + } + @pytest.mark.asyncio @pytest.mark.parametrize("peer", (True, False), ids=("peer_failover", "default_fallback")) async def test_health_rewrites_preserve_the_original_heuristic_v2_forecast(self, peer: bool) -> None: diff --git a/tests/test_litellm/router_strategy/test_llm_v2.py b/tests/test_litellm/router_strategy/test_llm_v2.py index 27d31cbe640..6fb6df3265d 100644 --- a/tests/test_litellm/router_strategy/test_llm_v2.py +++ b/tests/test_litellm/router_strategy/test_llm_v2.py @@ -477,6 +477,10 @@ async def test_user_turn_mode_reuses_forecast_until_a_new_user_requirement() -> assert first.model == second.model == "efficient" assert first.routing_decision["cause"] == "llm_v2_classifier" assert first.routing_decision["classifier_cost"] == 0.001 + assert second is not None and second.routing_decision is not None + assert second.routing_decision["cause"] == "user_turn_continuation" + assert "classifier_efficient_p_solve" not in second.routing_decision + assert "classifier_capable_p_solve" not in second.routing_decision client.acompletion.assert_awaited_once() client.acompletion.return_value = _response(_verdict(0.3, 0.9).model_dump_json()) updated: Final = await router.async_pre_routing_hook( diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index 811b444a2f2..f33d7aea311 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -113,6 +113,148 @@ describe("RoutingDecisionCard", () => { }, ); + it.each(["capability_classifier", "modality_escalation"])( + "shows the recorded Capability forecast for %s", + (cause) => { + render( + , + ); + + expect(screen.getByText("Capability estimates")).toBeInTheDocument(); + expect(screen.getByText("Efficient model solve chance")).toBeInTheDocument(); + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual(["0.0%", "86.4%", "82.0%"]); + expect(screen.getByText("Raw")).toBeInTheDocument(); + expect(screen.getByText("Calibrated")).toBeInTheDocument(); + expect(screen.getByText("Threshold")).toBeInTheDocument(); + expect(screen.getByText("uncertain")).toBeInTheDocument(); + expect(screen.getByText("UNC-2")).toBeInTheDocument(); + expect(screen.getByText("calibration-1")).toBeInTheDocument(); + expect(screen.getByText("Deep")).toBeInTheDocument(); + expect(screen.queryByText("FUSE v2 estimates")).not.toBeInTheDocument(); + }, + ); + + it("omits absent Capability fields while preserving a recorded zero threshold", () => { + render( + , + ); + + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual(["25.0%", "0.0%"]); + for (const label of ["Calibrated", "Calibration", "Boundary", "Rule"]) { + expect(screen.queryByText(label)).not.toBeInTheDocument(); + } + }); + + it.each(["llm_v2_classifier", "default_fallback"])( + "shows the original calibrated FUSE v2 forecast for %s", + (cause) => { + render( + , + ); + + expect(screen.getByText("FUSE v2 estimates")).toBeInTheDocument(); + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual([ + "25.0%", + "91.0%", + "75.0%", + "80.0%", + ]); + expect(screen.getByText("Efficient (raw)")).toBeInTheDocument(); + expect(screen.getByText("Capable (raw)")).toBeInTheDocument(); + expect(screen.getByText("Efficient (calibrated)")).toBeInTheDocument(); + expect(screen.getByText("Capable (calibrated)")).toBeInTheDocument(); + expect(screen.getAllByText(/percentage points$/).map((value) => value.textContent)).toEqual([ + "5.0 percentage points", + "10.0 percentage points", + ]); + expect(screen.getByText("Applied gap")).toBeInTheDocument(); + expect(screen.getByText("Allowed gap")).toBeInTheDocument(); + expect(screen.getByText("calibration-2")).toBeInTheDocument(); + expect(screen.getByText("llm-v2:verification=tests")).toBeInTheDocument(); + expect(screen.getByText("fallback-model")).toBeInTheDocument(); + expect(screen.queryByText("Capability estimates")).not.toBeInTheDocument(); + }, + ); + + it("uses raw FUSE v2 probabilities without calibration and preserves negative and zero gaps", () => { + render( + , + ); + + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual(["50.0%", "0.0%"]); + expect(screen.getAllByText(/percentage points$/).map((value) => value.textContent)).toEqual([ + "-50.0 percentage points", + "0.0 percentage points", + ]); + expect(screen.queryByText(/calibrated|Calibration/)).not.toBeInTheDocument(); + }); + + it.each([ + { classifier_efficient_p_solve: 0, classifier_max_quality_gap: 0.2 }, + { + classifier_efficient_p_solve: 0.4, + classifier_capable_p_solve: 0.9, + classifier_calibrated_efficient_p_solve: 0, + classifier_calibration_version: "partial-calibration", + classifier_max_quality_gap: 0.2, + }, + ])("shows partial FUSE v2 estimates without inventing an applied gap: %j", (fields) => { + render(); + + expect(screen.getByText("FUSE v2 estimates")).toBeInTheDocument(); + expect(screen.getByText("0.0%")).toBeInTheDocument(); + expect(screen.getByText("20.0 percentage points")).toBeInTheDocument(); + expect(screen.queryByText("Applied gap")).not.toBeInTheDocument(); + expect(screen.queryByText("Capable (calibrated)")).not.toBeInTheDocument(); + }); + + it.each([ + ["capability_classifier", "Capability"], + ["llm_v2_classifier", "FUSE v2"], + ["capability_classifier_fallback", "Capable tier, Capability classifier failed"], + ["llm_v2_fallback", "Capable tier, FUSE v2 classifier failed"], + ["session_affinity_pin", "Pinned to session"], + ])("labels %s without inventing a missing forecast", (cause, label) => { + render(); + + expect(screen.getByText(label)).toBeInTheDocument(); + expect(screen.getByText("MEDIUM")).toBeInTheDocument(); + expect(screen.queryByText(/estimates/)).not.toBeInTheDocument(); + }); + it("uses the persisted boundary snapshot, not today's defaults", () => { // Same score, boundaries the operator had configured lower: it lands in a // different band, and the card must say so. diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index 78dc25119d7..aeb6397daf8 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -24,6 +24,17 @@ export interface RoutingDecision { matched_keyword?: string; escalation_keyword?: string; classifier_model?: string; + classifier_p_solve?: number; + classifier_calibrated_p_solve?: number; + classifier_threshold?: number; + classifier_capability_boundary?: string; + classifier_primary_rule?: string; + classifier_calibration_version?: string; + classifier_efficient_p_solve?: number; + classifier_capable_p_solve?: number; + classifier_calibrated_efficient_p_solve?: number; + classifier_calibrated_capable_p_solve?: number; + classifier_max_quality_gap?: number; escalated?: boolean; tier_boundaries?: RoutingDecisionTierBoundaries; reasoning_override_min_score?: number; @@ -91,6 +102,10 @@ function describeReasoningOverride(tierLabel: string | undefined, floor: number const CONSTANT_CAUSE_LABELS: Record = { heuristic_scorer: "Heuristic scorer", heuristic_v2: "Heuristic v2", + capability_classifier: "Capability", + capability_classifier_fallback: "Capable tier, Capability classifier failed", + llm_v2_classifier: "FUSE v2", + llm_v2_fallback: "Capable tier, FUSE v2 classifier failed", heuristic_first_short_circuit: "Heuristic scorer, classifier skipped", hybrid_short_circuit: "Heuristic scorer, score clear of every boundary", classifier_plugin: "Custom classifier plugin", @@ -156,6 +171,71 @@ function Row({ label, children }: { label: string; children: React.ReactNode }) ); } +function PercentageRow({ label, value, unit = "%" }: { label: string; value?: number; unit?: string }) { + if (value === undefined) return null; + return ( + + {`${(value * 100).toFixed(1)}${unit}`} + + ); +} + +function CapabilityForecast({ decision }: { decision: RoutingDecision }) { + const { + classifier_p_solve: raw, + classifier_calibrated_p_solve: calibrated, + classifier_threshold: threshold, + classifier_capability_boundary: boundary, + classifier_primary_rule: rule, + classifier_calibration_version: version, + } = decision; + if ([raw, calibrated, threshold, boundary, rule].every((value) => value === undefined)) return null; + + return ( +
+
Capability estimates
+
Efficient model solve chance
+ + + + {boundary && {boundary}} + {rule && {rule}} + {version && {version}} +
+ ); +} + +function FuseV2Forecast({ decision }: { decision: RoutingDecision }) { + const { + classifier_efficient_p_solve: rawEfficient, + classifier_capable_p_solve: rawCapable, + classifier_calibrated_efficient_p_solve: calibratedEfficient, + classifier_calibrated_capable_p_solve: calibratedCapable, + classifier_max_quality_gap: allowedGap, + classifier_calibration_version: version, + } = decision; + if ([rawEfficient, rawCapable, calibratedEfficient, calibratedCapable, allowedGap].every((v) => v === undefined)) { + return null; + } + const isCalibrated = [calibratedEfficient, calibratedCapable, version].some((value) => value !== undefined); + const efficient = isCalibrated ? calibratedEfficient : rawEfficient; + const capable = isCalibrated ? calibratedCapable : rawCapable; + const gap = efficient !== undefined && capable !== undefined ? capable - efficient : undefined; + + return ( +
+
FUSE v2 estimates
+ + + + + + + {version && {version}} +
+ ); +} + export function RoutingDecisionCard({ decision, className, @@ -247,6 +327,9 @@ export function RoutingDecisionCard({ )} + + + {signals && signals.length > 0 && (