diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index c29f3b3a542..a3d6ccbd437 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -75,6 +75,7 @@ from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, RoutingDecisionCause, + StandardLoggingHeuristicV2Forecast, StandardLoggingRoutingDecision, StandardLoggingRoutingDecisionTierBoundaries, ) @@ -1043,6 +1044,7 @@ class ClassificationOutcome(NamedTuple): capability_forecast: CapabilityClassifierForecast | None = None llm_v2_forecast: LLMV2Decision | None = None jev_verdict: JevVerdict | None = None + heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome: @@ -1075,6 +1077,8 @@ def _with_classifier_forecast( decision: StandardLoggingRoutingDecision, outcome: ClassificationOutcome ) -> StandardLoggingRoutingDecision: """Attach validated forecasts and their applied policy to the routing decision.""" + if outcome.heuristic_v2_forecast is not None: + return {**decision, "heuristic_v2_forecast": outcome.heuristic_v2_forecast} if outcome.jev_verdict is not None: forecasted_decision: Final[StandardLoggingRoutingDecision] = { **decision, @@ -1772,6 +1776,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, context_escalation_original_tier: ComplexityTier | str | None = None, + heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1831,7 +1836,9 @@ class ComplexityRouter(CustomLogger): masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) if isinstance(masked_tier_litellm_params, Mapping): decision["tier_litellm_params"] = masked_tier_litellm_params - return decision + return ( + decision if heuristic_v2_forecast is None else {**decision, "heuristic_v2_forecast": heuristic_v2_forecast} + ) async def aclassify( self, @@ -1888,6 +1895,15 @@ class ComplexityRouter(CustomLogger): score=None, signals=(f"request-type:{request_type.value}", *probability_signals), cause="heuristic_v2", + heuristic_v2_forecast=StandardLoggingHeuristicV2Forecast( + probabilities={ + candidate.value: prediction.probabilities[index] + for index, candidate in enumerate(TIER_SEVERITY_ORDER, start=1) + }, + threshold=predictor.routing_threshold, + predicted_tier=tier.value, + request_type=request_type.value, + ), ) async def _classify_heuristic_first( @@ -3553,6 +3569,7 @@ class ComplexityRouter(CustomLogger): context_escalation_original_tier=( decision.get("context_escalation_original_tier") if decision is not None else None ), + heuristic_v2_forecast=decision.get("heuristic_v2_forecast") if decision is not None else None, ) from litellm.types.router import PreRoutingHookResponse as HookResponse @@ -3732,6 +3749,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), + heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict @@ -3776,6 +3794,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(None, default_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), + heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4d1eaf15680..b725acf6906 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2975,6 +2975,13 @@ LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN: Final[InternalCallOrigin] = "llm_as_a_judg BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll" +class StandardLoggingHeuristicV2Forecast(TypedDict): + probabilities: ReadOnly[Mapping[str, float]] + threshold: ReadOnly[float] + predicted_tier: ReadOnly[str] + request_type: ReadOnly[str] + + class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" @@ -2993,6 +3000,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_cost: float classifier_probabilities: ReadOnly[Mapping[str, float]] classifier_confidence: ReadOnly[float] + heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast] classifier_crux: str # writable-ok: added only when a capability verdict is available classifier_primary_rule: str # writable-ok: added only when a capability verdict is available classifier_capability_boundary: str # writable-ok: added only when a capability verdict is available @@ -3038,6 +3046,7 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "classifier_cost", "classifier_probabilities", "classifier_confidence", + "heuristic_v2_forecast", "classifier_primary_rule", "classifier_capability_boundary", "classifier_p_solve", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 318b96b24a7..ecd25ff654f 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -3721,7 +3721,11 @@ class TestLLMClassifier: assert outcome.score is not None @pytest.mark.asyncio - async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(self, mock_router_instance): + @pytest.mark.parametrize("redact", (False, True)) + async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier( + self, mock_router_instance: MagicMock, redact: bool, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", redact) router = ComplexityRouter( model_name="tier-router", litellm_router_instance=mock_router_instance, @@ -3754,6 +3758,21 @@ class TestLLMClassifier: "tier-probability:complex=0.892157", "tier-probability:reasoning=0.980392", ] + redacted: Final = Router._redact_prompt_text_if_needed( + request_kwargs={}, routing_decision=response.routing_decision + ) + assert ("signals" in redacted) is not redact + assert redacted["heuristic_v2_forecast"] == { + "probabilities": { + "SIMPLE": 11 / 102, + "MEDIUM": 21 / 102, + "COMPLEX": 91 / 102, + "REASONING": 100 / 102, + }, + "threshold": 0.8, + "predicted_tier": "COMPLEX", + "request_type": "general", + } def test_heuristic_v2_needs_no_classifier_model(self): config = ComplexityRouterConfig(classifier_type="heuristic_v2") @@ -8879,13 +8898,29 @@ class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape: ], ) @pytest.mark.asyncio - async def test_decision_reaches_the_spend_log_payload(self, request_kwargs, expected_bucket): + @pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_v2")) + async def test_decision_reaches_the_spend_log_payload(self, request_kwargs, expected_bucket, classifier_type): import datetime import json from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - router = Router(model_list=self.MODEL_LIST) + model_list: Final = [ + { + **row, + "litellm_params": { + **row["litellm_params"], + "complexity_router_config": { + **row["litellm_params"]["complexity_router_config"], + "classifier_type": classifier_type, + }, + }, + } + if row["model_name"] == "smart-router" + else row + for row in self.MODEL_LIST + ] + router = Router(model_list=model_list) response = await router.async_pre_routing_hook( model="smart-router", request_kwargs=request_kwargs, @@ -8915,6 +8950,15 @@ class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape: persisted = json.loads(payload["metadata"])["routing_decision"] assert persisted is not None, f"routing_decision dropped for {expected_bucket}" assert persisted["router_model_name"] == "smart-router" + if classifier_type == "heuristic_v2": + assert persisted["heuristic_v2_forecast"] == request_kwargs[expected_bucket]["routing_decision"][ + "heuristic_v2_forecast" + ] + assert set(persisted["heuristic_v2_forecast"]["probabilities"]) == { + "SIMPLE", "MEDIUM", "COMPLEX", "REASONING" + } + else: + assert "heuristic_v2_forecast" not in persisted class TestRoutingDecisionIsPerAttempt: @@ -14111,6 +14155,33 @@ class TestModalityRouting: BASE_TIERS = {"SIMPLE": "text-cheap", "MEDIUM": "vision-mid", "COMPLEX": "vision-big"} BASE_VISION = {"text-cheap": False, "vision-mid": True, "vision-big": True, "vision-default": True} + @pytest.mark.asyncio + async def test_modality_escalation_preserves_the_original_heuristic_v2_forecast( + self, mock_router_instance: MagicMock + ) -> None: + router: Final = self._router( + mock_router_instance, + { + "classifier_type": "heuristic_v2", + "heuristic_v2_artifact": _heuristic_v2_artifact(), + "tiers": {"COMPLEX": "text-cheap", "REASONING": "vision-big"}, + "modality_routing": True, + }, + self.BASE_VISION, + ) + original: Final = await router.aclassify("What color is this?") + result: Final = await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE + ) + + assert original.heuristic_v2_forecast is not None + assert result is not None and result.routing_decision is not None + assert result.model == "vision-big" + assert result.routing_decision["cause"] == "modality_escalation" + assert result.routing_decision["tier"] == "REASONING" + assert result.routing_decision["heuristic_v2_forecast"] == original.heuristic_v2_forecast + assert result.routing_decision["heuristic_v2_forecast"]["predicted_tier"] == "COMPLEX" + @staticmethod def _router(mock_router_instance, config, vision_by_model): """vision_by_model: model name -> True/False (deployment model_info) or None (undeclared).""" @@ -14486,6 +14557,69 @@ class TestModalityRouting: @pytest.mark.usefixtures("local_model_cost_map") class TestHealthFallbackDispatch: + @pytest.mark.asyncio + @pytest.mark.parametrize("peer", (True, False), ids=("peer_failover", "default_fallback")) + async def test_health_rewrites_preserve_the_original_heuristic_v2_forecast(self, peer: bool) -> None: + router: Final = self._router( + config={ + "classifier_type": "heuristic_v2", + "heuristic_v2_artifact": _heuristic_v2_artifact(), + "tiers": {"COMPLEX": ["primary", "peer"] if peer else "primary"}, + } + ) + + def select_primary(models: Sequence[str]) -> str: + return max(models) + + with patch( # test-quality-ok: force initial classification onto the failing group in a mixed tier pool + "litellm.router_strategy.complexity_router.complexity_router.random.choice", + side_effect=select_primary, + ): + original: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + self._unavailable(router, "primary-id", "cooldown") + result: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + + assert original is not None and original.routing_decision is not None + assert original.model == "primary" + assert original.routing_decision["cause"] == "heuristic_v2" + assert result is not None and result.routing_decision is not None + assert result.model == ("peer" if peer else "fallback") + assert result.routing_decision["cause"] == ("health_failover" if peer else "health_default_fallback") + assert result.routing_decision["heuristic_v2_forecast"] == original.routing_decision["heuristic_v2_forecast"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("pinned", (False, True), ids=("keyword_bypass", "session_pin")) + async def test_heuristic_v2_bypasses_have_no_fabricated_forecast(self, pinned: bool) -> None: + router: Final = self._router( + session=pinned, + config={ + "classifier_type": "heuristic_v2", + "heuristic_v2_artifact": _heuristic_v2_artifact(), + "tiers": {"COMPLEX": "primary"}, + "keyword_tier_rules": [{"keywords": ["quick lookup"], "tier": "COMPLEX"}], + }, + ) + original: Final = await router.async_pre_routing_hook( + model="health-router", + request_kwargs={"metadata": {"session_id": "v2-forecast"}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + result: Final = await router.async_pre_routing_hook( + model="health-router", + request_kwargs={"metadata": {"session_id": "v2-forecast"}}, + messages=[{"role": "user", "content": "quick lookup"}], + ) + + assert original is not None and original.routing_decision is not None + assert "heuristic_v2_forecast" in original.routing_decision + assert result is not None and result.routing_decision is not None + assert result.routing_decision["cause"] == ("session_affinity_pin" if pinned else "literal_keyword_match") + assert "heuristic_v2_forecast" not in result.routing_decision + @pytest.fixture(autouse=True) def httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index fd1777f802c..811b444a2f2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -1,8 +1,14 @@ import React from "react"; import { render, screen } from "@testing-library/react"; -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard"; +vi.mock("@/components/ui/badge", () => ({ + Badge: ({ children }: { children: React.ReactNode }) => {children}, +})); + +vi.mock("lucide-react", () => ({ Waypoints: () => null })); + const heuristic: RoutingDecision = { router_model_name: "smart-router", router_type: "complexity", @@ -14,6 +20,13 @@ const heuristic: RoutingDecision = { tier_boundaries: { simple_medium: 0.15, medium_complex: 0.35, complex_reasoning: 0.6 }, }; +const forecast = { + probabilities: { MEDIUM: 0.69321, SIMPLE: 0, COMPLEX: 0.81234, REASONING: 0.92345 }, + threshold: 0.69, + predicted_tier: "MEDIUM", + request_type: "code_generation", +}; + describe("RoutingDecisionCard", () => { it("renders nothing when the request carried no routing decision", () => { const { container } = render(); @@ -30,8 +43,76 @@ describe("RoutingDecisionCard", () => { expect(screen.getByText("(at or above 0.6, REASONING)")).toBeInTheDocument(); expect(screen.getByText("claude-sonnet")).toBeInTheDocument(); expect(screen.getByText("long (900 tokens)")).toBeInTheDocument(); + expect(screen.queryByText("Heuristic v2 estimates")).not.toBeInTheDocument(); }); + it("shows recorded v2 success estimates, including zero, when signals were redacted", () => { + render( + , + ); + + expect(screen.getByText("Heuristic v2 estimates")).toBeInTheDocument(); + expect(screen.getByText("Success by tier")).toBeInTheDocument(); + expect( + screen.getAllByText(/^(SIMPLE|MEDIUM|COMPLEX|REASONING) \d+\.\d%$/).map((badge) => badge.textContent), + ).toEqual(["SIMPLE 0.0%", "MEDIUM 69.3%", "COMPLEX 81.2%", "REASONING 92.3%"]); + expect(screen.getByText("Threshold")).toBeInTheDocument(); + expect(screen.getByText("69.0%")).toBeInTheDocument(); + expect(screen.getByText("Predicted tier")).toBeInTheDocument(); + expect(screen.getByText("MEDIUM")).toBeInTheDocument(); + expect(screen.getByText("Balanced")).toBeInTheDocument(); + expect(screen.getByText("code_generation")).toBeInTheDocument(); + expect(screen.queryByText("Score")).not.toBeInTheDocument(); + }); + + it.each([ + { threshold: 0, predicted_tier: "SIMPLE", expectedThreshold: "0.0%" }, + { threshold: 0.99, predicted_tier: "REASONING", expectedThreshold: "99.0%" }, + ])("keeps the prediction separate from an overridden tier at threshold $threshold", (scenario) => { + render( + , + ); + + expect(screen.getByText("Vision")).toBeInTheDocument(); + expect(screen.getByText("Escalated for image input")).toBeInTheDocument(); + expect(screen.getByText("Predicted tier")).toBeInTheDocument(); + expect(screen.getByText(scenario.predicted_tier)).toBeInTheDocument(); + expect(screen.getByText(scenario.expectedThreshold)).toBeInTheDocument(); + expect(screen.getByText("modality:image")).toBeInTheDocument(); + }); + + it.each([undefined, ["request-type:code_generation", "tier-probability:simple=0.100000"]])( + "preserves legacy v2 rows without inventing a forecast when signals are %j", + (signals) => { + render(); + + expect(screen.getByText("Heuristic v2")).toBeInTheDocument(); + expect(screen.getByText("SIMPLE")).toBeInTheDocument(); + expect(screen.queryByText("Heuristic v2 estimates")).not.toBeInTheDocument(); + expect(screen.queryByText("Threshold")).not.toBeInTheDocument(); + for (const signal of signals ?? []) expect(screen.getByText(signal)).toBeInTheDocument(); + }, + ); + it("uses the persisted boundary snapshot, not today's defaults", () => { // Same score, boundaries the operator had configured lower: it lands in a // different band, and the card must say so. diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index cf2c71e64c6..78dc25119d7 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -27,6 +27,12 @@ export interface RoutingDecision { escalated?: boolean; tier_boundaries?: RoutingDecisionTierBoundaries; reasoning_override_min_score?: number; + heuristic_v2_forecast?: { + probabilities: Record; + threshold: number; + predicted_tier: string; + request_type: string; + }; } const ROUTER_TYPE_LABELS: Record = { @@ -171,6 +177,7 @@ export function RoutingDecisionCard({ escalated, escalation_keyword: escalationKeyword, tier_boundaries: tierBoundaries, + heuristic_v2_forecast: forecast, } = decision; // On an override row the score did not decide the tier, so showing it against a @@ -220,6 +227,26 @@ export function RoutingDecisionCard({ {escalated !== undefined && {describeEscalation(escalated, escalationKeyword)}} + {forecast && ( +
+
Heuristic v2 estimates
+ + + {["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((predictedTier) => ( + + {predictedTier} {(forecast.probabilities[predictedTier] * 100).toFixed(1)}% + + ))} + + + + {(forecast.threshold * 100).toFixed(1)}% + + {forecast.predicted_tier} + {forecast.request_type} +
+ )} + {signals && signals.length > 0 && ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3e1e93f3ec8..6cb7ca7df5a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37896,6 +37896,19 @@ export interface components { */ total_tokens: number; }; + /** StandardLoggingHeuristicV2Forecast */ + StandardLoggingHeuristicV2Forecast: { + /** Predicted Tier */ + predicted_tier: string; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** Request Type */ + request_type: string; + /** Threshold */ + threshold: number; + }; /** * StandardLoggingRoutingDecision * @description Per-request provenance for a pre-routing strategy (auto-router) decision. @@ -37952,6 +37965,7 @@ export interface components { escalated?: boolean; /** Escalation Keyword */ escalation_keyword?: string; + heuristic_v2_forecast?: components["schemas"]["StandardLoggingHeuristicV2Forecast"]; /** Matched Keyword */ matched_keyword?: string; /** Reasoning Override Min Score */