fix(auto-router): show heuristic v2 score estimates in routing details

litellm/router_strategy/complexity_router/complexity_router.py

@@ -75,6 +75,7 @@ from litellm.types.utils import (
     AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
     ModelResponse,
     RoutingDecisionCause,
+    StandardLoggingHeuristicV2Forecast,
     StandardLoggingRoutingDecision,
     StandardLoggingRoutingDecisionTierBoundaries,
 )
@@ -1043,6 +1044,7 @@ class ClassificationOutcome(NamedTuple):
     capability_forecast: CapabilityClassifierForecast | None = None
     llm_v2_forecast: LLMV2Decision | None = None
     jev_verdict: JevVerdict | None = None
+    heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None
 
 
 def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome:
@@ -1075,6 +1077,8 @@ def _with_classifier_forecast(
     decision: StandardLoggingRoutingDecision, outcome: ClassificationOutcome
 ) -> StandardLoggingRoutingDecision:
     """Attach validated forecasts and their applied policy to the routing decision."""
+    if outcome.heuristic_v2_forecast is not None:
+        return {**decision, "heuristic_v2_forecast": outcome.heuristic_v2_forecast}
     if outcome.jev_verdict is not None:
         forecasted_decision: Final[StandardLoggingRoutingDecision] = {
             **decision,
@@ -1772,6 +1776,7 @@ class ComplexityRouter(CustomLogger):
         conversation_continuing: bool = True,
         tier_litellm_params: Mapping[str, object] | None = None,
         context_escalation_original_tier: ComplexityTier | str | None = None,
+        heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None,
     ) -> StandardLoggingRoutingDecision:
         """Assemble the per-request provenance record for this router's decision.
 
@@ -1831,7 +1836,9 @@ class ComplexityRouter(CustomLogger):
             masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params)
             if isinstance(masked_tier_litellm_params, Mapping):
                 decision["tier_litellm_params"] = masked_tier_litellm_params
-        return decision
+        return (
+            decision if heuristic_v2_forecast is None else {**decision, "heuristic_v2_forecast": heuristic_v2_forecast}
+        )
 
     async def aclassify(
         self,
@@ -1888,6 +1895,15 @@ class ComplexityRouter(CustomLogger):
             score=None,
             signals=(f"request-type:{request_type.value}", *probability_signals),
             cause="heuristic_v2",
+            heuristic_v2_forecast=StandardLoggingHeuristicV2Forecast(
+                probabilities={
+                    candidate.value: prediction.probabilities[index]
+                    for index, candidate in enumerate(TIER_SEVERITY_ORDER, start=1)
+                },
+                threshold=predictor.routing_threshold,
+                predicted_tier=tier.value,
+                request_type=request_type.value,
+            ),
         )
 
     async def _classify_heuristic_first(
@@ -3553,6 +3569,7 @@ class ComplexityRouter(CustomLogger):
             context_escalation_original_tier=(
                 decision.get("context_escalation_original_tier") if decision is not None else None
             ),
+            heuristic_v2_forecast=decision.get("heuristic_v2_forecast") if decision is not None else None,
         )
         from litellm.types.router import PreRoutingHookResponse as HookResponse
 
@@ -3732,6 +3749,7 @@ class ComplexityRouter(CustomLogger):
                         conversation_continuing=bool(decision.get("conversation_continuing", True)),
                         tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model),
                         context_escalation_original_tier=decision.get("context_escalation_original_tier"),
+                        heuristic_v2_forecast=decision.get("heuristic_v2_forecast"),
                     )
                     return response.model_copy(
                         update={  # mutable-ok: model_copy types update as a plain dict
@@ -3776,6 +3794,7 @@ class ComplexityRouter(CustomLogger):
             conversation_continuing=bool(decision.get("conversation_continuing", True)),
             tier_litellm_params=self._litellm_params_for_model(None, default_model),
             context_escalation_original_tier=decision.get("context_escalation_original_tier"),
+            heuristic_v2_forecast=decision.get("heuristic_v2_forecast"),
         )
         return response.model_copy(
             update={  # mutable-ok: model_copy types update as a plain dict

litellm/types/utils.py

@@ -2974,6 +2974,13 @@ LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN: Final[InternalCallOrigin] = "llm_as_a_judg
 BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
 
 
+class StandardLoggingHeuristicV2Forecast(TypedDict):
+    probabilities: ReadOnly[Mapping[str, float]]
+    threshold: ReadOnly[float]
+    predicted_tier: ReadOnly[str]
+    request_type: ReadOnly[str]
+
+
 class StandardLoggingRoutingDecision(TypedDict, total=False):
     """Per-request provenance for a pre-routing strategy (auto-router) decision."""
 
@@ -2992,6 +2999,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
     classifier_cost: float
     classifier_probabilities: ReadOnly[Mapping[str, float]]
     classifier_confidence: ReadOnly[float]
+    heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast]
     classifier_crux: str  # writable-ok: added only when a capability verdict is available
     classifier_primary_rule: str  # writable-ok: added only when a capability verdict is available
     classifier_capability_boundary: str  # writable-ok: added only when a capability verdict is available
@@ -3037,6 +3045,7 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
         "classifier_cost",
         "classifier_probabilities",
         "classifier_confidence",
+        "heuristic_v2_forecast",
         "classifier_primary_rule",
         "classifier_capability_boundary",
         "classifier_p_solve",

tests/test_litellm/router_strategy/test_complexity_router.py

@@ -3721,7 +3721,11 @@ class TestLLMClassifier:
         assert outcome.score is not None
 
     @pytest.mark.asyncio
-    async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(self, mock_router_instance):
+    @pytest.mark.parametrize("redact", (False, True))
+    async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(
+        self, mock_router_instance: MagicMock, redact: bool, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setattr(litellm, "turn_off_message_logging", redact)
         router = ComplexityRouter(
             model_name="tier-router",
             litellm_router_instance=mock_router_instance,
@@ -3754,6 +3758,21 @@ class TestLLMClassifier:
             "tier-probability:complex=0.892157",
             "tier-probability:reasoning=0.980392",
         ]
+        redacted: Final = Router._redact_prompt_text_if_needed(
+            request_kwargs={}, routing_decision=response.routing_decision
+        )
+        assert ("signals" in redacted) is not redact
+        assert redacted["heuristic_v2_forecast"] == {
+            "probabilities": {
+                "SIMPLE": 11 / 102,
+                "MEDIUM": 21 / 102,
+                "COMPLEX": 91 / 102,
+                "REASONING": 100 / 102,
+            },
+            "threshold": 0.8,
+            "predicted_tier": "COMPLEX",
+            "request_type": "general",
+        }
 
     def test_heuristic_v2_needs_no_classifier_model(self):
         config = ComplexityRouterConfig(classifier_type="heuristic_v2")
@@ -8879,13 +8898,29 @@ class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape:
         ],
     )
     @pytest.mark.asyncio
-    async def test_decision_reaches_the_spend_log_payload(self, request_kwargs, expected_bucket):
+    @pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_v2"))
+    async def test_decision_reaches_the_spend_log_payload(self, request_kwargs, expected_bucket, classifier_type):
         import datetime
         import json
 
         from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
 
-        router = Router(model_list=self.MODEL_LIST)
+        model_list: Final = [
+            {
+                **row,
+                "litellm_params": {
+                    **row["litellm_params"],
+                    "complexity_router_config": {
+                        **row["litellm_params"]["complexity_router_config"],
+                        "classifier_type": classifier_type,
+                    },
+                },
+            }
+            if row["model_name"] == "smart-router"
+            else row
+            for row in self.MODEL_LIST
+        ]
+        router = Router(model_list=model_list)
         response = await router.async_pre_routing_hook(
             model="smart-router",
             request_kwargs=request_kwargs,
@@ -8915,6 +8950,15 @@ class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape:
         persisted = json.loads(payload["metadata"])["routing_decision"]
         assert persisted is not None, f"routing_decision dropped for {expected_bucket}"
         assert persisted["router_model_name"] == "smart-router"
+        if classifier_type == "heuristic_v2":
+            assert persisted["heuristic_v2_forecast"] == request_kwargs[expected_bucket]["routing_decision"][
+                "heuristic_v2_forecast"
+            ]
+            assert set(persisted["heuristic_v2_forecast"]["probabilities"]) == {
+                "SIMPLE", "MEDIUM", "COMPLEX", "REASONING"
+            }
+        else:
+            assert "heuristic_v2_forecast" not in persisted
 
 
 class TestRoutingDecisionIsPerAttempt:
@@ -14104,6 +14148,33 @@ class TestModalityRouting:
     BASE_TIERS = {"SIMPLE": "text-cheap", "MEDIUM": "vision-mid", "COMPLEX": "vision-big"}
     BASE_VISION = {"text-cheap": False, "vision-mid": True, "vision-big": True, "vision-default": True}
 
+    @pytest.mark.asyncio
+    async def test_modality_escalation_preserves_the_original_heuristic_v2_forecast(
+        self, mock_router_instance: MagicMock
+    ) -> None:
+        router: Final = self._router(
+            mock_router_instance,
+            {
+                "classifier_type": "heuristic_v2",
+                "heuristic_v2_artifact": _heuristic_v2_artifact(),
+                "tiers": {"COMPLEX": "text-cheap", "REASONING": "vision-big"},
+                "modality_routing": True,
+            },
+            self.BASE_VISION,
+        )
+        original: Final = await router.aclassify("What color is this?")
+        result: Final = await router.async_pre_routing_hook(
+            model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE
+        )
+
+        assert original.heuristic_v2_forecast is not None
+        assert result is not None and result.routing_decision is not None
+        assert result.model == "vision-big"
+        assert result.routing_decision["cause"] == "modality_escalation"
+        assert result.routing_decision["tier"] == "REASONING"
+        assert result.routing_decision["heuristic_v2_forecast"] == original.heuristic_v2_forecast
+        assert result.routing_decision["heuristic_v2_forecast"]["predicted_tier"] == "COMPLEX"
+
     @staticmethod
     def _router(mock_router_instance, config, vision_by_model):
         """vision_by_model: model name -> True/False (deployment model_info) or None (undeclared)."""
@@ -14479,6 +14550,69 @@ class TestModalityRouting:
 
 @pytest.mark.usefixtures("local_model_cost_map")
 class TestHealthFallbackDispatch:
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("peer", (True, False), ids=("peer_failover", "default_fallback"))
+    async def test_health_rewrites_preserve_the_original_heuristic_v2_forecast(self, peer: bool) -> None:
+        router: Final = self._router(
+            config={
+                "classifier_type": "heuristic_v2",
+                "heuristic_v2_artifact": _heuristic_v2_artifact(),
+                "tiers": {"COMPLEX": ["primary", "peer"] if peer else "primary"},
+            }
+        )
+
+        def select_primary(models: Sequence[str]) -> str:
+            return max(models)
+
+        with patch(  # test-quality-ok: force initial classification onto the failing group in a mixed tier pool
+            "litellm.router_strategy.complexity_router.complexity_router.random.choice",
+            side_effect=select_primary,
+        ):
+            original: Final = await router.async_pre_routing_hook(
+                model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}]
+            )
+            self._unavailable(router, "primary-id", "cooldown")
+            result: Final = await router.async_pre_routing_hook(
+                model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}]
+            )
+
+        assert original is not None and original.routing_decision is not None
+        assert original.model == "primary"
+        assert original.routing_decision["cause"] == "heuristic_v2"
+        assert result is not None and result.routing_decision is not None
+        assert result.model == ("peer" if peer else "fallback")
+        assert result.routing_decision["cause"] == ("health_failover" if peer else "health_default_fallback")
+        assert result.routing_decision["heuristic_v2_forecast"] == original.routing_decision["heuristic_v2_forecast"]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("pinned", (False, True), ids=("keyword_bypass", "session_pin"))
+    async def test_heuristic_v2_bypasses_have_no_fabricated_forecast(self, pinned: bool) -> None:
+        router: Final = self._router(
+            session=pinned,
+            config={
+                "classifier_type": "heuristic_v2",
+                "heuristic_v2_artifact": _heuristic_v2_artifact(),
+                "tiers": {"COMPLEX": "primary"},
+                "keyword_tier_rules": [{"keywords": ["quick lookup"], "tier": "COMPLEX"}],
+            },
+        )
+        original: Final = await router.async_pre_routing_hook(
+            model="health-router",
+            request_kwargs={"metadata": {"session_id": "v2-forecast"}},
+            messages=[{"role": "user", "content": "Hello!"}],
+        )
+        result: Final = await router.async_pre_routing_hook(
+            model="health-router",
+            request_kwargs={"metadata": {"session_id": "v2-forecast"}},
+            messages=[{"role": "user", "content": "quick lookup"}],
+        )
+
+        assert original is not None and original.routing_decision is not None
+        assert "heuristic_v2_forecast" in original.routing_decision
+        assert result is not None and result.routing_decision is not None
+        assert result.routing_decision["cause"] == ("session_affinity_pin" if pinned else "literal_keyword_match")
+        assert "heuristic_v2_forecast" not in result.routing_decision
+
     @pytest.fixture(autouse=True)
     def httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None:
         monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)

ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx

@@ -1,8 +1,14 @@
 import React from "react";
 import { render, screen } from "@testing-library/react";
-import { describe, it, expect } from "vitest";
+import { describe, it, expect, vi } from "vitest";
 import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard";
 
+vi.mock("@/components/ui/badge", () => ({
+  Badge: ({ children }: { children: React.ReactNode }) => <span>{children}</span>,
+}));
+
+vi.mock("lucide-react", () => ({ Waypoints: () => null }));
+
 const heuristic: RoutingDecision = {
   router_model_name: "smart-router",
   router_type: "complexity",
@@ -14,6 +20,13 @@ const heuristic: RoutingDecision = {
   tier_boundaries: { simple_medium: 0.15, medium_complex: 0.35, complex_reasoning: 0.6 },
 };
 
+const forecast = {
+  probabilities: { MEDIUM: 0.69321, SIMPLE: 0, COMPLEX: 0.81234, REASONING: 0.92345 },
+  threshold: 0.69,
+  predicted_tier: "MEDIUM",
+  request_type: "code_generation",
+};
+
 describe("RoutingDecisionCard", () => {
   it("renders nothing when the request carried no routing decision", () => {
     const { container } = render(<RoutingDecisionCard decision={undefined} />);
@@ -30,8 +43,76 @@ describe("RoutingDecisionCard", () => {
     expect(screen.getByText("(at or above 0.6, REASONING)")).toBeInTheDocument();
     expect(screen.getByText("claude-sonnet")).toBeInTheDocument();
     expect(screen.getByText("long (900 tokens)")).toBeInTheDocument();
+    expect(screen.queryByText("Heuristic v2 estimates")).not.toBeInTheDocument();
   });
 
+  it("shows recorded v2 success estimates, including zero, when signals were redacted", () => {
+    render(
+      <RoutingDecisionCard
+        decision={{
+          cause: "heuristic_v2",
+          tier: "MEDIUM",
+          tier_label: "Balanced",
+          heuristic_v2_forecast: forecast,
+        }}
+      />,
+    );
+
+    expect(screen.getByText("Heuristic v2 estimates")).toBeInTheDocument();
+    expect(screen.getByText("Success by tier")).toBeInTheDocument();
+    expect(
+      screen.getAllByText(/^(SIMPLE|MEDIUM|COMPLEX|REASONING) \d+\.\d%$/).map((badge) => badge.textContent),
+    ).toEqual(["SIMPLE 0.0%", "MEDIUM 69.3%", "COMPLEX 81.2%", "REASONING 92.3%"]);
+    expect(screen.getByText("Threshold")).toBeInTheDocument();
+    expect(screen.getByText("69.0%")).toBeInTheDocument();
+    expect(screen.getByText("Predicted tier")).toBeInTheDocument();
+    expect(screen.getByText("MEDIUM")).toBeInTheDocument();
+    expect(screen.getByText("Balanced")).toBeInTheDocument();
+    expect(screen.getByText("code_generation")).toBeInTheDocument();
+    expect(screen.queryByText("Score")).not.toBeInTheDocument();
+  });
+
+  it.each([
+    { threshold: 0, predicted_tier: "SIMPLE", expectedThreshold: "0.0%" },
+    { threshold: 0.99, predicted_tier: "REASONING", expectedThreshold: "99.0%" },
+  ])("keeps the prediction separate from an overridden tier at threshold $threshold", (scenario) => {
+    render(
+      <RoutingDecisionCard
+        decision={{
+          cause: "modality_escalation",
+          tier: "REASONING",
+          tier_label: "Vision",
+          signals: ["modality:image"],
+          heuristic_v2_forecast: {
+            ...forecast,
+            threshold: scenario.threshold,
+            predicted_tier: scenario.predicted_tier,
+          },
+        }}
+      />,
+    );
+
+    expect(screen.getByText("Vision")).toBeInTheDocument();
+    expect(screen.getByText("Escalated for image input")).toBeInTheDocument();
+    expect(screen.getByText("Predicted tier")).toBeInTheDocument();
+    expect(screen.getByText(scenario.predicted_tier)).toBeInTheDocument();
+    expect(screen.getByText(scenario.expectedThreshold)).toBeInTheDocument();
+    expect(screen.getByText("modality:image")).toBeInTheDocument();
+  });
+
+  it.each([undefined, ["request-type:code_generation", "tier-probability:simple=0.100000"]])(
+    "preserves legacy v2 rows without inventing a forecast when signals are %j",
+    (signals) => {
+      render(<RoutingDecisionCard decision={{ cause: "heuristic_v2", tier: "SIMPLE", signals }} />);
+
+      expect(screen.getByText("Heuristic v2")).toBeInTheDocument();
+      expect(screen.getByText("SIMPLE")).toBeInTheDocument();
+      expect(screen.queryByText("Heuristic v2 estimates")).not.toBeInTheDocument();
+      expect(screen.queryByText("Threshold")).not.toBeInTheDocument();
+      for (const signal of signals ?? []) expect(screen.getByText(signal)).toBeInTheDocument();
+    },
+  );
+
   it("uses the persisted boundary snapshot, not today's defaults", () => {
     // Same score, boundaries the operator had configured lower: it lands in a
     // different band, and the card must say so.

ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx

@@ -27,6 +27,12 @@ export interface RoutingDecision {
   escalated?: boolean;
   tier_boundaries?: RoutingDecisionTierBoundaries;
   reasoning_override_min_score?: number;
+  heuristic_v2_forecast?: {
+    probabilities: Record<string, number>;
+    threshold: number;
+    predicted_tier: string;
+    request_type: string;
+  };
 }
 
 const ROUTER_TYPE_LABELS: Record<string, string> = {
@@ -171,6 +177,7 @@ export function RoutingDecisionCard({
     escalated,
     escalation_keyword: escalationKeyword,
     tier_boundaries: tierBoundaries,
+    heuristic_v2_forecast: forecast,
   } = decision;
 
   // On an override row the score did not decide the tier, so showing it against a
@@ -220,6 +227,26 @@ export function RoutingDecisionCard({
 
         {escalated !== undefined && <Row label="Escalated">{describeEscalation(escalated, escalationKeyword)}</Row>}
 
+        {forecast && (
+          <div className="mt-3 border-t pt-3">
+            <div className="mb-1 text-sm font-medium">Heuristic v2 estimates</div>
+            <Row label="Success by tier">
+              <span className="flex flex-wrap gap-1">
+                {["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((predictedTier) => (
+                  <Badge key={predictedTier} variant="outline" className="font-normal tabular-nums">
+                    {predictedTier} {(forecast.probabilities[predictedTier] * 100).toFixed(1)}%
+                  </Badge>
+                ))}
+              </span>
+            </Row>
+            <Row label="Threshold">
+              <span className="tabular-nums">{(forecast.threshold * 100).toFixed(1)}%</span>
+            </Row>
+            <Row label="Predicted tier">{forecast.predicted_tier}</Row>
+            <Row label="Request type">{forecast.request_type}</Row>
+          </div>
+        )}
+
         {signals && signals.length > 0 && (
           <Row label="Signals">
             <span className="flex flex-wrap gap-1">

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -37811,6 +37811,19 @@ export interface components {
              */
             total_tokens: number;
         };
+        /** StandardLoggingHeuristicV2Forecast */
+        StandardLoggingHeuristicV2Forecast: {
+            /** Predicted Tier */
+            predicted_tier: string;
+            /** Probabilities */
+            probabilities: {
+                [key: string]: number;
+            };
+            /** Request Type */
+            request_type: string;
+            /** Threshold */
+            threshold: number;
+        };
         /**
          * StandardLoggingRoutingDecision
          * @description Per-request provenance for a pre-routing strategy (auto-router) decision.
@@ -37867,6 +37880,7 @@ export interface components {
             escalated?: boolean;
             /** Escalation Keyword */
             escalation_keyword?: string;
+            heuristic_v2_forecast?: components["schemas"]["StandardLoggingHeuristicV2Forecast"];
             /** Matched Keyword */
             matched_keyword?: string;
             /** Reasoning Override Min Score */