diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 0ea2109166e..5a3f0f4ebfa 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -909,6 +909,28 @@ def _get_cost_breakdown_from_logging_obj( return original_cost, discount_amount, margin_total_amount, margin_percent +def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None) -> float | None: + """Cost of the auto-router's LLM classifier call, read from the request's routing_decision. + + The pre-routing hook records the decision in `litellm_metadata` on messages/batch-style + routes and in `metadata` on chat-style routes, so both buckets are consulted, in the same + precedence `get_or_create_metadata_bucket` writes them. + """ + data: Final = request_data or {} + for metadata_key in ("litellm_metadata", "metadata"): + metadata = data.get(metadata_key) + if not isinstance(metadata, dict): + continue + decision = metadata.get("routing_decision") + if not isinstance(decision, dict): + continue + cost = decision.get("classifier_cost") + if isinstance(cost, bool) or not isinstance(cost, (int, float)): + continue + return float(cost) + return None + + def _has_attribute_error_in_chain(exc: Exception) -> bool: """Walk the exception chain to find an AttributeError at any depth. @@ -1032,6 +1054,7 @@ class ProxyBaseLLMRequestProcessing: pass model_name: Final = ProxyBaseLLMRequestProcessing._get_deployment_model_name(litellm_logging_obj) + classifier_cost: Final = _classifier_cost_from_request_data(request_data) headers: Final = { "x-litellm-call-id": call_id, @@ -1050,6 +1073,7 @@ class ProxyBaseLLMRequestProcessing: str(margin_total_amount) if margin_total_amount is not None else None ), "x-litellm-response-cost-margin-percent": (str(margin_percent) if margin_percent is not None else None), + "x-litellm-classifier-cost": (str(classifier_cost) if classifier_cost is not None else None), "x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit), "x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit), "x-litellm-key-max-budget": str(user_api_key_dict.max_budget), diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 932118963c1..7bbe01191e3 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -211,6 +211,16 @@ def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[ return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} +def _response_cost_or_none(response: ModelResponse) -> float | None: + hidden_params: Final = response._hidden_params + if not isinstance(hidden_params, dict): + return None + cost: Final = hidden_params.get("response_cost") + if isinstance(cost, bool) or not isinstance(cost, (int, float)): + return None + return float(cost) + + def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None) -> bool | None: from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params, @@ -470,6 +480,7 @@ class ClassificationOutcome(NamedTuple): score: float | None signals: tuple[str, ...] cause: Literal["heuristic_scorer", "reasoning_override", "llm_classifier", "default_model_fallback"] + classifier_cost: float | None = None class ComplexityRouter(CustomLogger): @@ -830,6 +841,7 @@ class ComplexityRouter(CustomLogger): escalation_keyword: str | None = None, escalated: bool = False, classifier_model: str | None = None, + classifier_cost: float | None = None, conversation_continuing: bool = True, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -875,6 +887,8 @@ class ComplexityRouter(CustomLogger): decision["escalated"] = escalated if classifier_model is not None: decision["classifier_model"] = classifier_model + if classifier_cost is not None: + decision["classifier_cost"] = classifier_cost return decision async def aclassify( @@ -896,9 +910,13 @@ class ComplexityRouter(CustomLogger): return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) try: - tier = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages) + tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages) return ClassificationOutcome( - tier=tier, score=None, signals=(f"llm-classifier:{tier.value}",), cause="llm_classifier" + tier=tier, + score=None, + signals=(f"llm-classifier:{tier.value}",), + cause="llm_classifier", + classifier_cost=classifier_cost, ) except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the configured fallback path verbose_router_logger.warning( @@ -944,7 +962,7 @@ class ComplexityRouter(CustomLogger): system_prompt: str | None = None, request_kwargs: dict[str, Any] | None = None, messages: Sequence[Mapping[str, object]] | None = None, - ) -> ComplexityTier: + ) -> tuple[ComplexityTier, float | None]: """ Call the configured classifier model with a system/user role split and prior-turn context. @@ -1044,7 +1062,7 @@ class ComplexityRouter(CustomLogger): tier: Final = self.config.tier_for_label(raw_tier) if tier is None: raise ValueError(f"LLM classifier returned an unrecognized tier: {raw_tier!r}") - return tier + return tier, _response_cost_or_none(response) @staticmethod def _build_classifier_user_payload( @@ -1902,5 +1920,6 @@ class ComplexityRouter(CustomLogger): escalation_keyword=escalation_keyword, escalated=escalated, classifier_model=classifier_model, + classifier_cost=outcome.classifier_cost, ), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c52674ede40..8b2584b0efe 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2801,6 +2801,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): matched_keyword: str escalation_keyword: str classifier_model: str + classifier_cost: float escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries conversation_continuing: bool @@ -2824,6 +2825,7 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "request_type", "score", "classifier_model", + "classifier_cost", "escalated", "tier_boundaries", "conversation_continuing", diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 93735a84ef9..aacc7498ccb 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -833,6 +833,63 @@ class TestProxyBaseLLMRequestProcessing: assert "x-litellm-response-cost-margin-amount" not in headers assert "x-litellm-response-cost-margin-percent" not in headers + @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) + def test_get_custom_headers_classifier_cost_from_routing_decision(self, metadata_key): + """The auto-router's LLM classifier cost must surface as its own header. + + x-litellm-response-cost stays the final routed call's cost (it feeds the + margin/discount family and chargeback); the classifier's cost is read from the + routing_decision the pre-routing hook recorded in the request metadata. The + bucket is metadata on chat-style routes and litellm_metadata on messages-style + routes, so both must work. + """ + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0 + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + response_cost=0.00023, + request_data={ + metadata_key: { + "routing_decision": {"cause": "llm_classifier", "classifier_cost": 8.1e-05}, + } + }, + ) + + assert headers["x-litellm-classifier-cost"] == "8.1e-05" + assert float(headers["x-litellm-response-cost"]) == 0.00023 + + @pytest.mark.parametrize( + "request_data", + [ + None, + {}, + {"metadata": {}}, + {"metadata": {"routing_decision": {"cause": "heuristic_scorer"}}}, + {"metadata": {"routing_decision": {"cause": "llm_classifier", "classifier_cost": "bogus"}}}, + {"metadata": {"routing_decision": {"cause": "llm_classifier", "classifier_cost": True}}}, + ], + ) + def test_get_custom_headers_omits_classifier_cost_without_a_priced_decision(self, request_data): + """No routing decision, a decision without a classifier call, or a malformed cost + must all omit the header entirely rather than emit 0 or a junk value.""" + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0 + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + response_cost=0.00023, + request_data=request_data, + ) + + assert "x-litellm-classifier-cost" not in headers + def test_get_cost_breakdown_from_logging_obj_helper(self): """ Test the helper function that extracts cost breakdown information. diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3f639c8b20c..b3f1e929741 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1334,11 +1334,12 @@ class TestExtractUserMessageAndSystemPrompt: assert sys_prompt is None -def _llm_response(content: str): +def _llm_response(content: str, response_cost: float | None = None): """Build a fake acompletion response with the given message content.""" response = MagicMock() response.choices = [MagicMock()] response.choices[0].message.content = content + response._hidden_params = {} if response_cost is None else {"response_cost": response_cost} return response @@ -1552,6 +1553,61 @@ class TestLLMClassifier: assert call_kwargs["model"] == "haiku-classifier" assert call_kwargs["timeout"] == 0.4 + @pytest.mark.asyncio + async def test_aclassify_llm_success_captures_classifier_cost(self, llm_complexity_router, mock_router_instance): + """The classifier call is billed, so its cost must ride the outcome. + + The classifier's own spend-log row already accounts for the money; this value is + what lets the parent request report it per-request (routing_decision and the + x-litellm-classifier-cost header), which is otherwise invisible to the caller.""" + mock_router_instance.acompletion = AsyncMock( + return_value=_llm_response('{"tier": "COMPLEX"}', response_cost=8.1e-05) + ) + outcome = await llm_complexity_router.aclassify("hi") + assert outcome.cause == "llm_classifier" + assert outcome.classifier_cost == 8.1e-05 + + @pytest.mark.asyncio + async def test_aclassify_captures_cost_from_the_real_client_pipeline(self, llm_classifier_config): + """No injected hidden params here: a real Router serves the classifier via + mock_response, so litellm's own client wrapper (update_response_metadata -> + ResponseMetadata.set_hidden_params) computes and stamps response_cost from the + deployment's per-token pricing. Pins that the capture reads a field the normal + success path actually populates.""" + real_router = Router( + model_list=[ + { + "model_name": "haiku-classifier", + "litellm_params": { + "model": "openai/mock-classifier", + "api_key": "mock-key", + "mock_response": '{"tier": "COMPLEX"}', + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + }, + } + ] + ) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=real_router, + complexity_router_config=llm_classifier_config, + ) + outcome = await router.aclassify("hi") + assert outcome.cause == "llm_classifier" + assert outcome.classifier_cost == pytest.approx(1.35e-05) + + @pytest.mark.asyncio + async def test_aclassify_classifier_cost_is_none_when_call_is_unpriced( + self, llm_complexity_router, mock_router_instance + ): + """A classifier model with no pricing yields no cost; the outcome must say None, + never 0, so the header layer can distinguish unpriced from free.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}')) + outcome = await llm_complexity_router.aclassify("hi") + assert outcome.cause == "llm_classifier" + assert outcome.classifier_cost is None + @pytest.mark.asyncio async def test_aclassify_forwards_request_metadata_for_spend_tracking( self, llm_complexity_router, mock_router_instance @@ -4114,6 +4170,45 @@ class TestRoutingDecisionContents: assert "score" not in decision assert "tier_boundaries" not in decision + @pytest.mark.asyncio + async def test_llm_classifier_decision_carries_classifier_cost(self, llm_complexity_router, mock_router_instance): + """The decision must report what the classifier call cost the caller. + + The hook returns the record through PreRoutingHookResponse, whose pydantic + validation strips keys the TypedDict does not declare, so this also pins that + classifier_cost survives the per-request path end to end.""" + mock_router_instance.acompletion = AsyncMock( + return_value=_llm_response('{"tier": "REASONING"}', response_cost=8.1e-05) + ) + response = await llm_complexity_router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + assert response is not None + decision = response.routing_decision + assert decision is not None + assert decision["cause"] == "llm_classifier" + assert decision["classifier_cost"] == 8.1e-05 + + @pytest.mark.asyncio + async def test_llm_classifier_decision_omits_cost_when_call_is_unpriced( + self, llm_complexity_router, mock_router_instance + ): + """An unpriced classifier call records no classifier_cost key at all, matching + how every optional fact on this record is omitted rather than nulled.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + response = await llm_complexity_router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + assert response is not None + decision = response.routing_decision + assert decision is not None + assert decision["cause"] == "llm_classifier" + assert "classifier_cost" not in decision + @pytest.mark.asyncio async def test_llm_classifier_fallback_decision_reports_heuristic( self, llm_complexity_router, mock_router_instance @@ -4131,6 +4226,7 @@ class TestRoutingDecisionContents: assert decision is not None assert decision["cause"] == "heuristic_scorer" assert "classifier_model" not in decision + assert "classifier_cost" not in decision assert isinstance(decision["score"], float) @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 86de0df30bc..65434407f74 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32482,6 +32482,8 @@ export interface components { * @enum {string} */ cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + /** Classifier Cost */ + classifier_cost?: number; /** Classifier Model */ classifier_model?: string; /** Conversation Continuing */