fix(proxy): price a billed failure with Logging.get_router_model_id and the deployment's base_model like the success path

litellm/proxy/utils.py

@@ -267,7 +267,6 @@ from litellm.repositories.verification_token_repository import (
     VerificationTokenRepository,
 )
 from litellm.router_utils.common_utils import resolve_model_group_alias
-from litellm.router_utils.fallback_event_handlers import routed_deployment_id
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
 from litellm.types.llms.openai import ResponsesAPIResponse
@@ -281,6 +280,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionRe
 from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams
 from litellm.utils import (
     add_configured_callback_to_specific_event,
+    get_base_model_from_metadata,
 )
 
 if TYPE_CHECKING:
@@ -1017,7 +1017,8 @@ def _failure_usage_to_lift(
     request_body: Mapping[str, object],
     dispatched: bool,
     provider_billed: bool = False,
-    litellm_params: Mapping[str, object] | None = None,
+    deployment_id: str | None = None,
+    base_model: str | None = None,
 ) -> tuple[object, object] | None:
     """A stream that broke mid-flight still billed the provider for the chunks
     already delivered; the streaming handler stashes that recovered usage and
@@ -1026,8 +1027,11 @@ def _failure_usage_to_lift(
     estimated input-side Usage. When the failure is one the provider bills
     anyway (``provider_billed``: a read timeout or a client disconnect, where
     the provider had the prompt and kept working after LiteLLM hung up) that
-    usage is priced at the model's input rate; any other post-dispatch failure
-    stays at zero cost. The raw request body backfills
+    usage is priced at the model's input rate, the way the success path prices it:
+    at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``)
+    when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure
+    deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch
+    failure stays at zero cost. The raw request body backfills
     the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a
     chat-completions provider) without filling optional_params. Returns the
     (combined_usage_object, response_cost) pair to lift, or None."""
@@ -1061,18 +1065,18 @@ def _failure_usage_to_lift(
         return None
     if not provider_billed:
         return estimated_usage, 0.0
-    return estimated_usage, _estimated_input_cost(model_call_details, estimated_usage.prompt_tokens, litellm_params)
-
-
-def _deployment_cost_per_token(litellm_params: Mapping[str, object] | None, model: str) -> CostPerToken | None:
-    """The pricing the router registered for this deployment under its ``model_info.id``: the id
-    is read with the router's own ``routed_deployment_id`` from ``Logging.litellm_params`` (never
-    from the flattened ``model_call_details`` copy a request body key can overwrite) and priced
-    through ``deployment_pricing_model_info`` as the success path does; None means the cost map."""
-    model_info: Final = deployment_pricing_model_info(
-        model_id=routed_deployment_id(litellm_params) if litellm_params is not None else None,
-        deployment_model=model,
+    return estimated_usage, _estimated_input_cost(
+        model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model
     )
+
+
+def _deployment_cost_per_token(deployment_id: str | None, model: str) -> CostPerToken | None:
+    """The pricing the router registered for this deployment under its ``model_info.id``, read
+    through ``deployment_pricing_model_info`` exactly as the success path does; None means the
+    public cost map. The id is ``Logging.get_router_model_id()`` (the logging object's own
+    ``litellm_params``), never the flattened ``model_call_details`` copy a request body key can
+    overwrite."""
+    model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model)
     if model_info is None:
         return None
     return CostPerToken(
@@ -1084,17 +1088,20 @@ def _deployment_cost_per_token(litellm_params: Mapping[str, object] | None, mode
 def _estimated_input_cost(
     model_call_details: Mapping[str, object],
     prompt_tokens: int,
-    litellm_params: Mapping[str, object] | None,
+    deployment_id: str | None,
+    base_model: str | None,
 ) -> float:
-    """Input price for the estimated prompt; an unpriceable model (not in the cost map and no
-    deployment rate) keeps the row at $0 rather than dropping the failure row."""
+    """Input price for the estimated prompt: the deployment's registered rate when it has one, else
+    the cost map's rate for ``base_model`` or the deployment model (an Azure deployment name is not
+    in the cost map, its ``base_model`` is). An unpriceable model keeps the row at $0 rather than
+    dropping the failure row."""
     try:
         model: Final = str(model_call_details.get("model") or "")
         estimated_input_cost, _ = litellm.cost_per_token(
-            model=model,
+            model=base_model or model,
             custom_llm_provider=str(model_call_details.get("custom_llm_provider") or "") or None,
             prompt_tokens=prompt_tokens,
-            custom_cost_per_token=_deployment_cost_per_token(litellm_params, model),
+            custom_cost_per_token=_deployment_cost_per_token(deployment_id, model),
         )
     except Exception as exc:  # noqa: BLE001  # cost_per_token raises BadRequestError for an unmapped model and the row must still be written
         verbose_proxy_logger.debug(
@@ -1263,7 +1270,8 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed:
         request_body=request_data,
         dispatched=_first_handoff is not None,
         provider_billed=provider_billed,
-        litellm_params=getattr(_logging_obj, "litellm_params", None),
+        deployment_id=_logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None,
+        base_model=get_base_model_from_metadata(model_call_details=_model_call_details),
     )
     _entries: Final = (
         ("start_time", _model_call_details.get("start_time")),

tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py

@@ -2042,7 +2042,7 @@ def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the
         request_body={},
         dispatched=True,
         provider_billed=True,
-        litellm_params={"model": f"openai/{FAILURE_USAGE_MODEL}", "metadata": {"model_info": {"id": "dep-lit145"}}},
+        deployment_id="dep-lit145",
     )
 
     assert lifted is not None
@@ -2051,33 +2051,69 @@ def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the
     assert response_cost == pytest.approx(usage.prompt_tokens * 0.25)
 
 
-def test_the_deployment_id_is_resolved_like_the_success_path_litellm_metadata_first(monkeypatch):
-    """On /messages and /responses the router stamps the real deployment id into litellm_metadata;
-    a model_info on metadata (possibly client-supplied) must not pick the rate."""
-    from litellm.proxy.utils import _failure_usage_to_lift
+def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object:
+    """A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params()
+    shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff
+    instant, so the lift treats the request as dispatched."""
+    from litellm.litellm_core_utils.litellm_logging import Logging
 
-    for dep, rate in (("dep-lit145-real", 0.25), ("dep-lit145-wrong", 9.0)):
-        monkeypatch.setitem(
-            litellm.model_cost,
-            dep,
-            {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": rate, "output_cost_per_token": rate},
-        )
-
-    lifted = _failure_usage_to_lift(
-        model_call_details={"call_type": "acompletion", "model": FAILURE_USAGE_MODEL, "messages": ONE_USER_MESSAGE},
-        request_body={},
-        dispatched=True,
-        provider_billed=True,
-        litellm_params={
-            "model": f"openai/{FAILURE_USAGE_MODEL}",
-            "metadata": {"model_info": {"id": "dep-lit145-wrong"}},
-            "litellm_metadata": {"model_info": {"id": "dep-lit145-real"}},
-        },
+    logging_obj = Logging(
+        model=model,
+        messages=ONE_USER_MESSAGE,
+        stream=False,
+        call_type="acompletion",
+        start_time=real_datetime.datetime.now(),
+        litellm_call_id="call-lit145",
+        function_id="fn-lit145",
     )
+    logging_obj.update_environment_variables(model=model, user="", optional_params={}, litellm_params=litellm_params)
+    logging_obj.model_call_details["first_api_call_start_time"] = real_datetime.datetime.now()
+    logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE
+    logging_obj.model_call_details["call_type"] = "acompletion"
+    return logging_obj
 
-    assert lifted is not None
-    usage, response_cost = lifted
-    assert response_cost == pytest.approx(usage.prompt_tokens * 0.25)
+
+def test_the_deployment_rate_is_found_on_the_logging_objects_real_litellm_params(monkeypatch):
+    """``get_litellm_params()`` always carries a ``litellm_metadata`` key (None on chat
+    completions) next to the router-stamped ``metadata.model_info.id``; the failure row must
+    still price at the deployment's own rate, like ``Logging.get_router_model_id`` on the
+    success path."""
+    from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
+    from litellm.proxy.utils import _failure_fields_to_lift
+
+    monkeypatch.setitem(
+        litellm.model_cost,
+        "dep-lit145-real-shape",
+        {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 0.25},
+    )
+    litellm_params = get_litellm_params(metadata={"model_info": {"id": "dep-lit145-real-shape"}})
+    assert "litellm_metadata" in litellm_params and litellm_params["litellm_metadata"] is None
+    logging_obj = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, litellm_params)
+
+    lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
+
+    assert lifted["response_cost"] == pytest.approx(lifted["combined_usage_object"].prompt_tokens * 0.25)
+
+
+def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path():
+    """An Azure deployment name is not in the cost map; the success path prices it through
+    ``model_info.base_model`` (``get_base_model_from_metadata``) and so must the failure row."""
+    from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
+    from litellm.proxy.utils import _failure_fields_to_lift
+
+    logging_obj = _dispatched_failure_logging_obj(
+        "azure/lit145-not-in-cost-map",
+        get_litellm_params(metadata={"model_info": {"base_model": "azure/gpt-4.1"}}),
+    )
+    logging_obj.model_call_details["custom_llm_provider"] = "azure"
+
+    lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
+
+    expected_cost, _ = litellm.cost_per_token(
+        model="azure/gpt-4.1", prompt_tokens=lifted["combined_usage_object"].prompt_tokens
+    )
+    assert expected_cost > 0
+    assert lifted["response_cost"] == pytest.approx(expected_cost)
 
 
 def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price():
@@ -2096,7 +2132,7 @@ def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price(
         request_body={},
         dispatched=True,
         provider_billed=True,
-        litellm_params={"model": f"openai/{FAILURE_USAGE_MODEL}"},
+        deployment_id=None,
     )
 
     assert lifted is not None
@@ -2377,9 +2413,7 @@ def test_create_model_info_response_resolves_mode_through_deployment_model():
             ]
         )
 
-        response = create_model_info_response(
-            model_id="my-embeddings", provider="openai", llm_router=router
-        )
+        response = create_model_info_response(model_id="my-embeddings", provider="openai", llm_router=router)
     finally:
         litellm.model_cost.clear()
         litellm.model_cost.update(saved_model_cost)
@@ -2409,12 +2443,8 @@ def test_create_model_info_response_resolves_model_group_alias_to_target(model_g
         model_group_alias=model_group_alias,
     )
 
-    alias_response = create_model_info_response(
-        model_id="team-embeddings", provider="openai", llm_router=router
-    )
-    target_response = create_model_info_response(
-        model_id="my-embeddings", provider="openai", llm_router=router
-    )
+    alias_response = create_model_info_response(model_id="team-embeddings", provider="openai", llm_router=router)
+    target_response = create_model_info_response(model_id="my-embeddings", provider="openai", llm_router=router)
 
     assert alias_response["id"] == "team-embeddings"
     for field in ("mode", "max_input_tokens", "max_output_tokens"):
@@ -2497,7 +2527,9 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp
     with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
         await proxy_logging_obj.post_call_failure_hook(
             request_data={"metadata": {}},
-            original_exception=HTTPException(status_code=400, detail="Upstream passthrough request failed with status 400"),
+            original_exception=HTTPException(
+                status_code=400, detail="Upstream passthrough request failed with status 400"
+            ),
             user_api_key_dict=UserAPIKeyAuth(),
             traceback_str=upstream_traceback,
         )
@@ -2561,9 +2593,13 @@ def test_mcp_conversion_preserves_request_policy_and_isolates_guardrail_data(buc
     parent = {
         "model": "parent-model",
         bucket: {
-            "guardrails": ["policy-rule"], "guardrail_config": {"language": "en"},
-            "applied_policies": ["parent-policy"], "policy_sources": {"parent-policy": "model"},
-            "_guardrail_pipelines": [], "_pipeline_managed_guardrails": ["pipeline-rule"], "tags": ["review"],
+            "guardrails": ["policy-rule"],
+            "guardrail_config": {"language": "en"},
+            "applied_policies": ["parent-policy"],
+            "policy_sources": {"parent-policy": "model"},
+            "_guardrail_pipelines": [],
+            "_pipeline_managed_guardrails": ["pipeline-rule"],
+            "tags": ["review"],
         },
         "guardrails": [{"request-rule": {"extra_body": {"threshold": 0.9}}}],
         "guardrail_config": {"entities": ["EMAIL_ADDRESS"]},
@@ -2593,13 +2629,26 @@ def test_mcp_conversion_honors_only_authenticated_global_guardrail_opt_outs(opt_
     from litellm.responses.mcp.request_context import MCPRequestContext
 
     auth = UserAPIKeyAuth(metadata={"opted_out_global_guardrails": ["global-rule"] if opt_out else []})
-    context = MCPRequestContext.resolve(kwargs={"metadata": {
-        "user_api_key_auth": auth, "disable_global_guardrails": True,
-        "user_api_key_metadata": {"disable_global_guardrails": True},
-    }}, tools=None)
+    context = MCPRequestContext.resolve(
+        kwargs={
+            "metadata": {
+                "user_api_key_auth": auth,
+                "disable_global_guardrails": True,
+                "user_api_key_metadata": {"disable_global_guardrails": True},
+            }
+        },
+        tools=None,
+    )
     proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
-    kwargs = {"name": "execute", "arguments": {}, "user_api_key_auth": auth, "guardrail_context": context.guardrail_context}
-    synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs)
+    kwargs = {
+        "name": "execute",
+        "arguments": {},
+        "user_api_key_auth": auth,
+        "guardrail_context": context.guardrail_context,
+    }
+    synthetic = proxy_logging._convert_mcp_to_llm_format(
+        proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs
+    )
     guardrail = CustomGuardrail(guardrail_name="global-rule", event_hook="pre_mcp_call", default_on=True)
     assert guardrail.should_run_guardrail(synthetic, GuardrailEventHooks.pre_mcp_call) is (not opt_out)
     synthetic["metadata"]["user_api_key_metadata"]["opted_out_global_guardrails"].append("unrelated")
@@ -2613,19 +2662,26 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte
     from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyGuardrails
 
     registry = policy_registry.PolicyRegistry()
-    registry._policies = {"model-policy": Policy(
-        condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"])
-    )}
+    registry._policies = {
+        "model-policy": Policy(
+            condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"])
+        )
+    }
     registry._initialized = True
     monkeypatch.setattr(policy_registry, "_policy_registry", registry)
     monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
     proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
     kwargs = {
-        "name": "execute", "arguments": {},
+        "name": "execute",
+        "arguments": {},
         "user_api_key_auth": UserAPIKeyAuth(metadata={"policies": ["model-policy"]}),
-        "guardrail_context": MCPRequestContext.resolve_guardrail_context({"model": model, "guardrails": ["request-rule"]}),
+        "guardrail_context": MCPRequestContext.resolve_guardrail_context(
+            {"model": model, "guardrails": ["request-rule"]}
+        ),
     }
-    synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs)
+    synthetic = proxy_logging._convert_mcp_to_llm_format(
+        proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs
+    )
     assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected
     assert "request-rule" in synthetic["metadata"]["guardrails"]