diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bdf9ed47758..a8bd35644c9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -267,7 +267,6 @@ from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) from litellm.router_utils.common_utils import resolve_model_group_alias -from litellm.router_utils.fallback_event_handlers import routed_deployment_id from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES from litellm.types.llms.openai import ResponsesAPIResponse @@ -281,6 +280,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionRe from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams from litellm.utils import ( add_configured_callback_to_specific_event, + get_base_model_from_metadata, ) if TYPE_CHECKING: @@ -1017,7 +1017,8 @@ def _failure_usage_to_lift( request_body: Mapping[str, object], dispatched: bool, provider_billed: bool = False, - litellm_params: Mapping[str, object] | None = None, + deployment_id: str | None = None, + base_model: str | None = None, ) -> tuple[object, object] | None: """A stream that broke mid-flight still billed the provider for the chunks already delivered; the streaming handler stashes that recovered usage and @@ -1026,8 +1027,11 @@ def _failure_usage_to_lift( estimated input-side Usage. When the failure is one the provider bills anyway (``provider_billed``: a read timeout or a client disconnect, where the provider had the prompt and kept working after LiteLLM hung up) that - usage is priced at the model's input rate; any other post-dispatch failure - stays at zero cost. The raw request body backfills + usage is priced at the model's input rate, the way the success path prices it: + at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``) + when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure + deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch + failure stays at zero cost. The raw request body backfills the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a chat-completions provider) without filling optional_params. Returns the (combined_usage_object, response_cost) pair to lift, or None.""" @@ -1061,18 +1065,18 @@ def _failure_usage_to_lift( return None if not provider_billed: return estimated_usage, 0.0 - return estimated_usage, _estimated_input_cost(model_call_details, estimated_usage.prompt_tokens, litellm_params) - - -def _deployment_cost_per_token(litellm_params: Mapping[str, object] | None, model: str) -> CostPerToken | None: - """The pricing the router registered for this deployment under its ``model_info.id``: the id - is read with the router's own ``routed_deployment_id`` from ``Logging.litellm_params`` (never - from the flattened ``model_call_details`` copy a request body key can overwrite) and priced - through ``deployment_pricing_model_info`` as the success path does; None means the cost map.""" - model_info: Final = deployment_pricing_model_info( - model_id=routed_deployment_id(litellm_params) if litellm_params is not None else None, - deployment_model=model, + return estimated_usage, _estimated_input_cost( + model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model ) + + +def _deployment_cost_per_token(deployment_id: str | None, model: str) -> CostPerToken | None: + """The pricing the router registered for this deployment under its ``model_info.id``, read + through ``deployment_pricing_model_info`` exactly as the success path does; None means the + public cost map. The id is ``Logging.get_router_model_id()`` (the logging object's own + ``litellm_params``), never the flattened ``model_call_details`` copy a request body key can + overwrite.""" + model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model) if model_info is None: return None return CostPerToken( @@ -1084,17 +1088,20 @@ def _deployment_cost_per_token(litellm_params: Mapping[str, object] | None, mode def _estimated_input_cost( model_call_details: Mapping[str, object], prompt_tokens: int, - litellm_params: Mapping[str, object] | None, + deployment_id: str | None, + base_model: str | None, ) -> float: - """Input price for the estimated prompt; an unpriceable model (not in the cost map and no - deployment rate) keeps the row at $0 rather than dropping the failure row.""" + """Input price for the estimated prompt: the deployment's registered rate when it has one, else + the cost map's rate for ``base_model`` or the deployment model (an Azure deployment name is not + in the cost map, its ``base_model`` is). An unpriceable model keeps the row at $0 rather than + dropping the failure row.""" try: model: Final = str(model_call_details.get("model") or "") estimated_input_cost, _ = litellm.cost_per_token( - model=model, + model=base_model or model, custom_llm_provider=str(model_call_details.get("custom_llm_provider") or "") or None, prompt_tokens=prompt_tokens, - custom_cost_per_token=_deployment_cost_per_token(litellm_params, model), + custom_cost_per_token=_deployment_cost_per_token(deployment_id, model), ) except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written verbose_proxy_logger.debug( @@ -1263,7 +1270,8 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: request_body=request_data, dispatched=_first_handoff is not None, provider_billed=provider_billed, - litellm_params=getattr(_logging_obj, "litellm_params", None), + deployment_id=_logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None, + base_model=get_base_model_from_metadata(model_call_details=_model_call_details), ) _entries: Final = ( ("start_time", _model_call_details.get("start_time")), diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 70597e915d1..e499bb3785a 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2042,7 +2042,7 @@ def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the request_body={}, dispatched=True, provider_billed=True, - litellm_params={"model": f"openai/{FAILURE_USAGE_MODEL}", "metadata": {"model_info": {"id": "dep-lit145"}}}, + deployment_id="dep-lit145", ) assert lifted is not None @@ -2051,33 +2051,69 @@ def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the assert response_cost == pytest.approx(usage.prompt_tokens * 0.25) -def test_the_deployment_id_is_resolved_like_the_success_path_litellm_metadata_first(monkeypatch): - """On /messages and /responses the router stamps the real deployment id into litellm_metadata; - a model_info on metadata (possibly client-supplied) must not pick the rate.""" - from litellm.proxy.utils import _failure_usage_to_lift +def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object: + """A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params() + shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff + instant, so the lift treats the request as dispatched.""" + from litellm.litellm_core_utils.litellm_logging import Logging - for dep, rate in (("dep-lit145-real", 0.25), ("dep-lit145-wrong", 9.0)): - monkeypatch.setitem( - litellm.model_cost, - dep, - {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": rate, "output_cost_per_token": rate}, - ) - - lifted = _failure_usage_to_lift( - model_call_details={"call_type": "acompletion", "model": FAILURE_USAGE_MODEL, "messages": ONE_USER_MESSAGE}, - request_body={}, - dispatched=True, - provider_billed=True, - litellm_params={ - "model": f"openai/{FAILURE_USAGE_MODEL}", - "metadata": {"model_info": {"id": "dep-lit145-wrong"}}, - "litellm_metadata": {"model_info": {"id": "dep-lit145-real"}}, - }, + logging_obj = Logging( + model=model, + messages=ONE_USER_MESSAGE, + stream=False, + call_type="acompletion", + start_time=real_datetime.datetime.now(), + litellm_call_id="call-lit145", + function_id="fn-lit145", ) + logging_obj.update_environment_variables(model=model, user="", optional_params={}, litellm_params=litellm_params) + logging_obj.model_call_details["first_api_call_start_time"] = real_datetime.datetime.now() + logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE + logging_obj.model_call_details["call_type"] = "acompletion" + return logging_obj - assert lifted is not None - usage, response_cost = lifted - assert response_cost == pytest.approx(usage.prompt_tokens * 0.25) + +def test_the_deployment_rate_is_found_on_the_logging_objects_real_litellm_params(monkeypatch): + """``get_litellm_params()`` always carries a ``litellm_metadata`` key (None on chat + completions) next to the router-stamped ``metadata.model_info.id``; the failure row must + still price at the deployment's own rate, like ``Logging.get_router_model_id`` on the + success path.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145-real-shape", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 0.25}, + ) + litellm_params = get_litellm_params(metadata={"model_info": {"id": "dep-lit145-real-shape"}}) + assert "litellm_metadata" in litellm_params and litellm_params["litellm_metadata"] is None + logging_obj = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, litellm_params) + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + assert lifted["response_cost"] == pytest.approx(lifted["combined_usage_object"].prompt_tokens * 0.25) + + +def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path(): + """An Azure deployment name is not in the cost map; the success path prices it through + ``model_info.base_model`` (``get_base_model_from_metadata``) and so must the failure row.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + "azure/lit145-not-in-cost-map", + get_litellm_params(metadata={"model_info": {"base_model": "azure/gpt-4.1"}}), + ) + logging_obj.model_call_details["custom_llm_provider"] = "azure" + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + expected_cost, _ = litellm.cost_per_token( + model="azure/gpt-4.1", prompt_tokens=lifted["combined_usage_object"].prompt_tokens + ) + assert expected_cost > 0 + assert lifted["response_cost"] == pytest.approx(expected_cost) def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price(): @@ -2096,7 +2132,7 @@ def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price( request_body={}, dispatched=True, provider_billed=True, - litellm_params={"model": f"openai/{FAILURE_USAGE_MODEL}"}, + deployment_id=None, ) assert lifted is not None @@ -2377,9 +2413,7 @@ def test_create_model_info_response_resolves_mode_through_deployment_model(): ] ) - response = create_model_info_response( - model_id="my-embeddings", provider="openai", llm_router=router - ) + response = create_model_info_response(model_id="my-embeddings", provider="openai", llm_router=router) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2409,12 +2443,8 @@ def test_create_model_info_response_resolves_model_group_alias_to_target(model_g model_group_alias=model_group_alias, ) - alias_response = create_model_info_response( - model_id="team-embeddings", provider="openai", llm_router=router - ) - target_response = create_model_info_response( - model_id="my-embeddings", provider="openai", llm_router=router - ) + alias_response = create_model_info_response(model_id="team-embeddings", provider="openai", llm_router=router) + target_response = create_model_info_response(model_id="my-embeddings", provider="openai", llm_router=router) assert alias_response["id"] == "team-embeddings" for field in ("mode", "max_input_tokens", "max_output_tokens"): @@ -2497,7 +2527,9 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data={"metadata": {}}, - original_exception=HTTPException(status_code=400, detail="Upstream passthrough request failed with status 400"), + original_exception=HTTPException( + status_code=400, detail="Upstream passthrough request failed with status 400" + ), user_api_key_dict=UserAPIKeyAuth(), traceback_str=upstream_traceback, ) @@ -2561,9 +2593,13 @@ def test_mcp_conversion_preserves_request_policy_and_isolates_guardrail_data(buc parent = { "model": "parent-model", bucket: { - "guardrails": ["policy-rule"], "guardrail_config": {"language": "en"}, - "applied_policies": ["parent-policy"], "policy_sources": {"parent-policy": "model"}, - "_guardrail_pipelines": [], "_pipeline_managed_guardrails": ["pipeline-rule"], "tags": ["review"], + "guardrails": ["policy-rule"], + "guardrail_config": {"language": "en"}, + "applied_policies": ["parent-policy"], + "policy_sources": {"parent-policy": "model"}, + "_guardrail_pipelines": [], + "_pipeline_managed_guardrails": ["pipeline-rule"], + "tags": ["review"], }, "guardrails": [{"request-rule": {"extra_body": {"threshold": 0.9}}}], "guardrail_config": {"entities": ["EMAIL_ADDRESS"]}, @@ -2593,13 +2629,26 @@ def test_mcp_conversion_honors_only_authenticated_global_guardrail_opt_outs(opt_ from litellm.responses.mcp.request_context import MCPRequestContext auth = UserAPIKeyAuth(metadata={"opted_out_global_guardrails": ["global-rule"] if opt_out else []}) - context = MCPRequestContext.resolve(kwargs={"metadata": { - "user_api_key_auth": auth, "disable_global_guardrails": True, - "user_api_key_metadata": {"disable_global_guardrails": True}, - }}, tools=None) + context = MCPRequestContext.resolve( + kwargs={ + "metadata": { + "user_api_key_auth": auth, + "disable_global_guardrails": True, + "user_api_key_metadata": {"disable_global_guardrails": True}, + } + }, + tools=None, + ) proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) - kwargs = {"name": "execute", "arguments": {}, "user_api_key_auth": auth, "guardrail_context": context.guardrail_context} - synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) + kwargs = { + "name": "execute", + "arguments": {}, + "user_api_key_auth": auth, + "guardrail_context": context.guardrail_context, + } + synthetic = proxy_logging._convert_mcp_to_llm_format( + proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs + ) guardrail = CustomGuardrail(guardrail_name="global-rule", event_hook="pre_mcp_call", default_on=True) assert guardrail.should_run_guardrail(synthetic, GuardrailEventHooks.pre_mcp_call) is (not opt_out) synthetic["metadata"]["user_api_key_metadata"]["opted_out_global_guardrails"].append("unrelated") @@ -2613,19 +2662,26 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyGuardrails registry = policy_registry.PolicyRegistry() - registry._policies = {"model-policy": Policy( - condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"]) - )} + registry._policies = { + "model-policy": Policy( + condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"]) + ) + } registry._initialized = True monkeypatch.setattr(policy_registry, "_policy_registry", registry) monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) kwargs = { - "name": "execute", "arguments": {}, + "name": "execute", + "arguments": {}, "user_api_key_auth": UserAPIKeyAuth(metadata={"policies": ["model-policy"]}), - "guardrail_context": MCPRequestContext.resolve_guardrail_context({"model": model, "guardrails": ["request-rule"]}), + "guardrail_context": MCPRequestContext.resolve_guardrail_context( + {"model": model, "guardrails": ["request-rule"]} + ), } - synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) + synthetic = proxy_logging._convert_mcp_to_llm_format( + proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs + ) assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected assert "request-rule" in synthetic["metadata"]["guardrails"]