From e04e89c7010809cc051b14f0e43d71dd018cdaee Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Fri, 23 Oct 2026 22:53:04 -0700 Subject: [PATCH] fix(responses): honor per-call custom pricing on the native Responses API path (#48119) * fix(responses): honor per-call custom pricing on the native Responses API path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): narrow the custom pricing regression test to the type gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(main): expose register_custom_pricing_for_request for the responses path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): mark custom pricing test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): price per-call custom rates request-locally instead of registering them globally Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): bind per-call rates on the request logging object instead of inferring them in Logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): keep Router deployments on the cost-map pricing path Router inherits cache rates onto the deployment's cost-map entry, so a routed Responses call binds no request-local price and keeps billing cached tokens at the inherited rate. Moves the deployment-id lookup to router_utils so the Responses and completion paths share it. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): gate request-local pricing on the typed deployment model_info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 4 + litellm/litellm_core_utils/litellm_logging.py | 5 + litellm/responses/main.py | 57 ++++++- .../test_responses_api_request_body.py | 146 ++++++++++++++++-- 4 files changed, 199 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..99944662bf8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2264,6 +2264,8 @@ def response_cost_calculator( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") region_name: str | None = None, + custom_cost_per_token: CostPerToken | None = None, + custom_cost_per_second: float | None = None, ) -> float: """ Returns @@ -2292,6 +2294,8 @@ def response_cost_calculator( optional_params=optional_params, custom_pricing=custom_pricing, base_model=base_model, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, litellm_model_name=litellm_model_name, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d75d35e4531..41f13817d75 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -161,6 +161,7 @@ from litellm.types.utils import ( CachingDetails, CallTypes, CostBreakdown, + CostPerToken, CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, EmbeddingResponse, @@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass): prometheusLogger, \ slack_app custom_pricing: bool = False + custom_cost_per_token: CostPerToken | None = None + custom_cost_per_second: float | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass): "call_type": self.call_type, "optional_params": self.optional_params, "custom_pricing": custom_pricing, + "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, + "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 0b526adf4fa..e7380c26529 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,10 +6,10 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from typing_extensions import ( TypeIs, # noqa: TID251 # narrows a client input item before its mark is read assert_never, @@ -87,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.types.responses.main import * from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params +from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, + CallTypes, + CostPerToken, + CustomPricingLiteLLMParams, + all_litellm_params, +) from litellm.utils import ( ProviderConfigManager, client, @@ -546,6 +552,47 @@ async def aresponses_api_with_mcp( return response +def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: + return MappingProxyType( + litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + +class _DeploymentModelInfo(BaseModel): + model_config = ConfigDict(frozen=True) + id: object = None + + +_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter( + Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")] +) + + +def _is_router_deployment(deployment_model_info: object) -> bool: + """A Router stamps the deployment's model_info (with its id) onto every call it routes.""" + parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info) + return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None + + +_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) + + +def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None: + if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None: + return None + return _COST_PER_TOKEN.validate_python( + litellm_params.model_dump( + include={ + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + }, + exclude_none=True, + ) + ) + + def _bridges_to_chat_completions( responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool ) -> bool: @@ -1673,6 +1720,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, + **_per_call_custom_pricing(litellm_params), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -1683,6 +1731,9 @@ def responses( custom_llm_provider=custom_llm_provider, ) + if not _is_router_deployment(deployment_model_info): + litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) + litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second # Decode any litellm-encoded encrypted-content item IDs back to their original IDs input = with_compaction_items_as_summaries( with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 36b5d8f135b..bf571c74e22 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -8,7 +8,8 @@ import asyncio import copy import json import logging -from collections.abc import Awaitable, Callable +import math +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path from importlib import import_module from typing import Final @@ -17,10 +18,12 @@ from unittest.mock import AsyncMock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.llms.openai import ResponsesAPIResponse from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, @@ -333,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up( def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]: - return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False) + return tuple( + level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False + ) def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]: @@ -623,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -645,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs( from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -671,7 +678,9 @@ _ORIGINAL_WS_INPUT = [ @pytest.mark.asyncio @pytest.mark.parametrize("nested", [False, True]) -async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket +async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame( + nested: bool, +): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket from unittest.mock import MagicMock from litellm.responses.main import _aresponses_websocket @@ -682,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame ) with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -711,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}' with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -968,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped.""" monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com") injected_client = AsyncHTTPHandler() - mock_post = AsyncMock( - return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200) - ) + mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)) injected_client.post = mock_post await litellm.aresponses( @@ -1234,6 +1243,77 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) -> ] +async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse: + response: Final = ( + litellm.responses(model=model, input="hi", **pricing) + if sync_mode + else await litellm.aresponses(model=model, input="hi", **pricing) + ) + assert isinstance(response, ResponsesAPIResponse) + return response + + +def _response_cost(response: ResponsesAPIResponse) -> float: + hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params") + return hidden_params["response_cost"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map")) +async def test_responses_bills_per_call_custom_pricing_from_usage( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1])) + ) + pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + + response: Final = await _invoke_responses(sync_mode, model, pricing) + + assert response.usage is not None + expected: Final = ( + response.usage.input_tokens * pricing["input_cost_per_token"] + + response.usage.output_tokens * pricing["output_cost_per_token"] + ) + assert math.isclose(_response_cost(response), expected) + + +@pytest.mark.asyncio +async def test_concurrent_responses_calls_keep_their_own_per_call_prices( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + model: Final = "openai/gpt-5.4-nano" + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + cost_map_row_before: Final = copy.deepcopy(model_cost.get(model)) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano")) + ) + rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5)) + + responses: Final = await asyncio.gather( + *( + _invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate}) + for in_rate, out_rate in rates + ) + ) + + for response, (in_rate, out_rate) in zip(responses, rates, strict=True): + assert response.usage is not None + assert math.isclose( + _response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate + ) + assert model_cost.get(model) == cost_map_row_before + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize( @@ -1339,3 +1419,49 @@ async def test_responses_opencode_session_uses_effective_destination( assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") assert sent.headers["authorization"] == "Bearer test-key" assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"} + + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.asyncio +async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "nano", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates}, + "model_info": {"id": "nano-deployment"}, + } + ] + ) + payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano") + usage: Final = _JSON_OBJECT.validate_python(payload["usage"]) + cached_tokens: Final = 8 + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}}, + ) + ) + + response: Final = await router.aresponses(model="nano", input="hi") + + assert isinstance(response, ResponsesAPIResponse) + assert response.usage is not None + uncached_input: Final = response.usage.input_tokens - cached_tokens + full_price: Final = ( + response.usage.input_tokens * rates["input_cost_per_token"] + + response.usage.output_tokens * rates["output_cost_per_token"] + ) + floor: Final = ( + uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"] + ) + assert floor < _response_cost(response) < full_price