mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-27 06:13:07 +00:00
fix(responses): honor per-call custom pricing on the native Responses API path (#48119)
* fix(responses): honor per-call custom pricing on the native Responses API path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): narrow the custom pricing regression test to the type gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(main): expose register_custom_pricing_for_request for the responses path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): mark custom pricing test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): price per-call custom rates request-locally instead of registering them globally Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): bind per-call rates on the request logging object instead of inferring them in Logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): keep Router deployments on the cost-map pricing path Router inherits cache rates onto the deployment's cost-map entry, so a routed Responses call binds no request-local price and keeps billing cached tokens at the inherited rate. Moves the deployment-id lookup to router_utils so the Responses and completion paths share it. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): gate request-local pricing on the typed deployment model_info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
joshua-berri
GitHub
parent
f29e10c0ef
commit
e04e89c701
4 changed files with 199 additions and 13 deletions
|
|
@ -2264,6 +2264,8 @@ def response_cost_calculator(
|
|||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2292,6 +2294,8 @@ def response_cost_calculator(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
|
|
|
|||
|
|
@ -161,6 +161,7 @@ from litellm.types.utils import (
|
|||
CachingDetails,
|
||||
CallTypes,
|
||||
CostBreakdown,
|
||||
CostPerToken,
|
||||
CustomPricingLiteLLMParams,
|
||||
DynamicPromptManagementParamLiteral,
|
||||
EmbeddingResponse,
|
||||
|
|
@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prometheusLogger, \
|
||||
slack_app
|
||||
custom_pricing: bool = False
|
||||
custom_cost_per_token: CostPerToken | None = None
|
||||
custom_cost_per_second: float | None = None
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
|
|
@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"call_type": self.call_type,
|
||||
"optional_params": self.optional_params,
|
||||
"custom_pricing": custom_pricing,
|
||||
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
|
||||
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
|
||||
"prompt": prompt,
|
||||
"standard_built_in_tools_params": self.standard_built_in_tools_params,
|
||||
"router_model_id": router_model_id,
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ from contextlib import contextmanager
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from typing_extensions import (
|
||||
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
|
||||
assert_never,
|
||||
|
|
@ -87,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
|
|||
from litellm.types.responses.main import *
|
||||
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
|
||||
from litellm.types.utils import (
|
||||
ADDRESSED_RESPONSE_ID_FIELD,
|
||||
CallTypes,
|
||||
CostPerToken,
|
||||
CustomPricingLiteLLMParams,
|
||||
all_litellm_params,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
|
|
@ -546,6 +552,47 @@ async def aresponses_api_with_mcp(
|
|||
return response
|
||||
|
||||
|
||||
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
|
||||
return MappingProxyType(
|
||||
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
|
||||
)
|
||||
|
||||
|
||||
class _DeploymentModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
id: object = None
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
|
||||
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
|
||||
)
|
||||
|
||||
|
||||
def _is_router_deployment(deployment_model_info: object) -> bool:
|
||||
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
|
||||
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
|
||||
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
|
||||
|
||||
|
||||
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
|
||||
|
||||
|
||||
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
|
||||
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
|
||||
return None
|
||||
return _COST_PER_TOKEN.validate_python(
|
||||
litellm_params.model_dump(
|
||||
include={
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
},
|
||||
exclude_none=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _bridges_to_chat_completions(
|
||||
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
|
||||
) -> bool:
|
||||
|
|
@ -1673,6 +1720,7 @@ def responses(
|
|||
optional_params=dict(responses_api_request_params),
|
||||
litellm_params={
|
||||
**responses_api_request_params,
|
||||
**_per_call_custom_pricing(litellm_params),
|
||||
"aresponses": _is_async,
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": kwargs.get("model_info"),
|
||||
|
|
@ -1683,6 +1731,9 @@ def responses(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
if not _is_router_deployment(deployment_model_info):
|
||||
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
|
||||
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
|
||||
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
|
||||
input = with_compaction_items_as_summaries(
|
||||
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ import asyncio
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
import math
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from pathlib import Path
|
||||
from importlib import import_module
|
||||
from typing import Final
|
||||
|
|
@ -17,10 +18,12 @@ from unittest.mock import AsyncMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from tests.unit.llms.bedrock.slow_upstream import (
|
||||
STREAM_TIMEOUT_SECONDS,
|
||||
UPSTREAM_FIRST_BYTE_SECONDS,
|
||||
|
|
@ -333,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up(
|
|||
|
||||
|
||||
def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]:
|
||||
return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False)
|
||||
return tuple(
|
||||
level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False
|
||||
)
|
||||
|
||||
|
||||
def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]:
|
||||
|
|
@ -623,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_
|
|||
from litellm.responses.main import _aresponses_websocket
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -645,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs(
|
|||
from litellm.responses.main import _aresponses_websocket
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -671,7 +678,9 @@ _ORIGINAL_WS_INPUT = [
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket
|
||||
async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(
|
||||
nested: bool,
|
||||
): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.responses.main import _aresponses_websocket
|
||||
|
|
@ -682,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -711,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi
|
|||
first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}'
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -968,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo
|
|||
Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped."""
|
||||
monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com")
|
||||
injected_client = AsyncHTTPHandler()
|
||||
mock_post = AsyncMock(
|
||||
return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)
|
||||
)
|
||||
mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200))
|
||||
injected_client.post = mock_post
|
||||
|
||||
await litellm.aresponses(
|
||||
|
|
@ -1234,6 +1243,77 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) ->
|
|||
]
|
||||
|
||||
|
||||
async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse:
|
||||
response: Final = (
|
||||
litellm.responses(model=model, input="hi", **pricing)
|
||||
if sync_mode
|
||||
else await litellm.aresponses(model=model, input="hi", **pricing)
|
||||
)
|
||||
assert isinstance(response, ResponsesAPIResponse)
|
||||
return response
|
||||
|
||||
|
||||
def _response_cost(response: ResponsesAPIResponse) -> float:
|
||||
hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params")
|
||||
return hidden_params["response_cost"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync_mode", (True, False))
|
||||
@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map"))
|
||||
async def test_responses_bills_per_call_custom_pricing_from_usage(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str
|
||||
):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(model_cost))
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1]))
|
||||
)
|
||||
pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5}
|
||||
|
||||
response: Final = await _invoke_responses(sync_mode, model, pricing)
|
||||
|
||||
assert response.usage is not None
|
||||
expected: Final = (
|
||||
response.usage.input_tokens * pricing["input_cost_per_token"]
|
||||
+ response.usage.output_tokens * pricing["output_cost_per_token"]
|
||||
)
|
||||
assert math.isclose(_response_cost(response), expected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_responses_calls_keep_their_own_per_call_prices(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
model: Final = "openai/gpt-5.4-nano"
|
||||
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
|
||||
cost_map_row_before: Final = copy.deepcopy(model_cost.get(model))
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano"))
|
||||
)
|
||||
rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5))
|
||||
|
||||
responses: Final = await asyncio.gather(
|
||||
*(
|
||||
_invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate})
|
||||
for in_rate, out_rate in rates
|
||||
)
|
||||
)
|
||||
|
||||
for response, (in_rate, out_rate) in zip(responses, rates, strict=True):
|
||||
assert response.usage is not None
|
||||
assert math.isclose(
|
||||
_response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate
|
||||
)
|
||||
assert model_cost.get(model) == cost_map_row_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1339,3 +1419,49 @@ async def test_responses_opencode_session_uses_effective_destination(
|
|||
assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123")
|
||||
assert sent.headers["authorization"] == "Bearer test-key"
|
||||
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"}
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(model_cost))
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nano",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates},
|
||||
"model_info": {"id": "nano-deployment"},
|
||||
}
|
||||
]
|
||||
)
|
||||
payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano")
|
||||
usage: Final = _JSON_OBJECT.validate_python(payload["usage"])
|
||||
cached_tokens: Final = 8
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = await router.aresponses(model="nano", input="hi")
|
||||
|
||||
assert isinstance(response, ResponsesAPIResponse)
|
||||
assert response.usage is not None
|
||||
uncached_input: Final = response.usage.input_tokens - cached_tokens
|
||||
full_price: Final = (
|
||||
response.usage.input_tokens * rates["input_cost_per_token"]
|
||||
+ response.usage.output_tokens * rates["output_cost_per_token"]
|
||||
)
|
||||
floor: Final = (
|
||||
uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"]
|
||||
)
|
||||
assert floor < _response_cost(response) < full_price
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue