scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix(main)!: carry the full per-call price entry on the logging object instead of writing litellm.model_cost

Browse source 
Direct SDK calls build one ModelInfo from the cost map row plus every custom pricing field on the call and price from it, so audio, tiered and directional per-second rates apply without touching litellm.model_cost. Router calls keep their registered deployment entry. completion_cost(completion_response=...) alone no longer sees per-call prices, pass custom_cost_per_second or custom_cost_per_token
This commit is contained in:
devin-ai-integration[bot] 2026-11-01 00:17:15 +00:00
parent 5aebe2363d
commit 791c239503
6 changed files with 209 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
litellm/cost_calculator.py
View file

@ -382,6 +382,14 @@ def _per_second_pricing_cost(
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds)
def _per_second_pricing_cost_from_info(
model_info: ModelInfoBase,
response_time_ms: float | None,
audio_seconds: float = 0.0,
) -> tuple[float, float] | None:
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
return None
cost_per_second: Final = model_info.get("cost_per_second")
@ -404,7 +412,7 @@ def _per_second_pricing_cost(
)
verbose_logger.debug(
"For model=%s - cost_per_second: %s; response time: %s",
model,
model_info.get("key"),
resolved_cost_per_second,
response_time_ms,
)
@ -427,6 +435,7 @@ def cost_per_token(
### CUSTOM PRICING ###
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
per_call_model_info: ModelInfo | None = None,
### NUMBER OF QUERIES ###
number_of_queries: int | None = None,
### USAGE OBJECT ###
@ -538,6 +547,20 @@ def cost_per_token(
if response_cost is not None:
return response_cost[0], response_cost[1]
if per_call_model_info is not None:
per_second: Final = _per_second_pricing_cost_from_info(
per_call_model_info, response_time_ms, audio_transcription_file_duration
)
if per_second is not None:
return per_second
return generic_cost_per_token(
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider or "",
service_tier=service_tier,
model_info=per_call_model_info,
)
model_cost_ref: Final = litellm.model_cost
# Only callers that explicitly pass `custom_llm_provider` get the
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
@ -1668,6 +1691,7 @@ def completion_cost(
### CUSTOM PRICING ###
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
per_call_model_info: ModelInfo | None = None,
optional_params: dict | None = None,
custom_pricing: bool | None = None,
base_model: str | None = None,
@ -1721,6 +1745,7 @@ def completion_cost(
quality=quality,
n=n,
custom_cost_per_token=custom_cost_per_token,
per_call_model_info=per_call_model_info,
custom_cost_per_second=custom_cost_per_second,
optional_params=optional_params,
custom_pricing=custom_pricing,
@ -2206,6 +2231,7 @@ def completion_cost(
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
per_call_model_info=per_call_model_info,
prompt_characters=prompt_characters,
completion_characters=completion_characters,
cache_creation_input_tokens=cache_creation_input_tokens,
@ -2438,6 +2464,7 @@ def response_cost_calculator(
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
per_call_model_info: ModelInfo | None = None,
) -> float:
"""
Returns
@ -2494,6 +2521,7 @@ def response_cost_calculator(
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
per_call_model_info=per_call_model_info,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,

6
litellm/litellm_core_utils/litellm_logging.py
View file

@ -706,6 +706,7 @@ class Logging(LiteLLMLoggingBaseClass):
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
custom_model_info: ModelInfo | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -2205,6 +2206,7 @@ class Logging(LiteLLMLoggingBaseClass):
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"per_call_model_info": self.custom_model_info if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
@ -2401,6 +2403,10 @@ class Logging(LiteLLMLoggingBaseClass):
additional_headers=additional_headers,
)
def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None:
"""Price this call from its own full price entry instead of ``litellm.model_cost``."""
self.custom_model_info = custom_model_info
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False

69
litellm/main.py
View file

@ -162,7 +162,8 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s
from litellm.types.utils import (
AgenticLoopParams,
BaseUrlRoutingHint,
CostPerToken,
CustomPricingLiteLLMParams,
ModelInfo,
ModelResponseStream,
ProviderRoutingHints,
RawRequestTypedDict,
@ -1421,21 +1422,54 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
return values
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
_METADATA_MODEL_INFO_ID: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO: Final = TypeAdapter(ModelInfo)
def _per_call_cost_per_token(kwargs: Mapping[str, object]) -> CostPerToken | None:
"""The flat per-token prices a caller put on this one call, priced from the logging object and never written
into ``litellm.model_cost``, so the next call for the same model name is priced from the cost map again."""
if kwargs.get("input_cost_per_token") is None or kwargs.get("output_cost_per_token") is None:
def _is_router_call(kwargs: Mapping[str, object]) -> bool:
"""A Router stamps ``model_info.id`` into the call's metadata and registers that deployment's full custom pricing
under that id when the deployment is added, so the cost calculator prices those calls from the registered entry."""
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key)
if not isinstance(metadata, Mapping):
continue
model_info = _METADATA_MODEL_INFO_ID.validate_python(metadata).get("model_info")
if isinstance(model_info, Mapping) and model_info.get("id") is not None:
return True
return False
def _per_call_model_info(
model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"]
) -> ModelInfo | None:
"""The full price entry for one call: the cost map row for the model, if any, overlaid with every custom pricing
field the caller put on the call. It travels on the logging object and is never written into ``litellm.model_cost``,
so the next call for the same model name is priced from the cost map again."""
custom: Final = {
field: kwargs[field] for field in CustomPricingLiteLLMParams.model_fields if kwargs.get(field) is not None
}
if not custom:
return None
unmapped: Final[Mapping[str, object]] = {
"key": model,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": None,
"output_cost_per_token": None,
"litellm_provider": custom_llm_provider,
"mode": mode,
"supported_openai_params": None,
}
catalog: Final = _catalog_row(model, custom_llm_provider) or unmapped
return _MODEL_INFO.validate_python({**catalog, **custom})
def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None:
try:
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
return None
keys: Final = (
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
)
return _COST_PER_TOKEN.validate_python({key: kwargs[key] for key in keys if kwargs.get(key) is not None})
def _record_agentic_loop_params(
@ -5902,8 +5936,8 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
logging.custom_cost_per_token = _per_call_cost_per_token(kwargs)
logging.custom_cost_per_second = cost_per_second
if not _is_router_call(kwargs):
logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "chat"))
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
custom_prompt_dict = {}
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
@ -6910,7 +6944,6 @@ def embedding(
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
cost_per_second: Final = kwargs.get("cost_per_second", None)
openai_params: Final = [
"user",
"dimensions",
@ -6958,8 +6991,8 @@ def embedding(
)
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
logging.custom_cost_per_token = _per_call_cost_per_token(kwargs)
logging.custom_cost_per_second = cost_per_second
if not _is_router_call(kwargs):
logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "embedding"))
logging.update_environment_variables(
model=model,
user=user,

2
tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py
View file

@ -549,7 +549,7 @@ async def test_completion_sagemaker(
assert route.called
assert response.choices[0].message.content == "SageMaker reply"
assert litellm.completion_cost(completion_response=response) == pytest.approx(rate)
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate)
@pytest.mark.asyncio

121
tests/unit/test_register_model_custom_pricing.py
View file

@ -19,6 +19,14 @@ import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
from litellm.types.utils import (
Choices,
CompletionTokensDetailsWrapper,
Message,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -521,8 +529,8 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
"""The request that carries custom pricing is costed with that pricing from its own logging object, while
``litellm.model_cost`` keeps the built-in rate for siblings and gains no entry for the deployment."""
"""A Router registers the deployment's pricing under its id when the deployment is added. The routed request
is costed from that entry, and the shared backend key keeps the built-in rate for siblings."""
shared_key = "openai/text-embedding-3-small"
deployment_id = "lit3991-wildcard-embed-custom"
override_input_cost = 5e-05
@ -531,6 +539,16 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
assert builtin_input_cost != override_input_cost
try:
litellm.register_model(
{
deployment_id: {
"input_cost_per_token": override_input_cost,
"output_cost_per_token": override_input_cost * 2,
"litellm_provider": "openai",
"mode": "embedding",
}
}
)
response = litellm.embedding(
model=shared_key,
input=["hello"],
@ -545,7 +563,6 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
assert response._hidden_params["response_cost"] == pytest.approx(
response.usage.prompt_tokens * override_input_cost
)
assert deployment_id not in litellm.model_cost
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost
finally:
_restore_model_cost_entries(snapshot)
@ -1442,3 +1459,101 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
assert litellm.model_cost[builtin_key] == builtin_row_before
finally:
_restore_model_cost_entries(model_cost_entries)
def _audio_reply() -> ModelResponse:
usage: Final = Usage(
prompt_tokens=1000,
completion_tokens=1000,
total_tokens=2000,
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=1000, text_tokens=0),
completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=1000, text_tokens=0),
)
return ModelResponse(model="gpt-audio-2025-08-28", choices=[Choices(message=Message(content="hi"))], usage=usage)
_AUDIO_RATES: Final = {
"input_cost_per_token": 2.5e-06,
"output_cost_per_token": 1e-05,
"input_cost_per_audio_token": 4e-05,
"output_cost_per_audio_token": 8e-05,
}
def test_router_call_with_audio_rates_is_priced_per_modality_from_its_registered_deployment():
"""A Router registers the deployment's full custom pricing when the deployment is added. The per-call flat
prices must not override that entry, or 1000 audio input plus 1000 audio output tokens would bill at the text
rates ($0.0125) instead of the audio rates ($0.12)."""
deployment_id: Final = "audio-rates-deployment"
snapshot: Final = _snapshot_model_cost_entries([deployment_id])
try:
litellm.register_model({deployment_id: {**_AUDIO_RATES, "litellm_provider": "openai", "mode": "chat"}})
response: Final = litellm.completion(
model="openai/gpt-audio-2025-08-28",
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
metadata={"model_info": {"id": deployment_id}},
mock_response=_audio_reply(),
**_AUDIO_RATES,
)
assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05)
finally:
_restore_model_cost_entries(snapshot)
def test_direct_call_with_flat_rates_is_priced_from_them_without_writing_the_cost_map():
model_key: Final = "openai/lit-direct-flat-rates-model"
try:
response: Final = litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
mock_response=_audio_reply(),
input_cost_per_token=2.5e-06,
output_cost_per_token=1e-05,
)
assert response._hidden_params["response_cost"] == pytest.approx(1000 * 2.5e-06 + 1000 * 1e-05)
assert model_key not in litellm.model_cost
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()
def test_direct_call_with_audio_rates_is_priced_per_modality_without_writing_the_cost_map():
model_key: Final = "openai/lit-direct-audio-rates-model"
try:
response: Final = litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
mock_response=_audio_reply(),
input_cost_per_token=2.5e-06,
output_cost_per_token=1e-05,
input_cost_per_audio_token=4e-05,
output_cost_per_audio_token=8e-05,
)
assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05)
assert model_key not in litellm.model_cost
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()
def test_direct_call_with_directional_per_second_rates_is_priced_from_them_without_writing_the_cost_map():
model_key: Final = "openai/lit-direct-per-second-model"
try:
response: Final = litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
mock_response="ok",
input_cost_per_second=0.004,
)
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.004) == pytest.approx(
0.004 * response._response_ms / 1000
)
assert response._hidden_params["response_cost"] == pytest.approx(0.004 * response._response_ms / 1000, rel=0.2)
assert model_key not in litellm.model_cost
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()

7
tests/unit/test_utils.py
View file

@ -2333,7 +2333,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
@respx.mock
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_completion_registers_test_owned_model_prices() -> None:
def test_completion_custom_prices_price_the_call_without_writing_the_cost_map() -> None:
model: Final = "migration-owned-pricing-model"
messages: Final = [{"role": "user", "content": "price control"}]
litellm.register_model(
@ -2376,8 +2376,9 @@ def test_completion_registers_test_owned_model_prices() -> None:
)
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
assert litellm.model_cost[model]["input_cost_per_token"] == 0.001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.002
assert response._hidden_params["response_cost"] == pytest.approx(8 * 0.001 + 5 * 0.002)
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
assert response._hidden_params["response_cost"] == pytest.approx(0.018)