mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(spend): keep a NaN or infinite cost from wedging daily spend writes (#48829)
* fix(spend): keep a NaN or infinite cost from wedging daily spend writes A non-finite provider-reported cost now counts as not reported, so the token price bills the request. Any other non-finite cost is logged and recorded as 0, and spend log JSON writes NaN and Infinity as null so Postgres accepts the row. * fix(spend): price non-finite provider costs on every route and drain parked NaN batches Perplexity, OpenRouter pass-through, and Responses streaming now treat a NaN or infinite reported cost as unreported, so the token price bills the request. A daily spend batch parked in the Redis buffer by an older version with a NaN spend now drains as 0 instead of failing every flush. The cost header never carries NaN or Infinity, and a set holding a non-finite float no longer breaks spend log JSON * fix(spend): record non-finite savings as 0 and warn when a parked NaN batch drains An overflowing price on a compressed request made compression savings infinite, which still wedged the daily spend batch. Every savings column now records a non-finite value as 0 with a warning. The Redis drain warns when it reads a NaN an older version parked, since the spend summed into it is lost from daily spend. The writer test now runs the real payload builder and spend log insert * fix(spend): keep non-finite costs out of headers, responses, and admin JSON Writing NaN as null is now opt-in for spend log columns, the Redis buffer, and streamed frames, so admin writes and logging integrations keep their output. The cost calculator records an overflowing total as 0 with a warning, cost headers leave out non-finite values, a non-streaming NaN usage.cost is dropped instead of failing the response, and a parked NaN spend log row reads back as 0. * fix(spend): keep caller-set call ids from splitting the non-finite cost warning The overflow warning strips newlines from litellm_call_id, and neither non-finite cost warning prints the raw value, which said nothing beyond "non-finite" * fix(spend): drop non-finite Responses and OpenRouter costs and the breakdown of an unpriceable request * fix(types): drop a usage cost breakdown that holds a non-finite number * fix(streaming): keep a dropped non-finite final usage cost from falling back to an earlier chunk's cost * fix(spend): keep the dropped-cost flag out of reach of provider usage keys A provider usage key named dropped_non_finite_cost shadowed the Usage method on the instance, so the stream builder raised and dropped the request's spend. The builder now reads the flag through the class, and Usage sets the flag after copying provider keys, so a _dropped_non_finite_cost key cannot set it either. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
quietjules
GitHub
parent
43c07a0f12
commit
df47ec0171
30 changed files with 906 additions and 51 deletions
|
|
@ -982,25 +982,36 @@ def _usage_field(usage: object, name: str) -> object:
|
|||
return getattr(usage, name, None)
|
||||
|
||||
|
||||
def _cost_number(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||
def finite_cost(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value):
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
|
||||
def single_log_line(value: object) -> str:
|
||||
return str(value).replace("\r", "").replace("\n", "")
|
||||
|
||||
|
||||
def finite_reported_cost(value: object) -> float | None:
|
||||
cost: Final = finite_cost(value)
|
||||
if cost is None and isinstance(value, float):
|
||||
verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value)
|
||||
return cost
|
||||
|
||||
|
||||
def _reported_cost_value(value: object) -> float | None:
|
||||
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
|
||||
raw: Final[object] = value
|
||||
if isinstance(value, Mapping):
|
||||
return _cost_number(_usage_field(raw, "total_cost"))
|
||||
return _cost_number(value)
|
||||
return finite_reported_cost(_usage_field(raw, "total_cost"))
|
||||
return finite_reported_cost(value)
|
||||
|
||||
|
||||
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
|
||||
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
|
||||
if not isinstance(additional_headers, Mapping):
|
||||
return None
|
||||
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
|
||||
return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
|
||||
|
||||
|
||||
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
|
|||
is_classifier_call,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
finite_cost,
|
||||
get_provider_response_headers_from_hidden_params,
|
||||
is_expected_client_error,
|
||||
native_messages_reported_cost,
|
||||
|
|
@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
proxy_stamped_used_client_oauth_token,
|
||||
reconstruct_model_name,
|
||||
set_response_cost_in_hidden_params,
|
||||
single_log_line,
|
||||
)
|
||||
from litellm.litellm_core_utils.error_normalization import normalize_error
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
|
|
@ -2093,7 +2095,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id or hidden_params.get("model_id"),
|
||||
)
|
||||
return hidden_params["response_cost"]
|
||||
return self._recorded_response_cost(hidden_params["response_cost"])
|
||||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
|
|
@ -2193,7 +2195,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
)
|
||||
return total_response_cost
|
||||
return self._recorded_response_cost(total_response_cost)
|
||||
except Exception as e: # error calculating cost
|
||||
debug_info = StandardLoggingModelCostFailureDebugInformation(
|
||||
error_str=str(e),
|
||||
|
|
@ -2219,6 +2221,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_response_cost_calculator = response_cost_calculator
|
||||
|
||||
def _recorded_response_cost(self, cost: float | None) -> float | None:
|
||||
if cost is None or finite_cost(cost) is not None:
|
||||
return cost
|
||||
verbose_logger.warning(
|
||||
"Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id)
|
||||
)
|
||||
self.cost_breakdown = None
|
||||
return 0.0
|
||||
|
||||
def uncached_price(self, result: object) -> UncachedPrice:
|
||||
previous_cost_breakdown: Final = self.cost_breakdown
|
||||
previous_billed_token_rates: Final = self.billed_token_rates
|
||||
|
|
@ -7582,7 +7593,15 @@ def get_standard_logging_object_payload(
|
|||
)
|
||||
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
|
||||
raw_response_cost: Final = kwargs.get("response_cost")
|
||||
llm_response_cost: Final[float] = raw_response_cost or 0.0
|
||||
numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None
|
||||
priced_response_cost: Final = finite_cost(numeric_response_cost)
|
||||
if numeric_response_cost is not None and priced_response_cost is None:
|
||||
verbose_logger.warning(
|
||||
"Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s",
|
||||
numeric_response_cost,
|
||||
single_log_line(logging_obj.litellm_call_id),
|
||||
)
|
||||
llm_response_cost: Final[float] = priced_response_cost or 0.0
|
||||
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
|
||||
finalized_cost: Final = _finalized_cost_of(logging_obj)
|
||||
response_cost: Final[float] = (
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Final
|
||||
|
||||
|
|
@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str:
|
|||
return value.replace("\x00", "")
|
||||
|
||||
|
||||
def _finite_or_none(value: float | None) -> float | None:
|
||||
return None if isinstance(value, float) and not math.isfinite(value) else value
|
||||
|
||||
|
||||
def safe_json_structure(
|
||||
data: object,
|
||||
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
||||
value_transform: Callable[[str | None, str], str] | None = None,
|
||||
key: str | None = None,
|
||||
*,
|
||||
non_finite_as_null: bool = False,
|
||||
) -> object:
|
||||
"""
|
||||
Rebuild data out of JSON-native pieces while detecting circular references.
|
||||
If a circular reference is detected then a marker string is returned.
|
||||
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
|
||||
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With
|
||||
non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them.
|
||||
|
||||
value_transform, when given, is applied to every string leaf (and to the
|
||||
str() fallback for non-serializable objects) with the mapping key the leaf
|
||||
|
|
@ -43,7 +51,7 @@ def safe_json_structure(
|
|||
cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
|
||||
return _transform(key, cleaned)
|
||||
if isinstance(obj, (int, float, bool, type(None))):
|
||||
return obj
|
||||
return _finite_or_none(obj) if non_finite_as_null else obj
|
||||
# Check for circular reference.
|
||||
if id(obj) in seen:
|
||||
return "CircularReference Detected"
|
||||
|
|
@ -65,8 +73,9 @@ def safe_json_structure(
|
|||
seen.remove(id(obj))
|
||||
return tuple_result
|
||||
elif isinstance(obj, set):
|
||||
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
set_result: Final = sorted(sortable_values)
|
||||
members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
present: Final = sorted(member for member in members if member is not None)
|
||||
set_result: Final = present + [None] * members.count(None)
|
||||
seen.remove(id(obj))
|
||||
return set_result
|
||||
elif isinstance(obj, BaseModel):
|
||||
|
|
@ -88,6 +97,10 @@ def safe_dumps(
|
|||
data: object,
|
||||
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
||||
value_transform: Callable[[str | None, str], str] | None = None,
|
||||
*,
|
||||
non_finite_as_null: bool = False,
|
||||
) -> str:
|
||||
"""Serialize data to JSON text through safe_json_structure."""
|
||||
return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
|
||||
return json.dumps(
|
||||
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.core_helpers import (
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
|
||||
chunk_usage,
|
||||
finite_cost,
|
||||
provider_reported_usage_cost,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
|
|
@ -1014,7 +1015,9 @@ class ChunkProcessor:
|
|||
prompt_tokens_details, cache_creation_token_details
|
||||
)
|
||||
|
||||
if usage_chunk_dict["cost"] is not None:
|
||||
if Usage.dropped_non_finite_cost(usage_chunk):
|
||||
cost = None
|
||||
elif usage_chunk_dict["cost"] is not None:
|
||||
cost = usage_chunk_dict["cost"]
|
||||
|
||||
prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
|
||||
|
|
@ -1444,8 +1447,8 @@ def _set_stream_builder_response_cost(
|
|||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
|
||||
getattr(usage, "cost", None), (int, float)
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and (
|
||||
finite_cost(getattr(usage, "cost", None)) is not None
|
||||
):
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import finite_cost
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig):
|
|||
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
|
||||
dict[str, object], hidden_params["additional_headers"]
|
||||
)
|
||||
if isinstance(response_cost, str | int | float):
|
||||
additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
|
||||
reported_cost: Final = (
|
||||
finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None
|
||||
)
|
||||
if reported_cost is not None:
|
||||
additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost
|
||||
except Exception:
|
||||
# If we can't extract cost, continue without it - don't fail the response
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
|
||||
from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler:
|
|||
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
|
||||
)
|
||||
pricing: Final = _pricing_for(candidate_model_keys)
|
||||
reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None
|
||||
response_cost: Final = (
|
||||
usage.cost
|
||||
if custom_llm_provider == "openrouter" and usage.cost is not None
|
||||
reported_cost
|
||||
if reported_cost is not None
|
||||
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
|
||||
)
|
||||
usage_object: Final = Usage(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation
|
|||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
|
@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
total_cost = cost_info.get("total_cost")
|
||||
elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
|
||||
total_cost = float(cost_info)
|
||||
if total_cost is not None:
|
||||
# Return total cost as completion_cost (prompt_cost=0) since Perplexity
|
||||
# doesn't break down by input/output in their cost object
|
||||
return (0.0, float(total_cost))
|
||||
reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost))
|
||||
if reported_cost is not None:
|
||||
return (0.0, reported_cost)
|
||||
|
||||
## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
|
||||
## GET MODEL INFO
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
strip_bug_report_notice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
finite_cost,
|
||||
get_or_create_metadata_bucket,
|
||||
independent_snapshot,
|
||||
is_expected_client_error,
|
||||
|
|
@ -1812,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj(
|
|||
|
||||
|
||||
def _header_value(value: float | None) -> str | None:
|
||||
return str(value) if value is not None else None
|
||||
return str(value) if finite_cost(value) is not None else None
|
||||
|
||||
|
||||
def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str:
|
||||
if hidden_params.get("cache_hit") is True:
|
||||
return 0.0
|
||||
response_cost: Final = hidden_params.get("response_cost")
|
||||
return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else ""
|
||||
if isinstance(response_cost, str):
|
||||
return response_cost
|
||||
return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else ""
|
||||
|
||||
|
||||
def cost_headers(
|
||||
|
|
@ -4509,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"""
|
||||
if isinstance(chunk, dict):
|
||||
# Use safe_dumps for proper JSON serialization with circular reference detection
|
||||
chunk_str: Final = safe_dumps(chunk)
|
||||
chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True)
|
||||
return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n"
|
||||
else:
|
||||
return chunk
|
||||
|
|
@ -4885,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
obj, model_name, litellm_logging_obj
|
||||
)
|
||||
if maybe_modified is not None:
|
||||
lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "")
|
||||
lines[idx] = (
|
||||
"data: "
|
||||
+ safe_dumps(maybe_modified, non_finite_as_null=True)
|
||||
+ ("\r" if ln.endswith("\r") else "")
|
||||
)
|
||||
return "\n".join(lines)
|
||||
return None
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.constants import (
|
|||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line
|
||||
from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -652,7 +653,14 @@ class DBSpendUpdateWriter:
|
|||
end_time=end_time,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
payload["spend"] = response_cost or 0.0
|
||||
billable_cost: Final = finite_cost(response_cost)
|
||||
if response_cost is not None and billable_cost is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
|
||||
response_cost,
|
||||
single_log_line(payload["request_id"]),
|
||||
)
|
||||
payload["spend"] = billable_cost or 0.0
|
||||
if isinstance(payload["startTime"], datetime):
|
||||
payload["startTime"] = payload["startTime"].isoformat()
|
||||
if isinstance(payload["endTime"], datetime):
|
||||
|
|
@ -702,7 +710,7 @@ class DBSpendUpdateWriter:
|
|||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
self._batch_database_updates(
|
||||
response_cost=response_cost,
|
||||
response_cost=billable_cost,
|
||||
user_id=user_id,
|
||||
hashed_token=hashed_token,
|
||||
team_id=team_id,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from functools import reduce
|
||||
from functools import partial, reduce
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
|
|
@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT")
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _zero_for_non_finite_constant(constant: str) -> float:
|
||||
verbose_proxy_logger.warning(
|
||||
"Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant
|
||||
)
|
||||
return 0.0
|
||||
|
||||
|
||||
_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant)
|
||||
|
||||
|
||||
def _spend_log_json_default(value: object) -> str:
|
||||
return value.isoformat() if isinstance(value, datetime) else str(value)
|
||||
|
||||
|
|
@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str:
|
|||
|
||||
|
||||
def _decode_spend_log_row(encoded: str) -> dict[str, object] | None:
|
||||
decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded))
|
||||
decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded))
|
||||
return decoded if isinstance(decoded, dict) else None
|
||||
|
||||
|
||||
|
|
@ -825,13 +835,13 @@ class RedisUpdateBuffer:
|
|||
if slot_result is None:
|
||||
daily_results.append(None)
|
||||
else:
|
||||
list_of_daily = [json.loads(t) for t in slot_result]
|
||||
list_of_daily = [_load_buffered_transaction(t) for t in slot_result]
|
||||
aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily)
|
||||
daily_results.append(aggregated)
|
||||
|
||||
window_spend: Final = (
|
||||
WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(
|
||||
tuple(json.loads(transaction) for transaction in raw_results[6])
|
||||
tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6])
|
||||
)
|
||||
if raw_results[6] is not None
|
||||
else None
|
||||
|
|
@ -880,7 +890,7 @@ class RedisUpdateBuffer:
|
|||
)
|
||||
if list_of_transactions is None:
|
||||
return None
|
||||
return [json.loads(transaction) for transaction in list_of_transactions]
|
||||
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
|
||||
|
||||
async def get_all_daily_spend_update_transactions_from_redis_buffer(
|
||||
self,
|
||||
|
|
@ -1010,9 +1020,9 @@ class RedisUpdateBuffer:
|
|||
Parses the list of transactions from Redis
|
||||
"""
|
||||
if isinstance(list_of_transactions, list):
|
||||
return [json.loads(transaction) for transaction in list_of_transactions]
|
||||
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
|
||||
else:
|
||||
return [json.loads(list_of_transactions)]
|
||||
return [_load_buffered_transaction(list_of_transactions)]
|
||||
|
||||
@staticmethod
|
||||
def _combine_list_of_transactions(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
budget_reservation_from_metadata,
|
||||
finite_cost,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
get_parent_otel_span_from_kwargs,
|
||||
|
|
@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
raw_cost: Final = _str_keyed_fields(
|
||||
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
|
||||
).get("response_cost")
|
||||
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
|
||||
priced_cost: Final = finite_cost(raw_cost)
|
||||
response_cost: Final = (
|
||||
await apply_search_free_tier(
|
||||
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
|
||||
|
|
|
|||
|
|
@ -689,9 +689,13 @@ def compute_savings_spend(
|
|||
cost_breakdown=cost_breakdown,
|
||||
)
|
||||
)
|
||||
return SavingsSpend(
|
||||
savings: Final = SavingsSpend(
|
||||
compression=compression,
|
||||
prompt_caching=prompt_caching,
|
||||
autorouter=0.0 if autorouter is None else autorouter,
|
||||
gateway_injected_caching=gateway_injected_caching,
|
||||
)
|
||||
if all(isfinite(value) for value in savings):
|
||||
return savings
|
||||
verbose_proxy_logger.warning("Spend tracking - recording non-finite savings %s as 0 for model=%s", savings, model)
|
||||
return SavingsSpend(*(value if isfinite(value) else 0.0 for value in savings))
|
||||
|
|
|
|||
|
|
@ -912,7 +912,7 @@ def get_logging_payload(
|
|||
user=metadata.get("user_api_key_user_id", "") or "",
|
||||
team_id=metadata.get("user_api_key_team_id", "") or "",
|
||||
organization_id=metadata.get("user_api_key_org_id") or "",
|
||||
metadata=safe_dumps(clean_metadata),
|
||||
metadata=safe_dumps(clean_metadata, non_finite_as_null=True),
|
||||
cache_key=cache_key,
|
||||
spend=kwargs.get("response_cost", 0),
|
||||
total_tokens=usage.get("total_tokens", standard_logging_total_tokens),
|
||||
|
|
@ -1224,7 +1224,7 @@ def _get_messages_for_spend_logs_payload(
|
|||
messages: Final = standard_logging_payload.get("messages")
|
||||
if messages is not None:
|
||||
try:
|
||||
return safe_dumps(messages)
|
||||
return safe_dumps(messages, non_finite_as_null=True)
|
||||
except Exception:
|
||||
return "{}"
|
||||
return "{}"
|
||||
|
|
@ -1919,7 +1919,8 @@ def _get_proxy_server_request_for_spend_logs_payload(
|
|||
_request_body_json_str: Final = safe_dumps(
|
||||
_stored_request_without_requester_ip_address(stored_request_body)
|
||||
if drop_requester_ip_address
|
||||
else stored_request_body
|
||||
else stored_request_body,
|
||||
non_finite_as_null=True,
|
||||
)
|
||||
if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
|
||||
verbose_proxy_logger.info(
|
||||
|
|
@ -2000,7 +2001,7 @@ def _get_response_for_spend_logs_payload(
|
|||
if isinstance(sanitized_response, str):
|
||||
result_str = strip_null_bytes(sanitized_response)
|
||||
else:
|
||||
result_str = safe_dumps(sanitized_response)
|
||||
result_str = safe_dumps(sanitized_response, non_finite_as_null=True)
|
||||
if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str:
|
||||
verbose_proxy_logger.info(
|
||||
"Spend Log: response was truncated before storing in DB. %s",
|
||||
|
|
@ -2071,7 +2072,7 @@ def spend_log_row_with_retained_metadata(
|
|||
return row
|
||||
metadata: Final = _SPEND_LOGS_METADATA_ADAPTER.validate_json(metadata_json)
|
||||
retained: Final = {name: value for name, value in metadata.items() if fields.keeps(name)}
|
||||
return {**row, "metadata": safe_dumps(retained)}
|
||||
return {**row, "metadata": safe_dumps(retained, non_finite_as_null=True)}
|
||||
|
||||
|
||||
def _get_status_for_spend_log(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import builtins
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from enum import Enum
|
||||
from os import PathLike
|
||||
|
|
@ -83,6 +84,7 @@ from typing_extensions import (
|
|||
override,
|
||||
)
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel
|
||||
from litellm.types.responses.main import (
|
||||
CustomToolCallOutputItem,
|
||||
|
|
@ -1446,9 +1448,11 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
|
|||
@classmethod
|
||||
def parse_cost(cls, v: object) -> object:
|
||||
"""Normalise cost: accept either a float or a dict with a ``total_cost`` key."""
|
||||
if isinstance(v, dict):
|
||||
return v.get("total_cost")
|
||||
return v
|
||||
cost: Final = v.get("total_cost") if isinstance(v, dict) else v
|
||||
if isinstance(cost, float) and not math.isfinite(cost):
|
||||
verbose_logger.warning("Dropping a non-finite provider usage cost")
|
||||
return None
|
||||
return cost
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Collection, Mapping, Sequence
|
||||
|
|
@ -1971,6 +1972,19 @@ class ServerToolUse(LiteLLMBaseModel):
|
|||
return getattr(self, key)
|
||||
|
||||
|
||||
_COST_BREAKDOWN_ADAPTER: Final = TypeAdapter(Mapping[object, object])
|
||||
|
||||
|
||||
def _is_non_finite_float(value: object) -> bool:
|
||||
return isinstance(value, float) and not math.isfinite(value)
|
||||
|
||||
|
||||
def _carries_non_finite_number(cost: object) -> bool:
|
||||
if not isinstance(cost, Mapping):
|
||||
return _is_non_finite_float(cost)
|
||||
return any(_is_non_finite_float(member) for member in _COST_BREAKDOWN_ADAPTER.validate_python(cost).values())
|
||||
|
||||
|
||||
class Usage(SafeAttributeModel, CompletionUsage):
|
||||
_cache_creation_input_tokens: int = PrivateAttr(
|
||||
0
|
||||
|
|
@ -1978,6 +1992,7 @@ class Usage(SafeAttributeModel, CompletionUsage):
|
|||
_cache_read_input_tokens: int = PrivateAttr(
|
||||
0
|
||||
) # hidden param for prompt caching. Might change, once openai introduces their equivalent.
|
||||
_dropped_non_finite_cost: bool = PrivateAttr(False)
|
||||
|
||||
server_tool_use: ServerToolUse | None = None
|
||||
cost: float | None = None
|
||||
|
|
@ -2084,10 +2099,14 @@ class Usage(SafeAttributeModel, CompletionUsage):
|
|||
else: # maintain openai compatibility in usage object if possible
|
||||
del self.server_tool_use
|
||||
|
||||
if cost is not None:
|
||||
self.cost = cost
|
||||
else:
|
||||
dropped_non_finite_cost: Final = cost is not None and _carries_non_finite_number(cost)
|
||||
if cost is None:
|
||||
del self.cost
|
||||
elif dropped_non_finite_cost:
|
||||
verbose_logger.warning("Dropping a non-finite provider usage cost")
|
||||
del self.cost
|
||||
else:
|
||||
self.cost = cost
|
||||
|
||||
## ANTHROPIC MAPPING ##
|
||||
if "cache_creation_input_tokens" in params and isinstance(params["cache_creation_input_tokens"], int):
|
||||
|
|
@ -2102,6 +2121,11 @@ class Usage(SafeAttributeModel, CompletionUsage):
|
|||
|
||||
for k, v in params.items():
|
||||
setattr(self, k, v)
|
||||
self._dropped_non_finite_cost = dropped_non_finite_cost
|
||||
|
||||
@staticmethod
|
||||
def dropped_non_finite_cost(usage: "Usage") -> bool:
|
||||
return usage._dropped_non_finite_cost
|
||||
|
||||
def __contains__(self, key) -> bool:
|
||||
# Define custom behavior for the 'in' operator
|
||||
|
|
|
|||
166
tests/integration/spend/test_non_finite_cost_spend_writes.py
Normal file
166
tests/integration/spend/test_non_finite_cost_spend_writes.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import json
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.openai_wire import answering_model_discovery
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
PROMPT_TOKENS: Final = 5
|
||||
COMPLETION_TOKENS: Final = 3
|
||||
INPUT_RATE: Final = 0.001
|
||||
OUTPUT_RATE: Final = 0.002
|
||||
TOKEN_PRICE: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE
|
||||
OVERFLOWING_INPUT_RATE: Final = 1e308
|
||||
POISONED_UPSTREAM: Final = "poisoned-upstream"
|
||||
CONTROL_UPSTREAM: Final = "control-upstream"
|
||||
DAILY_TOTALS: Final = "SELECT model_group, SUM(spend)::float8 AS spend, SUM(api_requests)::int8 AS api_requests"
|
||||
DAILY_USER_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyUserSpend" WHERE api_key=%s GROUP BY model_group'
|
||||
DAILY_TEAM_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyTeamSpend" WHERE api_key=%s GROUP BY model_group'
|
||||
DAILY_TAG_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyTagSpend" WHERE api_key=%s AND tag=%s GROUP BY model_group'
|
||||
|
||||
Poison = Literal["reported_nan_cost", "overflowing_price"]
|
||||
|
||||
|
||||
def _frame(payload: dict[str, JsonValue]) -> bytes:
|
||||
return b"data: " + json.dumps(payload).encode() + b"\n\n"
|
||||
|
||||
|
||||
def _stream(upstream_model: str, reported_cost: float | None) -> tuple[bytes, ...]:
|
||||
chunk: Final = {"id": f"chatcmpl-{uuid4().hex}", "object": "chat.completion.chunk", "created": 1}
|
||||
usage: Final[dict[str, JsonValue]] = {
|
||||
"prompt_tokens": PROMPT_TOKENS,
|
||||
"completion_tokens": COMPLETION_TOKENS,
|
||||
"total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
|
||||
}
|
||||
reported_usage: Final = usage if reported_cost is None else {**usage, "cost": reported_cost}
|
||||
return (
|
||||
_frame({**chunk, "model": upstream_model, "choices": [{"index": 0, "delta": {"content": "ok"}}]}),
|
||||
_frame(
|
||||
{
|
||||
**chunk,
|
||||
"model": upstream_model,
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
"usage": reported_usage,
|
||||
}
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
|
||||
|
||||
def _deployment(alias: str, upstream_model: str, wire_url: str, input_rate: float) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model_name": alias,
|
||||
"litellm_params": {
|
||||
"model": f"openai/{upstream_model}",
|
||||
"api_base": f"{wire_url}/v1",
|
||||
"api_key": "integration-provider-key",
|
||||
"input_cost_per_token": input_rate,
|
||||
"output_cost_per_token": OUTPUT_RATE,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _proxy_config(directory: Path, deployments: tuple[dict[str, JsonValue], ...]) -> Path:
|
||||
config: Final = directory / "non_finite_cost_config.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": list(deployments),
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
"disable_spend_logs": False,
|
||||
"proxy_batch_write_at": 1,
|
||||
"proxy_batch_polling_interval": 1,
|
||||
},
|
||||
"router_settings": {"disable_cooldowns": True},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _by_model_group(rows: list[dict[str, JsonValue]]) -> dict[str, float]:
|
||||
return {str(row["model_group"]): float(str(row["spend"])) for row in rows}
|
||||
|
||||
|
||||
def _both_requests_rows(query: str, parameters: tuple[str, ...]) -> list[dict[str, JsonValue]]:
|
||||
return eventually(lambda: read_rows(query, parameters), lambda rows: len(rows) == 2, seconds=70)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("poison", ["reported_nan_cost", "overflowing_price"])
|
||||
@pytest.mark.timeout(240)
|
||||
def test_a_non_finite_cost_never_blocks_the_next_requests_spend_rows(
|
||||
gateway: Gateway, tmp_path: Path, poison: Poison
|
||||
) -> None:
|
||||
poisoned_alias: Final = f"non-finite-cost-{uuid4().hex}"
|
||||
control_alias: Final = f"finite-cost-{uuid4().hex}"
|
||||
tag: Final = f"non-finite-cost-{uuid4().hex}"
|
||||
poisoned_spend: Final = TOKEN_PRICE if poison == "reported_nan_cost" else 0.0
|
||||
poisoned_input_rate: Final = INPUT_RATE if poison == "reported_nan_cost" else OVERFLOWING_INPUT_RATE
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.target.endswith("/chat/completions"), request.target
|
||||
upstream_model: Final = json.loads(request.body)["model"]
|
||||
reported_cost: Final = (
|
||||
float("nan") if upstream_model == POISONED_UPSTREAM and poison == "reported_nan_cost" else None
|
||||
)
|
||||
return Reply(content_type="text/event-stream", chunks=_stream(upstream_model, reported_cost))
|
||||
|
||||
with wire_server(answering_model_discovery(respond)) as wire:
|
||||
config: Final = _proxy_config(
|
||||
tmp_path,
|
||||
(
|
||||
_deployment(poisoned_alias, POISONED_UPSTREAM, wire.url, poisoned_input_rate),
|
||||
_deployment(control_alias, CONTROL_UPSTREAM, wire.url, INPUT_RATE),
|
||||
),
|
||||
)
|
||||
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario:
|
||||
team: Final = scenario.team()
|
||||
key: Final = scenario.key(user_id=f"non-finite-cost-{uuid4().hex}", team_id=team)
|
||||
digest: Final = sha256(key.encode()).hexdigest()
|
||||
responses: Final = tuple(
|
||||
candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": alias,
|
||||
"messages": [{"role": "user", "content": f"non-finite cost {uuid4().hex}"}],
|
||||
"stream": True,
|
||||
"metadata": {"tags": [tag]},
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
for alias in (poisoned_alias, control_alias)
|
||||
)
|
||||
assert all(response.status_code == 200 for response in responses), [response.text for response in responses]
|
||||
|
||||
expected: Final = {poisoned_alias: pytest.approx(poisoned_spend), control_alias: pytest.approx(TOKEN_PRICE)}
|
||||
spend_logs: Final = _both_requests_rows(
|
||||
'SELECT model_group, spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (digest,)
|
||||
)
|
||||
assert _by_model_group(spend_logs) == expected, spend_logs
|
||||
daily_rows: Final = (
|
||||
_both_requests_rows(DAILY_USER_QUERY, (digest,)),
|
||||
_both_requests_rows(DAILY_TEAM_QUERY, (digest,)),
|
||||
_both_requests_rows(DAILY_TAG_QUERY, (digest, tag)),
|
||||
)
|
||||
assert [_by_model_group(rows) for rows in daily_rows] == [expected] * 3, daily_rows
|
||||
assert [{row["api_requests"] for row in rows} for rows in daily_rows] == [{1}] * 3, daily_rows
|
||||
key_total: Final = poisoned_spend + TOKEN_PRICE
|
||||
key_spend: Final = eventually(
|
||||
lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)),
|
||||
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(key_total),
|
||||
seconds=70,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
assert float(str(key_spend[0]["spend"])) == pytest.approx(key_total), key_spend
|
||||
|
|
@ -3089,6 +3089,50 @@ def test_convert_to_model_response_object_function_output():
|
|||
assert result._hidden_params is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reported_cost",
|
||||
[
|
||||
float("nan"),
|
||||
float("inf"),
|
||||
{"input_tokens_cost": 0.0001, "request_cost": 0.005, "total_cost": float("nan")},
|
||||
{"input_tokens_cost": float("inf"), "request_cost": 0.005, "total_cost": 0.0051},
|
||||
],
|
||||
ids=["nan", "inf", "breakdown-with-nan-total", "breakdown-with-inf-part"],
|
||||
)
|
||||
def test_a_non_finite_provider_usage_cost_is_dropped_so_the_response_is_strict_json(reported_cost: object):
|
||||
response: Final = convert_to_model_response_object(
|
||||
response_object={
|
||||
"id": "chatcmpl-non-finite-cost",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, "cost": reported_cost},
|
||||
},
|
||||
model_response_object=ModelResponse(),
|
||||
)
|
||||
|
||||
usage: Final = json.loads(json.dumps(response.model_dump(), allow_nan=False))["usage"]
|
||||
assert "cost" not in usage
|
||||
assert usage["total_tokens"] == 4
|
||||
|
||||
|
||||
def test_a_finite_provider_cost_breakdown_with_non_string_keys_is_kept():
|
||||
response: Final = convert_to_model_response_object(
|
||||
response_object={
|
||||
"id": "chatcmpl-int-key-cost",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, "cost": {1: 0.5}},
|
||||
},
|
||||
model_response_object=ModelResponse(),
|
||||
)
|
||||
|
||||
assert response.usage.cost == {1: 0.5}
|
||||
|
||||
|
||||
_URL_CITATION: Final = {
|
||||
"type": "url_citation",
|
||||
"url_citation": {
|
||||
|
|
|
|||
|
|
@ -26,15 +26,24 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
normalize_drop_params,
|
||||
preserve_upstream_non_openai_attributes,
|
||||
process_response_headers,
|
||||
provider_reported_usage_cost,
|
||||
qualify_agentic_followup_model,
|
||||
qualify_provider_stripped_model,
|
||||
reconstruct_model_name,
|
||||
redact_nested_match_and_regex_keys,
|
||||
response_cost_in_hidden_params,
|
||||
set_provider_response_headers_in_hidden_params,
|
||||
unbind_budget_reservation_from_callbacks,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.utils import Delta, ImageResponse, ModelResponseStream, StreamingChoices, TranscriptionResponse
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
ImageResponse,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
TranscriptionResponse,
|
||||
)
|
||||
|
||||
|
||||
def test_parent_otel_span_reads_dict_like_metadata_by_index():
|
||||
|
|
@ -790,6 +799,23 @@ class TestReceiveStampSurvivesASecondMetadataBucket:
|
|||
assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reported", [float("nan"), float("inf"), {"total_cost": float("nan")}])
|
||||
def test_a_non_finite_provider_reported_cost_counts_as_unreported(reported: object) -> None:
|
||||
assert provider_reported_usage_cost({"cost": reported}, "openrouter") is None
|
||||
assert provider_reported_usage_cost({"cost": {"total_cost": 0.25}}, "openrouter") == 0.25
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header_cost", [float("nan"), float("inf")])
|
||||
def test_a_non_finite_response_cost_header_counts_as_unreported(header_cost: float) -> None:
|
||||
response: Final = ModelResponse()
|
||||
response._hidden_params = {"additional_headers": {RESPONSE_COST_HEADER: header_cost}}
|
||||
finite: Final = ModelResponse()
|
||||
finite._hidden_params = {"additional_headers": {RESPONSE_COST_HEADER: 0.25}}
|
||||
|
||||
assert response_cost_in_hidden_params(response) is None
|
||||
assert response_cost_in_hidden_params(finite) == 0.25
|
||||
|
||||
|
||||
def test_billed_usage_cost_bills_any_chat_upstream_without_its_own_reader() -> None:
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
|
|
|||
|
|
@ -6308,6 +6308,60 @@ def test_standard_logging_payload_resolves_kubernetes_pod_routing_from_metadata_
|
|||
assert payload["metadata"][KUBERNETES_POD_ROUTING_KEY] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")])
|
||||
def test_get_standard_logging_object_payload_logs_a_non_finite_response_cost_as_zero(
|
||||
logging_obj: LitellmLogging, response_cost: float
|
||||
) -> None:
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
|
||||
|
||||
now: Final = datetime(2026, 1, 1)
|
||||
payload: Final = get_standard_logging_object_payload(
|
||||
kwargs={"model": "gpt-4o", "messages": [], "litellm_params": {}, "response_cost": response_cost},
|
||||
init_response_obj={},
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
logging_obj=logging_obj,
|
||||
status="success",
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["response_cost"] == 0
|
||||
|
||||
|
||||
def test_an_overflowing_custom_price_records_a_zero_response_cost(caplog: pytest.LogCaptureFixture) -> None:
|
||||
messages: Final = [{"role": "user", "content": "hello world"}]
|
||||
start_time: Final = datetime.datetime.now()
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=start_time,
|
||||
litellm_call_id="overflow\nforged log line",
|
||||
function_id="overflow",
|
||||
)
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
response: Final = litellm.completion(
|
||||
model="gpt-4o",
|
||||
mock_response="hi there",
|
||||
messages=messages,
|
||||
input_cost_per_token=1e308,
|
||||
output_cost_per_token=1e308,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
logging_obj._process_hidden_params_and_response_cost(
|
||||
response, start_time=start_time, end_time=datetime.datetime.now()
|
||||
)
|
||||
|
||||
assert response._hidden_params["response_cost"] == 0.0
|
||||
payload: Final = logging_obj.model_call_details["standard_logging_object"]
|
||||
assert json.loads(json.dumps(payload, allow_nan=False, default=str))["response_cost"] == 0.0
|
||||
warnings: Final = [r.getMessage() for r in caplog.records if "non-finite response_cost" in r.getMessage()]
|
||||
assert warnings == ["Recording a non-finite response_cost as 0 for litellm_call_id=overflowforged log line"]
|
||||
|
||||
|
||||
def test_get_standard_logging_object_payload_carries_matched_access_groups(logging_obj):
|
||||
"""Access groups stamped at auth time reach the logging payload, so integrations see what a request billed."""
|
||||
from datetime import datetime
|
||||
|
|
|
|||
|
|
@ -169,6 +169,31 @@ def test_null_byte_stripped_in_fallback_str():
|
|||
assert json.loads(out)["obj"] == "objrepr"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
|
||||
def test_non_finite_floats_are_written_as_json_null(value: float) -> None:
|
||||
out = safe_dumps(
|
||||
{"cost_breakdown": {"total_cost": value, "output_cost": 0.006}, "costs": [value]}, non_finite_as_null=True
|
||||
)
|
||||
|
||||
assert json.loads(out, parse_constant=lambda token: pytest.fail(f"non-JSON token {token}")) == {
|
||||
"cost_breakdown": {"total_cost": None, "output_cost": 0.006},
|
||||
"costs": [None],
|
||||
}
|
||||
|
||||
|
||||
def test_a_set_holding_a_non_finite_float_sorts_its_finite_members_and_puts_null_last() -> None:
|
||||
assert json.loads(safe_dumps({"costs": {2.0, float("inf"), 1.0}}, non_finite_as_null=True)) == {
|
||||
"costs": [1.0, 2.0, None]
|
||||
}
|
||||
|
||||
|
||||
def test_non_finite_floats_keep_their_value_unless_the_caller_asks_for_null() -> None:
|
||||
assert json.loads(safe_dumps({"timeout": float("inf"), "costs": [float("-inf")]})) == {
|
||||
"timeout": float("inf"),
|
||||
"costs": [float("-inf")],
|
||||
}
|
||||
|
||||
|
||||
def test_clean_strings_are_not_run_through_replace():
|
||||
"""Regression for LIT-3910.
|
||||
|
||||
|
|
|
|||
|
|
@ -1077,6 +1077,47 @@ def test_cost_field_in_usage_chunks():
|
|||
assert usage.completion_tokens == 5
|
||||
|
||||
|
||||
|
||||
def _perplexity_usage_chunk(cost: Mapping[str, float], finish_reason: str | None) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl-pplx",
|
||||
created=1745513206,
|
||||
model="sonar",
|
||||
choices=[StreamingChoices(finish_reason=finish_reason, index=0, delta=Delta(content="ok"))],
|
||||
usage={"prompt_tokens": 26, "completion_tokens": 1, "total_tokens": 27, "cost": dict(cost)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("poisoned_field", ["request_cost", "total_cost"])
|
||||
def test_a_non_finite_final_stream_cost_does_not_fall_back_to_an_earlier_chunk_cost(poisoned_field: str):
|
||||
final_cost: Final = {"input_tokens_cost": 0.0012, "output_tokens_cost": 1e-05, "request_cost": 0.0025}
|
||||
chunks: Final = [
|
||||
_perplexity_usage_chunk({"input_tokens_cost": 0, "output_tokens_cost": 0, "total_cost": 0}, None),
|
||||
_perplexity_usage_chunk({**final_cost, "total_cost": 0.00371, poisoned_field: float("nan")}, "stop"),
|
||||
]
|
||||
|
||||
response: Final = stream_chunk_builder(chunks=chunks)
|
||||
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) is None
|
||||
assert response.usage.total_tokens == 27
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_key", ["dropped_non_finite_cost", "_dropped_non_finite_cost"])
|
||||
def test_a_provider_usage_key_named_like_the_dropped_cost_flag_keeps_the_stream_cost(provider_key: str):
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-flag-key",
|
||||
created=1745513206,
|
||||
model="sonar",
|
||||
choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="ok"))],
|
||||
usage={"prompt_tokens": 26, "completion_tokens": 1, "total_tokens": 27, "cost": 0.00371, provider_key: True},
|
||||
)
|
||||
|
||||
response: Final = stream_chunk_builder(chunks=[chunk])
|
||||
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) == 0.00371
|
||||
|
||||
def test_stream_chunk_builder_tolerates_trailing_chunk_without_choices():
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/32051
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -464,6 +465,34 @@ def test_openrouter_cost_tracking_non_streaming():
|
|||
)
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cost", [float("nan"), float("inf")])
|
||||
def test_openrouter_chat_leaves_a_non_finite_provider_cost_out_of_the_response_headers(cost: float) -> None:
|
||||
body: Final = {
|
||||
"id": "gen-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1234567890,
|
||||
"model": "openai/gpt-5.6-luna",
|
||||
"choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", "index": 0}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11, "cost": cost},
|
||||
}
|
||||
|
||||
result: Final = OpenrouterConfig().transform_response(
|
||||
model="openrouter/openai/gpt-5.6-luna",
|
||||
raw_response=httpx.Response(
|
||||
200, content=json.dumps(body).encode(), headers={"content-type": "application/json"}
|
||||
),
|
||||
model_response=litellm.ModelResponse(),
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Reply with ok"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
assert "llm_provider-x-litellm-response-cost" not in result._hidden_params.get("additional_headers", {})
|
||||
|
||||
def test_openrouter_cost_tracking_streaming():
|
||||
"""
|
||||
Test OpenRouter cost tracking for streaming completions.
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ reasoning.encrypted_content for multi-turn stateless workflows.
|
|||
Related issue: https://github.com/BerriAI/litellm/issues/22189
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -157,6 +159,29 @@ def test_openrouter_responses_returns_provider_reported_cost(cost):
|
|||
assert get_response_cost_from_hidden_params(response.hidden_params) == cost
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cost", [float("nan"), float("inf")])
|
||||
def test_openrouter_responses_drops_a_non_finite_provider_cost(cost: float) -> None:
|
||||
body: Final = {
|
||||
"id": "resp_test",
|
||||
"created_at": 1,
|
||||
"model": "unmapped-model",
|
||||
"object": "response",
|
||||
"output": [],
|
||||
"status": "completed",
|
||||
"usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5, "cost": cost},
|
||||
}
|
||||
response: Final = OpenRouterResponsesAPIConfig().transform_response_api_response(
|
||||
model="openrouter/unmapped-model",
|
||||
raw_response=httpx.Response(
|
||||
200, content=json.dumps(body).encode(), headers={"content-type": "application/json"}
|
||||
),
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert get_response_cost_from_hidden_params(response.hidden_params) is None
|
||||
assert json.loads(json.dumps(response.model_dump(), allow_nan=False, default=str))["usage"]["cost"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"event_type",
|
||||
["response.completed", "response.incomplete", "response.failed"],
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@ search queries, and reasoning tokens.
|
|||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -140,6 +142,26 @@ class TestPerplexityCostCalculator:
|
|||
assert prompt_cost == 0.0
|
||||
assert completion_cost == 0.008
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reported_cost",
|
||||
[float("nan"), float("inf"), {"total_cost": float("nan")}],
|
||||
ids=["nan", "inf", "nan-total-cost-dict"],
|
||||
)
|
||||
def test_a_non_finite_perplexity_cost_is_priced_like_no_reported_cost_with_a_warning(
|
||||
self, reported_cost: float | dict, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
unreported: Final = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
reported: Final = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
reported.cost = reported_cost
|
||||
|
||||
token_priced: Final = perplexity_cost_per_token(model="sonar-pro", usage=unreported)
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
reported_priced: Final = perplexity_cost_per_token(model="sonar-pro", usage=reported)
|
||||
|
||||
assert sum(token_priced) > 0
|
||||
assert reported_priced == token_priced
|
||||
assert [record for record in caplog.records if "non-finite provider-reported cost" in record.getMessage()]
|
||||
|
||||
OFF_PEAK_MODEL = "sonar-off-peak-test"
|
||||
OFF_PEAK_WINDOW = "14:00-00:00"
|
||||
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.proxy._types import DailyTagSpendTransaction
|
||||
|
|
@ -379,6 +381,104 @@ async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_fail
|
|||
assert restored_spend["org_member_list_transactions"] == {member_key: 1.5}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_single_parked_batch_with_a_nan_spend_drains_as_zero(
|
||||
redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock
|
||||
):
|
||||
mock_redis_cache.async_lpop = AsyncMock(
|
||||
return_value=json.dumps({"key_list_transactions": {"nan-key": float("nan"), "ok-key": 0.5}}, allow_nan=True)
|
||||
)
|
||||
|
||||
drained: Final = await redis_update_buffer.get_all_update_transactions_from_redis_buffer()
|
||||
|
||||
assert drained is not None
|
||||
assert drained["key_list_transactions"] == {"nan-key": 0.0, "ok-key": 0.5}
|
||||
|
||||
|
||||
PARKED_DAILY_KEY: Final = "user-1_2025-01-15_api-key-123_gpt-4o_openai_/chat/completions"
|
||||
|
||||
|
||||
def _parked_daily_batch(spend: float) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
PARKED_DAILY_KEY: {
|
||||
"spend": spend,
|
||||
"prompt_tokens": 12,
|
||||
"completion_tokens": 7,
|
||||
"api_requests": 1,
|
||||
"successful_requests": 1,
|
||||
"failed_requests": 0,
|
||||
}
|
||||
},
|
||||
allow_nan=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_nan_spend_parked_by_an_older_version_drains_from_the_pipeline_as_zero(
|
||||
redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock
|
||||
):
|
||||
entity_batch: Final = json.dumps({"key_list_transactions": {"hashed-key": float("nan")}}, allow_nan=True)
|
||||
window_batches: Final = tuple(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"entity_type": "key",
|
||||
"entity_id": "hashed-key",
|
||||
"window_duration": "30d",
|
||||
"window_start": "2026-08-01T00:00:00.000000",
|
||||
"spend": spend,
|
||||
"started_at": None,
|
||||
}
|
||||
],
|
||||
allow_nan=True,
|
||||
)
|
||||
for spend in (float("nan"), 3.0)
|
||||
)
|
||||
mock_redis_cache.async_lpop_pipeline = AsyncMock(
|
||||
return_value=[
|
||||
[entity_batch],
|
||||
[_parked_daily_batch(float("nan")), _parked_daily_batch(0.0042)],
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
list(window_batches),
|
||||
]
|
||||
)
|
||||
|
||||
(
|
||||
db_spend,
|
||||
daily_user,
|
||||
*_rest,
|
||||
window_spend,
|
||||
) = await redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline()
|
||||
|
||||
assert db_spend is not None
|
||||
assert db_spend["key_list_transactions"] == {"hashed-key": 0}
|
||||
assert daily_user is not None
|
||||
assert daily_user[PARKED_DAILY_KEY]["spend"] == 0.0042
|
||||
assert daily_user[PARKED_DAILY_KEY]["api_requests"] == 2
|
||||
assert window_spend is not None
|
||||
assert [transaction["spend"] for transaction in window_spend] == [3.0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_nan_spend_parked_by_an_older_version_drains_from_a_daily_key_as_zero(
|
||||
redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
mock_redis_cache.async_lpop = AsyncMock(
|
||||
return_value=[_parked_daily_batch(float("inf")), _parked_daily_batch(0.0042)]
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
|
||||
daily_tag: Final = await redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer()
|
||||
|
||||
assert daily_tag is not None
|
||||
assert daily_tag[PARKED_DAILY_KEY]["spend"] == 0.0042
|
||||
assert [record.getMessage() for record in caplog.records if "Infinity" in record.getMessage()] != []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_all_transactions_from_redis_buffer_pipeline_no_redis():
|
||||
"""When redis_cache is None, should return all Nones"""
|
||||
|
|
@ -724,6 +824,10 @@ class _ListRedis:
|
|||
def __init__(self) -> None:
|
||||
self.rows: list[str] = []
|
||||
|
||||
async def async_rpush(self, key: str, values: list[str]) -> int:
|
||||
self.rows.extend(values)
|
||||
return len(self.rows)
|
||||
|
||||
async def async_rpush_and_trim(self, key: str, values: list[str], max_len: int) -> int:
|
||||
self.rows.extend(values)
|
||||
pushed_len = len(self.rows)
|
||||
|
|
@ -752,6 +856,32 @@ async def test_store_spend_logs_in_redis_drops_oldest_rows_past_the_cap():
|
|||
assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("spend", [float("nan"), float("inf")])
|
||||
async def test_a_non_finite_daily_spend_buffered_in_redis_drains_as_zero(spend: float):
|
||||
buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis())
|
||||
queue: Final = DailySpendUpdateQueue()
|
||||
await queue.add_update(json.loads(_parked_daily_batch(spend)))
|
||||
await queue.add_update(json.loads(_parked_daily_batch(0.0042)))
|
||||
|
||||
await buffer.store_in_memory_daily_tag_spend_updates_in_redis(queue)
|
||||
drained: Final = await buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer()
|
||||
|
||||
assert drained is not None
|
||||
assert drained[PARKED_DAILY_KEY]["spend"] == 0.0
|
||||
assert drained[PARKED_DAILY_KEY]["api_requests"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_spend_log_row_parked_with_a_nan_spend_is_read_back_with_zero_spend():
|
||||
buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis())
|
||||
buffer.should_commit_spend_updates_to_redis = MagicMock(return_value=True)
|
||||
|
||||
assert await buffer.store_spend_logs_in_redis([{"request_id": "nan-row", "spend": float("nan")}]) is True
|
||||
|
||||
assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == ({"request_id": "nan-row", "spend": 0.0},)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_spend_logs_in_redis_reports_failure_without_redis():
|
||||
buffer = RedisUpdateBuffer(redis_cache=None)
|
||||
|
|
|
|||
|
|
@ -2226,6 +2226,60 @@ async def test_shutdown_drain_finishes_the_rollups_update_database_started():
|
|||
assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")])
|
||||
async def test_update_database_records_a_non_finite_response_cost_as_zero(response_cost: float) -> None:
|
||||
db_writer: Final = DBSpendUpdateWriter()
|
||||
prisma: Final = _tool_usage_prisma()
|
||||
prisma.get_request_status = MagicMock(return_value="success")
|
||||
response: Final = litellm.ModelResponse(
|
||||
model="gpt-5.6-luna",
|
||||
usage=litellm.Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
|
||||
)
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.disable_spend_logs", False),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", prisma),
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
|
||||
):
|
||||
await db_writer.update_database(
|
||||
token="test-token",
|
||||
user_id="test-user",
|
||||
end_user_id=None,
|
||||
team_id="test-team",
|
||||
org_id=None,
|
||||
kwargs={
|
||||
"model": "gpt-5.6-luna",
|
||||
"custom_llm_provider": "openai",
|
||||
"call_type": "acompletion",
|
||||
"litellm_call_id": "req-non-finite-cost",
|
||||
"response_cost": response_cost,
|
||||
"litellm_params": {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "test-team"}},
|
||||
},
|
||||
completion_response=response,
|
||||
start_time=datetime.now(timezone.utc),
|
||||
end_time=datetime.now(timezone.utc),
|
||||
response_cost=response_cost,
|
||||
)
|
||||
await drain_logging_tasks()
|
||||
|
||||
assert [row["spend"] for row in prisma.spend_log_transactions] == [0]
|
||||
spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
|
||||
assert spend["key_list_transactions"] == {"test-token": 0}
|
||||
assert spend["team_list_transactions"] == {"test-team": 0}
|
||||
daily_user: Final = (
|
||||
await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
)
|
||||
daily_team: Final = (
|
||||
await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
)
|
||||
assert daily_user is not None and daily_team is not None
|
||||
(user_row,) = daily_user.values()
|
||||
(team_row,) = daily_team.values()
|
||||
assert (user_row["api_requests"], user_row["spend"]) == (1, 0)
|
||||
assert (team_row["api_requests"], team_row["spend"]) == (1, 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -323,6 +323,35 @@ def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row(
|
|||
assert response["kwargs"]["combined_usage_object"].total_tokens == 302
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reported_cost", [float("nan"), float("inf")])
|
||||
def test_openrouter_non_finite_cost_logs_the_token_price(monkeypatch: pytest.MonkeyPatch, reported_cost: float) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"openrouter/typesafe/jev-1.13",
|
||||
{"input_cost_per_token": 1.0, "output_cost_per_token": 3.0, "litellm_provider": "openrouter", "mode": "chat"},
|
||||
)
|
||||
logging_obj: Final = _logging_obj()
|
||||
|
||||
result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
|
||||
httpx_response=_response(),
|
||||
response_body={
|
||||
"model": "typesafe/jev-1.13-20260917",
|
||||
"usage": {"input_tokens": 2, "output_tokens": 1, "cost": reported_cost},
|
||||
},
|
||||
logging_obj=logging_obj,
|
||||
url_route="https://openrouter.ai/api/alpha/decisions",
|
||||
result='{"answers": {}}',
|
||||
start_time=datetime(2000, 1, 1),
|
||||
end_time=datetime(2000, 1, 1),
|
||||
cache_hit=False,
|
||||
request_body={"model": "typesafe/jev-1.13"},
|
||||
custom_llm_provider="openrouter",
|
||||
)
|
||||
|
||||
assert result["kwargs"]["response_cost"] == pytest.approx(5.0)
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(5.0)
|
||||
|
||||
|
||||
def test_success_handler_dispatches_openrouter_to_the_shared_handler():
|
||||
logging_obj = _logging_obj()
|
||||
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
|
||||
|
|
|
|||
|
|
@ -137,6 +137,34 @@ def test_compression_savings_priced_at_input_rate():
|
|||
assert result.prompt_caching == 0.0
|
||||
|
||||
|
||||
def test_an_overflowing_deployment_price_records_finite_savings() -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "overflowing-sonnet",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-sonnet-5-5",
|
||||
"input_cost_per_token": 1e308,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
deployment_id: Final = router.get_model_list(model_name="overflowing-sonnet")[0]["model_info"]["id"]
|
||||
|
||||
result: Final = compute_savings_spend(
|
||||
model="claude-sonnet-5-5",
|
||||
custom_llm_provider="anthropic",
|
||||
compression_saved_tokens=4389,
|
||||
gateway_injected_cache=True,
|
||||
usage_object=_caching_usage(read=1000, written=0),
|
||||
model_id=deployment_id,
|
||||
llm_router=lambda: router,
|
||||
)
|
||||
|
||||
assert result == (0.0, 0.0, 0.0, 0.0)
|
||||
|
||||
|
||||
def test_prompt_caching_savings_priced_at_input_minus_cache_read():
|
||||
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
|
||||
# A model that supports prompt caching must charge less to read from cache;
|
||||
|
|
|
|||
|
|
@ -12496,6 +12496,55 @@ class TestResponseCostFromLoggingObjForCacheHits:
|
|||
)
|
||||
assert {name: headers.get(name) for name in cost_headers} == dict.fromkeys(cost_headers, expected_header)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("response_cost", "expected_header"),
|
||||
((float("nan"), None), (float("inf"), None), (0.0042, "0.0042"), (2, "2")),
|
||||
ids=("nan", "inf", "finite", "int"),
|
||||
)
|
||||
async def test_a_non_finite_response_cost_sends_no_cost_header(
|
||||
self, response_cost: float, expected_header: str | None
|
||||
):
|
||||
response: Final = litellm.ModelResponse()
|
||||
response._hidden_params = {"response_cost": response_cost}
|
||||
|
||||
headers: Final = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response(
|
||||
response=response,
|
||||
request_data={"model": "gpt-5.4-mini"},
|
||||
request=Request({"type": "http", "method": "POST", "headers": []}),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
logging_obj=self._logging_obj(),
|
||||
version=None,
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
)
|
||||
|
||||
assert headers.get("x-litellm-response-cost") == expected_header
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_overflowing_cost_breakdown_sends_no_infinite_cost_header(self):
|
||||
response: Final = litellm.ModelResponse()
|
||||
response._hidden_params = {"response_cost": 0.0}
|
||||
logging_obj: Final = self._logging_obj()
|
||||
logging_obj.cost_breakdown = {
|
||||
"original_cost": float("inf"),
|
||||
"input_cost": float("inf"),
|
||||
"output_cost": 0.0012,
|
||||
"total_cost": float("inf"),
|
||||
}
|
||||
|
||||
headers: Final = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response(
|
||||
response=response,
|
||||
request_data={"model": "gpt-5.4-mini"},
|
||||
request=Request({"type": "http", "method": "POST", "headers": []}),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
logging_obj=logging_obj,
|
||||
version=None,
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
)
|
||||
|
||||
cost_headers: Final = {name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")}
|
||||
assert cost_headers == {"x-litellm-response-cost-output": "0.0012"}
|
||||
|
||||
def test_a_fresh_zero_cost_v1_messages_call_still_sends_no_cost_header(self):
|
||||
logging_obj: Final = self._logging_obj()
|
||||
logging_obj.model_call_details["response_cost"] = 0.0
|
||||
|
|
|
|||
|
|
@ -5900,6 +5900,28 @@ def test_stream_chunk_builder_keeps_provider_reported_usage_cost():
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reported_cost", [float("nan"), float("inf")])
|
||||
def test_stream_chunk_builder_bills_the_token_price_when_the_provider_cost_is_not_finite(reported_cost: float):
|
||||
usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=reported_cost)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("gpt-4o", "Hello "),
|
||||
_stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=_stream_builder_logging_obj()
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(model="gpt-4o", usage_object=response.usage)
|
||||
token_price: Final = prompt_cost + completion_cost
|
||||
assert token_price > 0
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(token_price)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(token_price)
|
||||
|
||||
|
||||
def test_completion_cost_on_a_stamped_stream_still_bills_the_reported_charge():
|
||||
usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue