scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 267

Merge stable/1.104.x (#47718) into litellm_backport_p0_batch_translation_cost_stable_1_104_x

Browse source 
# Conflicts:
#	litellm/model_prices_and_context_window_backup.json
#	litellm/proxy/hooks/proxy_track_cost_callback.py
#	litellm/responses/litellm_completion_transformation/streaming_iterator.py
#	model_prices_and_context_window.json
#	tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
#	tests/unit/test_utils.py
This commit is contained in:
Yuneng Jiang 2026-10-20 23:07:37 -07:00
commit bd382afc17
No known key found for this signature in database
73 changed files with 3754 additions and 208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
ci_cd/generate_model_prices_schema.py
View file

@ -62,17 +62,6 @@ OFF_PEAK_WINDOW: Final[JsonSchema] = {
}
OBJECT_KEYS: dict[str, JsonSchema] = {
"search_free_requests": {
"type": "object",
"description": "Free Google Search grounding requests per period, shared across models in the same shared_group.",
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
"properties": {
"count": {"type": "integer", "minimum": 0},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string", "pattern": "\\S"},
},
},
"off_peak_pricing": {
"type": "object",
"description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
@ -102,6 +91,17 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
},
"additionalProperties": False,
},
"search_free_requests": {
"type": "object",
"description": "Free Google Search grounding requests per period, shared across models in the same shared_group.",
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
"properties": {
"count": {"type": "integer", "minimum": 0},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string", "pattern": "\\S"},
},
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",

12
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -767,7 +767,11 @@ class CheckBatchCost:
from litellm.files.main import afile_content
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
from litellm.litellm_core_utils.litellm_logging import (
deployment_pricing_model_info,
mask_api_base_credentials,
resolve_vertex_location_for_cost,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
@ -904,6 +908,12 @@ class CheckBatchCost:
custom_llm_provider=batch_file_provider,
model_name=model_name,
model_info=deployment_model_info,
vertex_location=resolve_vertex_location_for_cost(
custom_llm_provider=llm_provider,
litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True),
optional_params=None,
model=model_name,
),
)
error_file_failed_requests: Final = await count_error_file_failed_requests(
response,

1
litellm/__init__.py
View file

@ -556,6 +556,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
apply_search_free_tier: bool = False
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:

34
litellm/batches/batch_utils.py
View file

@ -73,6 +73,7 @@ async def calculate_batch_cost_and_usage(
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
model_name: str | None = None,
model_info: ModelInfo | None = None,
vertex_location: str | None = None,
) -> BatchCostUsageResult:
"""
Calculate the cost and usage of a batch.
@ -82,16 +83,21 @@ async def calculate_batch_cost_and_usage(
pricing. Threaded through to batch_cost_calculator so that
deployment-specific pricing (e.g. input_cost_per_token_batches)
is used instead of the global cost map.
vertex_location: The Vertex AI location the batch ran on; non-global
locations apply the model's regional-endpoint uplift multiplier.
"""
first_row: Final = file_content_dictionary[0] if file_content_dictionary else None
if _uses_native_vertex_output(custom_llm_provider, model_name, first_row):
return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name, model_info=model_info)
return calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name, model_info=model_info, vertex_location=vertex_location
)
return _aggregate_batch_cost_usage_models(
entries=file_content_dictionary,
custom_llm_provider=custom_llm_provider,
model_name=model_name,
model_info=model_info,
vertex_location=vertex_location,
)
@ -101,6 +107,7 @@ async def _handle_completed_batch(
model_name: str | None = None,
litellm_params: dict | None = None,
model_info: ModelInfo | None = None,
vertex_location: str | None = None,
) -> BatchCostUsageResult:
"""Fetch a completed batch's output file and aggregate its cost, usage, and
models in a single pass over the JSONL lines, so the parsed file content is
@ -114,6 +121,8 @@ async def _handle_completed_batch(
model_info: Optional deployment-level model info with custom pricing,
threaded through so a deployment's configured rates win over the
global cost map.
vertex_location: The Vertex AI location the batch ran on; non-global
locations apply the model's regional-endpoint uplift multiplier.
"""
# A completed batch whose request lines all failed has no output file - the
# results are written to a separate error_file_id and output_file_id is None.
@ -141,7 +150,10 @@ async def _handle_completed_batch(
output_file_result: Final = (
calculate_vertex_ai_batch_cost_and_usage(
_iter_batch_output_entries(file_content), model_name, model_info=model_info
_iter_batch_output_entries(file_content),
model_name,
model_info=model_info,
vertex_location=vertex_location,
)
if _uses_native_vertex_output(
custom_llm_provider, model_name, next(_iter_batch_output_entries(file_content), None)
@ -151,6 +163,7 @@ async def _handle_completed_batch(
custom_llm_provider=custom_llm_provider,
model_name=model_name,
model_info=model_info,
vertex_location=vertex_location,
)
)
@ -186,6 +199,7 @@ def _classify_output_line_stats(
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
model_info: ModelInfo | None,
vertex_location: str | None = None,
) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
"""Classify every output line in a single pass, so counting failures never needs
a second read of a potentially huge output file. A line the provider reported as
@ -196,7 +210,7 @@ def _classify_output_line_stats(
if not _batch_response_was_successful(entry, custom_llm_provider):
yield _LineOutcome.PROVIDER_FAILED
continue
stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
yield stats if stats is not None else _LineOutcome.UNCOSTABLE
@ -205,13 +219,14 @@ def _safe_output_line_stats(
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
model_info: ModelInfo | None,
vertex_location: str | None = None,
) -> _BatchOutputLineStats | None:
"""Return the stats for one provider-successful batch output line, or None when
it cannot be costed, so a single bad line never aborts the whole batch's cost
accounting."""
custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
try:
return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
except Exception as e: # noqa: BLE001 # any single line's costing failure must not abort the whole batch
verbose_logger.warning(
"batch output line could not be costed, so it is billed at $0 and the rest of the batch "
@ -227,6 +242,7 @@ def _compute_output_line_stats(
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
model_info: ModelInfo | None,
vertex_location: str | None = None,
) -> _BatchOutputLineStats:
response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
@ -241,6 +257,7 @@ def _compute_output_line_stats(
model_name=model_name,
response_model=response_model,
model_info=model_info,
vertex_location=vertex_location,
)
return _BatchOutputLineStats(
prompt_cost=line_prompt_cost,
@ -270,6 +287,7 @@ def _output_line_cost(
model_name: str | None,
response_model: str | None,
model_info: ModelInfo | None,
vertex_location: str | None = None,
) -> tuple[float, float]:
"""(prompt_cost, completion_cost) for one output line, priced at batch rates."""
from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
@ -290,6 +308,7 @@ def _output_line_cost(
model=cost_model,
custom_llm_provider=custom_llm_provider,
model_info=model_info,
vertex_location=vertex_location,
)
@ -298,11 +317,14 @@ def _aggregate_batch_cost_usage_models(
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None = None,
model_info: ModelInfo | None = None,
vertex_location: str | None = None,
) -> BatchCostUsageResult:
"""Aggregate cost, usage, models, and pass/fail counts from batch output
entries in a single pass, holding one small stats record per line instead
of the parsed file."""
all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
all_results: Final = tuple(
_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info, vertex_location)
)
line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
successful_requests: Final = len(all_results) - failed_requests
@ -349,6 +371,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
vertex_ai_batch_responses: Iterable[dict],
model_name: str | None = None,
model_info: ModelInfo | None = None,
vertex_location: str | None = None,
) -> BatchCostUsageResult:
"""
Cost and usage of a native Vertex predictions.jsonl, one
@ -367,6 +390,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
model_info=model_info,
calculate_usage=VertexGeminiConfig._calculate_usage,
cost_calculator=batch_cost_calculator,
vertex_location=vertex_location,
)
for row in vertex_ai_batch_responses
)

1
litellm/constants.py
View file

@ -1871,6 +1871,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"apply_search_free_tier",
"cost_margin_config",
"block_requests_for_models_without_pricing",
"budget_exceeded_throttle_percentage",

7
litellm/cost_calculator.py
View file

@ -36,6 +36,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_batch_cost_rates,
get_billable_input_tokens,
get_token_type_cost_breakdown,
get_vertex_regional_endpoint_uplift,
parse_prompt_tokens_details,
select_cost_metric_for_model,
)
@ -631,6 +632,7 @@ def cost_per_token(
model=model,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
vertex_location=vertex_location,
)
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
@ -2664,6 +2666,8 @@ def batch_cost_calculator(
custom_llm_provider: str | None = None,
model_info: ModelInfo | None = None,
data_residency: str | None = None,
*,
vertex_location: str | None = None,
) -> tuple[float, float]:
"""
Calculate the cost of a batch job.
@ -2749,7 +2753,8 @@ def batch_cost_calculator(
total_prompt_cost *= uplift
total_completion_cost *= uplift
return total_prompt_cost, total_completion_cost
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
def _batch_prompt_cost(

27
litellm/decisions/call.py
View file

@ -1,9 +1,11 @@
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from typing import Final, TypeAlias
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError
from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
import litellm
@ -30,6 +32,7 @@ from litellm.types.decisions import (
OpenAIDecisionResponse,
UnsupportedDecisionsRequest,
)
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
@ -148,6 +151,30 @@ def _provider_ir_request(
return _UnsupportedSafetyIdentifier()
_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
def _jsonable_fallback(value: object) -> object:
return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64")
)
sent_fields: Final = json.dumps(
dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
)
return ChatCompletionUserMessage(role="user", content=sent_fields)
def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
"""Record every request field the caller sent as the logged `messages`, before any validation."""
logging_obj: Final = kwargs.get("litellm_logging_obj")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.update_messages([_logged_request_message(caller_request)])
def prepare_call(
*,
model: str,

9
litellm/decisions/main.py
View file

@ -13,6 +13,7 @@ from litellm.decisions.call import (
SystemOneQuestions,
SystemOneRequestFields,
asend,
log_caller_request,
prepare_call,
send,
)
@ -59,6 +60,14 @@ def _prepare_decisions_call(
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> DecisionsCall:
log_caller_request(
kwargs,
{
"input": decision_input,
"questions": questions,
**({"state": kwargs["state"]} if "state" in kwargs else {}),
},
)
return prepare_call(
model=model,
fields=_request_fields(decision_input, questions, kwargs),

79
litellm/litellm_core_utils/litellm_logging.py
View file

@ -42,6 +42,7 @@ from litellm.constants import (
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
EMPTY_MAPPING,
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
PROVIDER_REQUEST_ID_HEADERS,
SENTRY_DENYLIST,
SENTRY_PII_DENYLIST,
@ -484,7 +485,7 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None:
return None
def _resolve_vertex_location_for_cost(
def resolve_vertex_location_for_cost(
custom_llm_provider: str | None,
litellm_params: Mapping[str, object] | None,
optional_params: Mapping[str, object] | None,
@ -710,6 +711,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._defer_async_logging: bool = False
self._enqueue_deferred_logging: Callable[[], None] | None = None
self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self._async_success_enqueued: bool = False
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
@ -1401,15 +1403,20 @@ class Logging(LiteLLMLoggingBaseClass):
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
)
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = self.model_call_details["api_call_start_time"]
# A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks
# without any provider handoff, so it must not stamp the handoff instants.
if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details:
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = self.model_call_details[
"api_call_start_time"
]
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
for callback in callbacks:
@ -1876,7 +1883,7 @@ class Logging(LiteLLMLoggingBaseClass):
router_model_id = self.get_router_model_id()
## RESPONSE COST ##
custom_pricing: Final = self._custom_pricing_for(priced_result)
custom_pricing: Final = self.custom_pricing_for(priced_result)
prompt = self._prompt_for_cost_calculation()
@ -1903,7 +1910,7 @@ class Logging(LiteLLMLoggingBaseClass):
if hasattr(self, "litellm_params") and self.litellm_params
else None
),
"vertex_location": _resolve_vertex_location_for_cost(
"vertex_location": resolve_vertex_location_for_cost(
custom_llm_provider=self.model_call_details.get("custom_llm_provider", None),
litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None),
optional_params=self.optional_params,
@ -2015,7 +2022,7 @@ class Logging(LiteLLMLoggingBaseClass):
model=model,
completion_response=result,
custom_llm_provider=custom_llm_provider,
custom_pricing=self._custom_pricing_for(result),
custom_pricing=self.custom_pricing_for(result),
base_model=_get_base_model_from_metadata(model_call_details=self.model_call_details),
router_model_id=router_model_id or self.get_router_model_id(),
region_name=_resolve_mantle_region_for_cost(
@ -2040,20 +2047,23 @@ class Logging(LiteLLMLoggingBaseClass):
usage=usage,
)
def _custom_pricing_for(self, result: object) -> bool:
litellm_params: Final = getattr(self, "litellm_params", None)
def spilled_over_ptu_for(self, result: object) -> bool:
result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({})
additional_headers: Final = (
result_hidden_params.get("additional_headers")
if isinstance(result_hidden_params, dict)
else getattr(result_hidden_params, "additional_headers", None)
)
spilled_over: Final = is_spilled_over_ptu_request(
model_info=_deployment_model_info(litellm_params),
return is_spilled_over_ptu_request(
model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
response_headers=self.model_call_details.get("response_headers"),
additional_headers=additional_headers,
)
return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False
return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
def _prompt_for_cost_calculation(self) -> str:
"""
@ -2260,6 +2270,16 @@ class Logging(LiteLLMLoggingBaseClass):
except Exception:
return True
@property
def async_success_log_enqueued(self) -> bool:
"""True once the success log task was actually created, not merely claimed, so a later failure raised
around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second
time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""
return self._async_success_enqueued
def mark_async_success_log_enqueued(self) -> None:
self._async_success_enqueued = True
def has_run_logging(
self,
event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
@ -3261,6 +3281,12 @@ class Logging(LiteLLMLoggingBaseClass):
model_name=self.get_deployment_model_for_cost(),
litellm_params=self.litellm_params,
model_info=self.get_router_deployment_model_info(),
vertex_location=resolve_vertex_location_for_cost(
custom_llm_provider=self.custom_llm_provider,
litellm_params=self.litellm_params if hasattr(self, "litellm_params") else None,
optional_params=None,
model=self.get_deployment_model_for_cost() or self.model,
),
)
result._hidden_params["response_cost"] = batch_result.cost
@ -6519,13 +6545,16 @@ def get_standard_logging_object_payload(
)
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
# The router overrides completion_response.model to the model-group alias before
# this payload is built, so cost-map lookup via that alias always misses.
# Fall back to the actual deployment model set by the router in metadata.
if base_model is None:
base_model = metadata.get("deployment")
custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params)
configured_base_model: Final = _get_base_model_from_metadata(model_call_details=kwargs)
spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
# The router overrides completion_response.model to the model-group alias before this payload is
# built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
# A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
# which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
base_model: Final = (
metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
)
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))

60
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1071,6 +1071,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
)
def _modality_token_cost(
model_info: ModelInfo,
modality_cost_key: str,
tokens: int,
prompt_base_cost: float,
service_tier: str | None,
) -> float:
"""
Audio, image and video prompt tokens bill at their modality rate when the model has one.
Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
`_get_token_base_cost` has already resolved for the service tier and long-context tier.
"""
tiered_key: Final = _get_service_tier_cost_key(modality_cost_key, service_tier)
cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
if model_info.get(cost_key) is None:
return float(tokens) * prompt_base_cost
return calculate_cost_component(model_info, cost_key, tokens)
def _calculate_input_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
@ -1082,8 +1101,14 @@ def _calculate_input_cost(
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
A model priced per character (`input_cost_per_character`) bills the reported character count
in place of its text tokens, so the same prompt is never charged under both units.
"""
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
bills_characters: Final = (
prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None
)
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
@ -1101,28 +1126,37 @@ def _calculate_input_cost(
if prompt_tokens_details["audio_tokens"] and not (
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
):
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_audio_token",
prompt_tokens_details["audio_tokens"],
prompt_base_cost,
service_tier,
)
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"] and not (
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
):
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
if model_info.get(image_token_cost_key) is None:
image_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_image_token",
prompt_tokens_details["image_tokens"],
prompt_base_cost,
service_tier,
)
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"] and not (
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
):
video_token_cost_key = "input_cost_per_video_token"
if model_info.get(video_token_cost_key) is None:
video_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_video_token",
prompt_tokens_details["video_tokens"],
prompt_base_cost,
service_tier,
)
### CACHE WRITING COST - Now uses tiered pricing
if (

29
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -4,6 +4,7 @@ from functools import reduce
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
@ -101,6 +102,32 @@ def response_timing_metrics(
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
try:
return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
except ValidationError:
return {}
def _existing_provider_response_model(result: object) -> str | None:
"""The pricing model a response already carries, like the decisions path sets, wins over the response model."""
hidden_params: Final[object] = getattr(result, "_hidden_params", None)
carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
hidden_params, "provider_response_model", None
)
return carried if isinstance(carried, str) and carried else None
def _provider_response_model(result: object) -> str | None:
"""The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
return model if isinstance(model, str) and model else None
class ResponseMetadata:
"""
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@ -133,6 +160,8 @@ class ResponseMetadata:
preserve_litellm_internal_headers=True,
),
"litellm_model_name": model,
"provider_response_model": _existing_provider_response_model(self.result)
or _provider_response_model(self.result),
}
self._update_hidden_params(new_params)

23
litellm/litellm_core_utils/request_timeout_resolver.py
View file

@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
from __future__ import annotations
from typing import Final
from collections.abc import Mapping
from typing import Final, Literal, Protocol
from pydantic import TypeAdapter
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
def get_configured_request_timeout() -> float | None:
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
return timeout
return None
class _DeploymentParams(Protocol):
def get(self, key: str, /) -> object: ...
def get_service_tier_timeout(
service_tier: object,
litellm_params: _DeploymentParams,
key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
) -> float | None:
"""Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
if not isinstance(service_tier, str):
return None
timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
return None if timeouts is None else timeouts.get(service_tier)

15
litellm/llms/anthropic/chat/transformation.py
View file

@ -103,6 +103,7 @@ from ..common_utils import (
process_anthropic_headers,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
tools_marked_for_a_tool_config_point,
)
if TYPE_CHECKING:
@ -2056,6 +2057,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Remove internal LiteLLM parameters that should not be sent to Anthropic API
optional_params.pop("is_vertex_request", None)
optional_params.pop("client_metadata", None)
request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature
dict[str, object], optional_params
)
marked_tools: Final = tools_marked_for_a_tool_config_point(
request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
)
if marked_tools is not None:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature
Mapping[str, object], litellm_params
)
AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1)
# ``top_k`` is a provider-specific kwarg that bypasses
# ``map_openai_params``; gate it here, the single boundary shared by
@ -2075,6 +2089,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"model": model,
"messages": anthropic_messages,
**optional_params,
**({} if marked_tools is None else {"tools": list(marked_tools)}),
}
self._apply_output_config(data=data, model=model, optional_params=optional_params)

43
litellm/llms/anthropic/common_utils.py
View file

@ -33,6 +33,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_OAUTH_BETA_HEADER,
ANTHROPIC_OAUTH_TOKEN_PREFIX,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesToolChoice,
@ -73,7 +74,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
_DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@ -115,6 +116,40 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None
)
def _tool_config_point_control(points: object) -> Mapping[object, object] | None:
point_fields: Final = tuple(_validated_claude_code_mapping(point) for point in validated_object_list(points) or ())
tool_config_point: Final = next(
(fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None
)
if tool_config_point is None:
return None
control: Final = tool_config_point.get("control")
return {"type": "ephemeral"} if control is None else _validated_claude_code_mapping(control)
def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None:
"""Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches.
Anthropic caches tools as one prefix ending at the marked tool, so the last one carries
the mark, skipping tool-search tools whose transform drops cache markers. A client that
already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked.
"""
control: Final = _tool_config_point_control(points)
tool_list: Final = validated_object_list(tools) or ()
tool_fields: Final = tuple(_validated_claude_code_mapping(tool) for tool in tool_list)
if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields):
return None
cacheable: Final = tuple(
(position, fields)
for position, fields in enumerate(tool_fields)
if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
)
if not cacheable:
return None
position, fields = cacheable[-1]
return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :])
def is_claude_code_user_agent(user_agent: str) -> bool:
"""Claude Code sends its API calls through the Anthropic SDK as `claude-cli/<version>` and its own
fetches, such as gateway model discovery, as `claude-code/<version>`"""
@ -128,9 +163,9 @@ def _validated_claude_code_mapping(value: object) -> dict[object, object] | None
return None
def _validated_claude_code_list(value: object) -> list[object] | None:
def validated_object_list(value: object) -> Sequence[object] | None:
try:
return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value)
return OBJECT_LIST_ADAPTER.validate_python(value)
except ValidationError:
return None
@ -157,7 +192,7 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None
def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None:
if isinstance(system, str):
return (system,)
blocks: Final = _validated_claude_code_list(system)
blocks: Final = validated_object_list(system)
if blocks is None:
return None
block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks)

10
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,7 @@ from functools import partial
from typing import Any, Final, cast
import litellm
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
@ -19,6 +20,7 @@ from litellm.llms.anthropic.common_utils import (
sanitize_tool_use_ids_in_anthropic_messages,
strip_empty_content_blocks_from_anthropic_messages,
strip_provider_specific_fields_from_anthropic_messages,
tools_marked_for_a_tool_config_point,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
@ -636,7 +638,13 @@ def anthropic_messages_handler(
f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}"
)
local_vars.update(kwargs)
request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped
marked_tools: Final = tools_marked_for_a_tool_config_point(
local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None)
)
if marked_tools is not None:
AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1)
local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})})
anthropic_messages_optional_request_params: Final = (
AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
params=local_vars,

38
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -4,11 +4,12 @@ import asyncio
import json
import traceback
from collections import deque
from collections.abc import AsyncIterator, Iterator, Mapping
from collections.abc import AsyncGenerator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
@ -25,6 +26,7 @@ from .transformation import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.types.llms.openai import ResponseIncompleteEvent
class AnthropicResponsesStreamWrapper:
@ -391,8 +393,11 @@ class AnthropicResponsesStreamWrapper:
raise StopAsyncIteration
async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]:
"""Yield SSE-encoded bytes for each Anthropic event chunk."""
def async_anthropic_sse_wrapper(self) -> "AnthropicResponsesSSEByteStream":
"""SSE-encoded bytes for each Anthropic event chunk."""
return AnthropicResponsesSSEByteStream(self)
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
async for chunk in self:
if isinstance(chunk, dict):
event_type: str = str(chunk.get("type", "message"))
@ -400,3 +405,30 @@ class AnthropicResponsesStreamWrapper:
yield payload.encode()
else:
yield chunk
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
return partial_responses_incomplete_event_on_disconnect(self.responses_stream)
class AnthropicResponsesSSEByteStream:
"""
The byte stream the proxy relays for a default-route /v1/messages request.
Keeps the Responses API iterator reachable so a client disconnect can bill
the partial output the way native /v1/responses streams do.
"""
def __init__(self, wrapper: AnthropicResponsesStreamWrapper) -> None:
self._wrapper: Final = wrapper
self._frames: Final = wrapper._sse_frames() # pyright: ignore[reportPrivateUsage] # the wrapper's own byte stream
def __aiter__(self) -> "AnthropicResponsesSSEByteStream":
return self
async def __anext__(self) -> bytes:
return await self._frames.__anext__()
async def aclose(self) -> None:
await self._frames.aclose()
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
return self._wrapper.partial_response_on_disconnect()

9
litellm/llms/vertex_ai/batches/transformation.py
View file

@ -54,6 +54,8 @@ class NativeVertexBatchCostCalculator(Protocol):
model: str,
custom_llm_provider: str | None = None,
model_info: ModelInfo | None = None,
*,
vertex_location: str | None = None,
) -> tuple[float, float]: ...
@ -90,6 +92,7 @@ def native_vertex_batch_row_stats(
model_info: ModelInfo | None,
calculate_usage: Callable[[GenerateContentResponseBody], Usage],
cost_calculator: NativeVertexBatchCostCalculator,
vertex_location: str | None = None,
) -> NativeVertexBatchRowStats | None:
"""
Usage and cost of one native Vertex predictions.jsonl row, a
@ -121,7 +124,11 @@ def native_vertex_batch_row_stats(
)
try:
prompt_cost, completion_cost = cost_calculator(
usage=usage, model=model, custom_llm_provider="vertex_ai", model_info=model_info
usage=usage,
model=model,
custom_llm_provider="vertex_ai",
model_info=model_info,
vertex_location=vertex_location,
)
except Exception as e: # noqa: BLE001 # one unpriceable row must not abort the batch's cost accounting
verbose_logger.warning(

34
litellm/llms/vertex_ai/vertex_embeddings/transformation.py
View file

@ -1,14 +1,34 @@
import types
from typing import Final, Literal
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
from litellm.types.utils import EmbeddingResponse, Usage
from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage
from .types import *
class _PredictMetadata(TypedDict, total=False):
billableCharacterCount: ReadOnly[int]
class _PredictResponseMetadata(TypedDict, total=False):
metadata: ReadOnly[_PredictMetadata]
_PREDICT_RESPONSE_METADATA: Final = TypeAdapter(_PredictResponseMetadata)
def billable_character_count(response: _PredictResponseMetadata) -> int | None:
"""
`metadata.billableCharacterCount` of a Vertex text embedding predict response, the count Google
bills per-character embedding models (text-embedding-005, text-multilingual-embedding-002) on.
"""
return response.get("metadata", {}).get("billableCharacterCount")
class VertexAITextEmbeddingConfig(BaseModel):
"""
Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#TextEmbeddingInput
@ -248,7 +268,15 @@ class VertexAITextEmbeddingConfig(BaseModel):
model_response.object = "list"
model_response.data = embedding_response
model_response.model = model
usage: Final = Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens)
billable_characters: Final = billable_character_count(_PREDICT_RESPONSE_METADATA.validate_python(response))
usage: Final = Usage(
prompt_tokens=input_tokens,
completion_tokens=0,
total_tokens=input_tokens,
prompt_tokens_details=(
None if billable_characters is None else PromptTokensDetailsWrapper(character_count=billable_characters)
),
)
setattr(model_response, "usage", usage)
return model_response

19
litellm/passthrough/timeout_utils.py
View file

@ -5,6 +5,8 @@ from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
_SECONDS: Final = TypeAdapter(float)
@ -48,21 +50,27 @@ def resolve_llm_passthrough_timeout(
Anthropic /v1/messages).
Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout
-> 600s default.
timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout
-> general_settings.pass_through_request_timeout -> 600s default.
Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the
non-streaming chain above.
kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] ->
litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above.
Only the first set value is validated as seconds, so a value in a lower-precedence
field never fails the call.
"""
request: Final = kwargs if kwargs is not None else _NO_PARAMS
deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
service_tier: Final = request.get("service_tier")
stream_candidates: Final = (
(request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout)
(
request.get("stream_timeout"),
get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"),
deployment.get("stream_timeout"),
router_stream_timeout,
)
if request.get("stream")
else ()
)
@ -70,6 +78,7 @@ def resolve_llm_passthrough_timeout(
*stream_candidates,
request.get("timeout"),
request.get("request_timeout"),
get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"),
deployment.get("timeout"),
deployment.get("request_timeout"),
router_timeout,

12
litellm/proxy/common_request_processing.py
View file

@ -111,6 +111,7 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
from litellm.router import Router
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.common_utils import resolve_model_group_alias
@ -418,6 +419,13 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
# A natural end-of-stream success event already fired and released the
# slot; do not bill again, and let the caller skip the slot release.
return True
partial_responses_event: Final = partial_responses_incomplete_event_on_disconnect(response)
if partial_responses_event is not None:
verbose_proxy_logger.debug(
"Billing partial streamed Responses API spend after client disconnect, litellm_call_id=%s",
logging_obj.litellm_call_id,
)
return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event)
chunks: Final[object] = getattr(response, "chunks", None)
if not isinstance(chunks, list) or not chunks:
return False
@ -447,6 +455,10 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
partial_usage: Final = getattr(partial_response, "usage", None)
if isinstance(partial_usage, Usage):
backfill_missing_cache_usage_fields(partial_usage)
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
try:
await logging_obj.dispatch_success_handlers(
partial_response,

27
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -119,6 +119,23 @@ def coerce_numeric_form_fields(
}
def _parse_form_metadata(raw_metadata: str) -> object:
"""The JSON body path rejects a lone surrogate escape (``"\\ud800"``) with a
400 because no provider or database can encode it. ``json.loads`` accepts
the escape, so the multipart ``metadata`` field gets the same check."""
parsed: Final[object] = json.loads(raw_metadata)
try:
json.dumps(parsed, ensure_ascii=False).encode("utf-8")
except UnicodeEncodeError as e:
raise ProxyException(
message=f"Invalid form payload: metadata {e}",
type="invalid_request_error",
param="metadata",
code=status.HTTP_400_BAD_REQUEST,
) from e
return parsed
async def _read_request_body(request: Request | None) -> dict:
"""
Safely read the request body and parse it as JSON.
@ -158,9 +175,13 @@ async def _read_request_body(request: Request | None) -> dict:
param="request_body",
code=status.HTTP_400_BAD_REQUEST,
)
parsed_body = dict(form_data)
if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str):
parsed_body["metadata"] = json.loads(parsed_body["metadata"])
form_body: Final = dict(form_data)
raw_metadata: Final = form_body.get("metadata")
parsed_body = (
{**form_body, "metadata": _parse_form_metadata(raw_metadata)}
if isinstance(raw_metadata, str)
else form_body
)
else:
# Read the request body
body: Final = await request.body()

48
litellm/proxy/db/db_spend_update_writer.py
View file

@ -88,6 +88,7 @@ if TYPE_CHECKING:
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
else:
PrismaClient = Any
ProxyLogging = Any
@ -97,6 +98,34 @@ RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, Call
_SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _turn_chains_through_the_db(payload: Mapping[str, object], llm_router: "Router | None") -> bool:
"""A Responses turn the chat-completions bridge replays from its spend log on the next `previous_response_id`.
Native Responses providers hold the conversation themselves, so their rows can wait for the batch write.
A row with no router deployment behind it is treated as bridged, which keeps today's immediate flush.
"""
if payload.get("call_type") not in RESPONSES_SESSION_CALL_TYPES:
return False
model_id: Final = payload.get("model_id")
deployment: Final = (
llm_router.get_deployment(model_id=model_id) if llm_router is not None and isinstance(model_id, str) else None
)
if deployment is None:
return True
from litellm.responses.main import provider_stores_the_conversation
litellm_params: Final = deployment.litellm_params
return not provider_stores_the_conversation(
litellm_params.model,
litellm_params.custom_llm_provider,
{
"use_chat_completions_api": litellm_params.use_chat_completions_api,
"model_info": deployment.model_info.model_dump(),
"api_base": litellm_params.api_base,
},
)
def _org_member_transaction_key(org_id: str, user_id: str) -> str:
return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
@ -527,12 +556,13 @@ class DBSpendUpdateWriter:
get_request_model_access_groups,
)
llm_router: Final = get_llm_router()
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=completion_response,
start_time=start_time,
end_time=end_time,
llm_router=get_llm_router(),
llm_router=llm_router,
)
payload["spend"] = response_cost or 0.0
if isinstance(payload["startTime"], datetime):
@ -547,7 +577,10 @@ class DBSpendUpdateWriter:
payload["team_id"] = team_id
if not await self._record_spend_log(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
payload=payload,
prisma_client=prisma_client,
disable_spend_logs=disable_spend_logs,
llm_router=llm_router,
):
return False
@ -608,14 +641,18 @@ class DBSpendUpdateWriter:
return True
async def _record_spend_log(
self, payload: SpendLogsPayload, prisma_client: "PrismaClient | None", disable_spend_logs: bool
self,
payload: SpendLogsPayload,
prisma_client: "PrismaClient | None",
disable_spend_logs: bool,
llm_router: "Router | None" = None,
) -> bool:
if prisma_client is not None and _is_batch_cost_row(payload):
return await self._claim_batch_cost_spend_log(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
)
if disable_spend_logs is False:
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client)
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router)
return True
async def _claim_batch_cost_spend_log(
@ -1462,6 +1499,7 @@ class DBSpendUpdateWriter:
payload: dict | SpendLogsPayload,
prisma_client: PrismaClient | None = None,
spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"),
llm_router: "Router | None" = None,
) -> PrismaClient | None:
verbose_proxy_logger.debug(
"Writing spend log to db - request_id: {}, spend: {}".format(
@ -1472,7 +1510,7 @@ class DBSpendUpdateWriter:
from litellm.proxy.utils import enqueue_spend_logs, request_spend_log_flush
await enqueue_spend_logs(prisma_client, (payload,))
if payload.get("call_type") in RESPONSES_SESSION_CALL_TYPES:
if _turn_chains_through_the_db(_SPEND_METADATA_ADAPTER.validate_python(payload), llm_router):
request_spend_log_flush(prisma_client)
else:
verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.")

138
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -3,10 +3,11 @@ import traceback
from collections.abc import Callable, Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
@ -33,6 +34,7 @@ from litellm.proxy.db.db_spend_update_writer import (
debitable_model_access_groups,
get_llm_router,
)
from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.spend_tracking.spend_event import (
ObjectMapping,
@ -53,7 +55,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
get_request_model_access_groups,
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend
from litellm.types.utils import (
CallTypes,
LiteLLMBatch,
@ -126,6 +128,79 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
return proxy_logging_obj.db_spend_update_writer
def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
attempts: Final = request_data.get("billed_failed_attempts")
if not isinstance(attempts, tuple):
return ()
lifted: Final = cast("tuple[object, ...]", attempts) # cast-ok: lifted by _failure_fields_to_lift, checked below
return tuple(attempt for attempt in lifted if isinstance(attempt, BilledFailedAttempt))
def _failure_spend_rows(
request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object
) -> tuple[tuple[float, Mapping[str, object]], ...]:
"""One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the
request id, then every earlier attempt the provider also billed under its own row."""
final_row: Final[Mapping[str, object]] = (
request_data
if late_guardrails_snapshot is None
else {**request_data, "standard_logging_object": late_guardrails_snapshot}
)
return (
(final_cost, final_row),
*(
(attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))
for position, attempt in enumerate(_billed_failed_attempts(request_data))
),
)
def _attempt_spend_row_kwargs(
request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int
) -> Mapping[str, object]:
"""The final attempt's row kwargs re-keyed for one earlier billed attempt: its own
``request_id`` (``<id>-attempt-<position>``, since ``request_id`` is the spend-log primary key
and the retry count restarts after a fallback hop), its estimated usage and cost, its retry
count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""
suffix: Final = f"-attempt-{position}"
raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict
litellm_params: Final = dict(raw_params)
metadata: Final = {
**cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict
"attempted_retries": attempt.attempted_retries,
}
metadata.pop("previous_models", None)
if attempt.deployment_id is not None:
metadata["model_info"] = {
**cast("Mapping[str, object]", metadata.get("model_info") or {}), # cast-ok: router dict
"id": attempt.deployment_id,
}
litellm_params["metadata"] = metadata
row: Final = {
**request_data,
"litellm_params": litellm_params,
"combined_usage_object": attempt.usage,
"response_cost": attempt.response_cost,
}
row.pop("billed_failed_attempts", None)
call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
if isinstance(call_id, str):
row["litellm_call_id"] = f"{call_id}{suffix}"
standard_logging_object: Final = request_data.get("standard_logging_object")
if isinstance(standard_logging_object, dict):
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
if isinstance(payload.get("id"), str):
row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}
return row
def _str_keyed_fields(value: object) -> Mapping[str, object]:
if not isinstance(value, Mapping):
return {}
return cast("Mapping[str, object]", value) # cast-ok: payload dicts are str-keyed
class _ProxyDBLogger(CustomLogger):
def __init__(
self,
@ -293,20 +368,40 @@ class _ProxyDBLogger(CustomLogger):
existing_metadata.get("standard_logging_guardrail_information")
)
await self._spend_writer().update_database(
token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
response_cost=recovered_response_cost,
user_id=user_api_key_dict.user_id,
end_user_id=user_api_key_dict.end_user_id,
team_id=user_api_key_dict.team_id,
kwargs=request_data,
completion_response=original_exception,
await self._write_failure_spend_rows(
user_api_key_dict=user_api_key_dict,
original_exception=original_exception,
start_time=actual_start_time,
end_time=datetime.now(),
org_id=user_api_key_dict.org_id,
project_id=user_api_key_dict.project_id,
spend_rows=_failure_spend_rows(
cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below
recovered_response_cost,
None,
),
)
async def _write_failure_spend_rows(
self,
user_api_key_dict: UserAPIKeyAuth,
original_exception: Exception,
start_time: datetime,
spend_rows: Sequence[tuple[float, Mapping[str, object]]],
) -> None:
"""One spend row (and one key/team/user spend deduction) per ``(response_cost, kwargs)`` pair."""
for row_cost, row_kwargs in spend_rows:
await self._spend_writer().update_database(
token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
response_cost=row_cost,
user_id=user_api_key_dict.user_id,
end_user_id=user_api_key_dict.end_user_id,
team_id=user_api_key_dict.team_id,
kwargs=dict(row_kwargs),
completion_response=original_exception,
start_time=start_time,
end_time=datetime.now(),
org_id=user_api_key_dict.org_id,
project_id=user_api_key_dict.project_id,
)
@log_db_metrics
async def _PROXY_track_cost_callback(
self,
@ -371,8 +466,21 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
raw_cost: Final = _str_keyed_fields(
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
).get("response_cost")
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
response_cost: Final = (
sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
await apply_search_free_tier(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists
response_cost=priced_cost,
completion_response=completion_response,
counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache,
now=datetime.now(timezone.utc),
)
if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None
else priced_cost
)
tags: Final = _get_request_tags_for_cost_tracking(
sl_object=sl_object,

123
litellm/proxy/hooks/search_free_tier.py Normal file
View file

@ -0,0 +1,123 @@
from collections.abc import MutableMapping
from datetime import datetime
from types import MappingProxyType
from typing import Final, Protocol
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage
_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"})
_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"})
_TTL_GRACE_SECONDS: Final = 3600
_COUNTED_KWARG: Final = "search_free_tier_counted"
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
class SharedCounter(Protocol):
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ...
class _UsageCarrier(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
usage: Usage | None = None
def freed_units(allowance: int, units: int, total_after: int) -> int:
"""Units of this request that landed under the allowance, given the shared total after they were added."""
return max(0, min(units, allowance - (total_after - units)))
def search_free_tier_key(provider: str, allowance: SearchFreeRequests, now: datetime) -> str:
period_stamp: Final = now.strftime(_PERIOD_STAMP[allowance["period"]])
return f"litellm:search_free_tier:{provider}:{allowance['shared_group']}:{period_stamp}"
def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int:
reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC")
return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS
def _usage_of(completion_response: object) -> Usage | None:
try:
return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage
except ValidationError:
return None
def _grounding_queries(usage: Usage) -> int:
details: Final = usage.prompt_tokens_details
from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None
return from_details or get_web_search_requests_from_usage(usage) or 0
def _billable_units(usage: Usage, model_info: ModelInfo) -> int:
queries: Final = _grounding_queries(usage)
if queries == 0:
return 0
return queries if model_info.get("web_search_billing_unit") == "per_query" else 1
def _record_freed_cost(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
original_cost: float,
freed_cost: float,
) -> float:
reduced_cost: Final = original_cost - freed_cost
breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost)
sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload
{
**breakdown,
"total_cost": reduced_cost,
"original_cost": breakdown.get("original_cost", original_cost),
"discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost,
}
)
sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload
kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"]
return reduced_cost
async def apply_search_free_tier(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
response_cost: float,
completion_response: object,
counter: SharedCounter | None,
now: datetime,
) -> float:
"""Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged."""
model_info: Final = sl_object["model_map_information"]["model_map_value"]
allowance: Final = model_info.get("search_free_requests") if model_info is not None else None
usage: Final = _usage_of(completion_response)
if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]:
return response_cost
units: Final = _billable_units(usage, model_info)
if units == 0 or kwargs.get(_COUNTED_KWARG):
return response_cost
if counter is None:
verbose_proxy_logger.warning(
"apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price"
)
return response_cost
provider: Final = sl_object["custom_llm_provider"] or model_info["litellm_provider"]
key: Final = search_free_tier_key(provider, allowance, now)
try:
total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now))
except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row
verbose_proxy_logger.warning(
"search free tier counter %s is unavailable, grounding is billed at full price", key
)
return response_cost
kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once
freed: Final = freed_units(allowance["count"], units, int(total_after))
if freed == 0:
return response_cost
freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units)
return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost)

6
litellm/proxy/litellm_pre_call_utils.py
View file

@ -358,8 +358,9 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess
# ``litellm.model_cost`` map. Both effects belong to deployment configuration,
# not to user-supplied request bodies, so the proxy strips them before they
# reach the call path. Built from the Pydantic model so newly-added pricing
# fields are covered automatically.
_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
# fields are covered automatically. ``base_model`` selects which cost-map entry
# prices the row, so a caller naming a cheaper model lowers their own spend.
_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"}
# ``model_info`` carries the same pricing fields when read by
# ``use_custom_pricing_for_model``; strip from metadata for the same reason.
# ``standard_logging_guardrail_information`` is proxy-written telemetry summed
@ -373,6 +374,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
{
"attempted_fallbacks",
"original_model_group",
"previous_models",
"request_retry_count",
CLIENT_OUTPUT_CEILING_METADATA_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,

306
litellm/proxy/utils.py
View file

@ -46,11 +46,13 @@ from typing import (
overload,
)
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm import _custom_logger_compatible_callbacks_literal
from litellm.constants import (
DEFAULT_MODEL_CREATED_AT_TIME,
LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
MAX_TEAM_LIST_LIMIT,
PROXY_REJECTED_BEFORE_ROUTING_KEY,
@ -143,7 +145,8 @@ from litellm.litellm_core_utils.core_helpers import (
independent_snapshot,
is_expected_client_error,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.served_output_texts import (
@ -236,6 +239,7 @@ from litellm.repositories.verification_token_repository import (
VerificationTokenRepository,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.deployment_pricing import pricing_models
from litellm.secret_managers.main import str_to_bool
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
from litellm.types.llms.openai import ResponsesAPIResponse
@ -248,6 +252,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionRe
from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams
from litellm.utils import (
_add_custom_logger_callback_to_specific_event, # pyright: ignore[reportPrivateUsage] # only string-to-logger helper
_get_base_model_from_metadata, # pyright: ignore[reportPrivateUsage] # main exposes it as get_base_model_from_metadata
)
if TYPE_CHECKING:
@ -939,20 +944,48 @@ def _failure_usage_to_lift(
model_call_details: Mapping[str, object],
request_body: Mapping[str, object],
dispatched: bool,
provider_billed: bool = False,
deployment_id: str | None = None,
base_model: str | None = None,
) -> tuple[object, object] | None:
"""A stream that broke mid-flight still billed the provider for the chunks
already delivered; the streaming handler stashes that recovered usage and
cost in model_call_details, so prefer it. Otherwise a request that was
dispatched to a provider and failed without upstream usage gets an
estimated input-side Usage with zero cost. The raw request body backfills
the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a
chat-completions provider) without filling optional_params. Returns the
estimated input-side Usage. When the failure is one the provider bills
anyway (``provider_billed``: a read timeout or a client disconnect, where
the provider had the prompt and kept working after LiteLLM hung up) that
usage is priced at the model's input rate, the way the success path prices it:
at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``)
when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure
deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch
failure stays at zero cost. Returns the
(combined_usage_object, response_cost) pair to lift, or None."""
recovered_usage: Final = model_call_details.get("combined_usage_object")
if recovered_usage is not None:
return recovered_usage, model_call_details.get("response_cost")
if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL):
if not dispatched:
return None
if model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL) and not provider_billed:
# The proxy-only error handler stamps this flag on a client disconnect too, even though the
# request had already reached the provider, so a billed failure keeps its estimate.
return None
estimated_usage: Final = _estimated_prompt_usage(model_call_details, request_body)
if estimated_usage is None:
return None
if not provider_billed:
return estimated_usage, 0.0
return estimated_usage, _estimated_input_cost(
model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model
)
def _estimated_prompt_usage(
model_call_details: Mapping[str, object], request_body: Mapping[str, object]
) -> Usage | None:
"""The input-only Usage estimate for the prompt this request sent, or None when the call type has no
estimable input. The raw request body backfills the system prompt when the SDK bridges an endpoint
(e.g. /v1/messages on a chat-completions provider) without filling optional_params."""
if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES:
return None
optional_params: Final = model_call_details.get("optional_params")
@ -962,14 +995,205 @@ def _failure_usage_to_lift(
else None
)
system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions")
estimated_usage: Final = _estimate_dispatched_failure_usage(
return _estimate_dispatched_failure_usage(
model=str(model_call_details.get("model") or ""),
request_input=model_call_details.get("messages"),
system_input=system_input,
)
if estimated_usage is None:
@dataclass(frozen=True)
class BilledFailedAttempt:
"""An earlier retry attempt the provider also billed, written as its own spend row."""
attempted_retries: int
deployment_id: str | None
usage: Usage
response_cost: float
_NO_RETRY_METADATA: Final[Mapping[str, object]] = MappingProxyType({})
def _retry_metadata_bucket(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]:
"""The request metadata bucket the router writes retry state into, ``litellm_metadata`` when the
route carries one (/messages, /responses) and ``metadata`` otherwise, same order as the deployment id."""
if litellm_params is None:
return _NO_RETRY_METADATA
for metadata_key in ("litellm_metadata", "metadata"):
metadata = litellm_params.get(metadata_key)
if isinstance(metadata, dict):
return cast("Mapping[str, object]", metadata) # cast-ok: untyped router dict, keys read with .get
return _NO_RETRY_METADATA
def _attempt_number(record: Mapping[str, object], index: int) -> int:
"""The attempt a retry record describes: its own ``attempted_retries`` count, else its position."""
recorded: Final = record.get("attempted_retries")
return recorded if isinstance(recorded, int) else index
def _retry_attempt_records(litellm_params: Mapping[str, object] | None) -> tuple[Mapping[str, object], ...]:
"""The router's ``RetryAttemptRecord`` breadcrumbs, one per failed attempt, read from the same bucket."""
if litellm_params is None:
return ()
for metadata_key in ("litellm_metadata", "metadata"):
metadata = litellm_params.get(metadata_key)
if not isinstance(metadata, dict):
continue
records = cast("Mapping[str, object]", metadata).get("previous_models") # cast-ok: untyped router dict
if isinstance(records, (list, tuple)):
return tuple(
cast("Mapping[str, object]", record) # cast-ok: RetryAttemptRecord TypedDict, read with .get
for record in cast("Sequence[object]", records) # cast-ok: untyped router tuple
if isinstance(record, dict)
)
return ()
def _record_deployment_id(record: Mapping[str, object]) -> str | None:
deployment_id: Final = record.get("deployment_id")
return deployment_id if isinstance(deployment_id, str) else None
def _attempt_identity(record: Mapping[str, object]) -> tuple[str | None, int | None]:
"""What tells one provider call from another across the request: the deployment that served it
and the retry count within its model group. Two records with the same identity describe the
same call (the fallback handler re-logs the attempt that exhausted the previous group)."""
attempted_retries: Final = record.get("attempted_retries")
return _record_deployment_id(record), attempted_retries if isinstance(attempted_retries, int) else None
def _timed_out_earlier_records(
litellm_params: Mapping[str, object] | None, final_identity: tuple[str | None, int | None]
) -> tuple[Mapping[str, object], ...]:
"""The ``Timeout`` breadcrumbs for calls other than the final attempt, one per provider call."""
by_identity: Final = {
_attempt_identity(record): record
for record in _retry_attempt_records(litellm_params)
if record.get("exception_type") == litellm.Timeout.__name__
}
return tuple(record for identity, record in by_identity.items() if identity != final_identity)
def _record_pricing(
model_call_details: Mapping[str, object], record: Mapping[str, object], base_model: str | None
) -> tuple[Mapping[str, object], str | None]:
"""The ``(model_call_details, base_model)`` to price one earlier attempt with: the record's own
deployment model and ``base_model`` when it served a different model than the final attempt
(the provider is then inferred from that model), else the final attempt's."""
record_model: Final = record.get("model")
if not isinstance(record_model, str) or record_model == model_call_details.get("model"):
return model_call_details, base_model
record_base_model: Final = record.get("base_model")
return (
{**model_call_details, "model": record_model, "custom_llm_provider": None},
record_base_model if isinstance(record_base_model, str) else None,
)
def _billed_earlier_attempt(
model_call_details: Mapping[str, object],
record: Mapping[str, object],
index: int,
estimated_usage: Usage,
base_model: str | None,
) -> BilledFailedAttempt:
pricing_details, pricing_base_model = _record_pricing(model_call_details, record, base_model)
return BilledFailedAttempt(
attempted_retries=_attempt_number(record, index),
deployment_id=_record_deployment_id(record),
usage=estimated_usage,
response_cost=_estimated_input_cost(
pricing_details,
estimated_usage.prompt_tokens,
deployment_id=_record_deployment_id(record),
base_model=pricing_base_model,
),
)
def _billed_earlier_attempts(
model_call_details: Mapping[str, object],
litellm_params: Mapping[str, object] | None,
estimated_usage: Usage,
base_model: str | None,
deployment_id: str | None,
) -> tuple[BilledFailedAttempt, ...]:
"""Every earlier retry attempt the provider also billed. The router records one
``RetryAttemptRecord`` per failed attempt in ``metadata.previous_models``, the final attempt
included, which the failure row itself bills. The final attempt is the record with the
request's own deployment and retry count (``attempted_retries`` restarts at 0 after a fallback
hop, so the count alone cannot tell the groups apart). An earlier attempt that ended in a
``Timeout`` had the same prompt accepted by its deployment, so it is billed like the final
attempt, at that deployment's input rate (else the cost map's rate for the record's own
``base_model`` or model, which a fallback hop can make a different model than the final one).
Only the exception class survives on the record, so an earlier connect timeout is billed too;
any other failure class is not."""
final_attempt: Final = _retry_metadata_bucket(litellm_params).get("attempted_retries")
final_identity: Final = (deployment_id, final_attempt if isinstance(final_attempt, int) else None)
return tuple(
_billed_earlier_attempt(model_call_details, record, index, estimated_usage, base_model)
for index, record in enumerate(_timed_out_earlier_records(litellm_params, final_identity))
)
def _deployment_priced_input_cost(
model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo
) -> float:
"""Input price for the estimated prompt at the deployment's registered rates, read through
``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success
path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``)
is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0."""
usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens)
try:
prompt_cost, _ = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""),
model_info=deployment_pricing,
)
except Exception as exc: # noqa: BLE001 # a malformed registered entry must not drop the failure row
verbose_proxy_logger.debug("failure spend estimate: could not price deployment of %s: %s", model, exc)
return 0.0
return prompt_cost
def _priced_input_cost(pricing_model: str, custom_llm_provider: str | None, prompt_tokens: int) -> float | None:
try:
estimated_input_cost, _ = litellm.cost_per_token(
model=pricing_model,
custom_llm_provider=custom_llm_provider,
prompt_tokens=prompt_tokens,
)
except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written
verbose_proxy_logger.debug("failure spend estimate: could not price %s: %s", pricing_model, exc)
return None
return estimated_usage, 0.0
return estimated_input_cost
def _estimated_input_cost(
model_call_details: Mapping[str, object],
prompt_tokens: int,
deployment_id: str | None,
base_model: str | None,
) -> float:
"""Input price for the estimated prompt: the deployment's registered rates when it has any (the id
is ``Logging.get_router_model_id()``, never the flattened ``model_call_details`` copy a request body
key can overwrite), else the cost map's rate for ``base_model``, else the deployment model, the same
candidate order ``completion_cost`` walks (an Azure deployment name is not in the cost map, its
``base_model`` is; a ``base_model`` the cost map does not know falls through to the deployment
model). A model no candidate can price keeps the row at $0 rather than dropping the failure row."""
model: Final = str(model_call_details.get("model") or "")
custom_llm_provider: Final = str(model_call_details.get("custom_llm_provider") or "") or None
deployment_pricing: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model)
if deployment_pricing is not None:
return _deployment_priced_input_cost(model, custom_llm_provider, prompt_tokens, deployment_pricing)
pricing_models_to_try: Final = pricing_models(base_model, model)
candidate_costs: Final = (
_priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens) for pricing_model in pricing_models_to_try
)
return next((cost for cost in candidate_costs if cost is not None), 0.0)
_EMPTY_LIFT: Final = MappingProxyType({})
@ -1085,7 +1309,38 @@ def _call_type_for_route(route: str | None) -> str | None:
_PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException)
def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]:
_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16
def _exception_chain(exception: BaseException) -> tuple[BaseException, ...]:
chain: Final[list[BaseException]] = [exception] # mutable-ok: built once, returned frozen
for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH):
following = chain[-1].__cause__ or chain[-1].__context__
if following is None or following in chain:
break
chain.append(following)
return tuple(chain)
def _timed_out_reading_from_provider(exception: Exception) -> bool:
"""A read timeout means the request was sent and the provider was working on it; a connect
timeout (``httpx.ConnectTimeout``) never reached the provider, so it is not billed."""
return isinstance(exception, litellm.Timeout) and any(
isinstance(cause, httpx.ReadTimeout) for cause in _exception_chain(exception)
)
def _client_disconnected(exception: Exception) -> bool:
"""The 499 the proxy itself raises when the caller hung up while the provider was still working on the
request. A provider's own 499 arrives as a litellm exception and is a plain upstream failure."""
return isinstance(exception, HTTPException) and exception.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
def _provider_billed_the_failure(exception: Exception) -> bool:
return _timed_out_reading_from_provider(exception) or _client_disconnected(exception)
def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: bool = False) -> Mapping[str, object]:
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
request_data (it is not serialisable), so the caller merges these fields
onto request_data first: the request start and first-handoff instants for
@ -1097,10 +1352,28 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str,
return _EMPTY_LIFT
_model_call_details: Final = getattr(_logging_obj, "model_call_details", {})
_first_handoff: Final = _model_call_details.get("first_api_call_start_time")
_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)
_base_model: Final = _get_base_model_from_metadata(model_call_details=_model_call_details)
_deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None
_success_already_logged: Final = isinstance(_logging_obj, Logging) and _logging_obj.async_success_log_enqueued
_billed: Final = provider_billed and not _success_already_logged
_usage_to_lift: Final = _failure_usage_to_lift(
model_call_details=_model_call_details,
request_body=request_data,
dispatched=_first_handoff is not None,
provider_billed=_billed,
deployment_id=_deployment_id,
base_model=_base_model,
)
_prompt_estimate: Final = (
_estimated_prompt_usage(_model_call_details, request_data)
if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
else None
)
_earlier_attempts: Final = (
_billed_earlier_attempts(_model_call_details, _litellm_params, _prompt_estimate, _base_model, _deployment_id)
if _prompt_estimate is not None
else ()
)
_entries: Final = (
("start_time", _model_call_details.get("start_time")),
@ -1109,6 +1382,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str,
("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]),
("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)),
("standard_logging_object", _model_call_details.get("standard_logging_object")),
("billed_failed_attempts", _earlier_attempts or None),
)
return MappingProxyType({key: value for key, value in _entries if value is not None})
@ -3252,7 +3526,11 @@ class ProxyLogging:
original_exception=original_exception,
)
request_data.update(await offload_token_count(_failure_fields_to_lift)(request_data))
request_data.update(
await offload_token_count(_failure_fields_to_lift)(
request_data, provider_billed=_provider_billed_the_failure(original_exception)
)
)
# Remove before callbacks iterate — not serialisable
request_data.pop("litellm_logging_obj", None)
@ -7790,7 +8068,11 @@ MAX_SPEND_LOG_ISOLATION_FAILURES_PER_BATCH: Final = 256
def _is_transient_spend_log_write_error(e: Exception) -> bool:
return PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e)
return (
PrismaDBExceptionHandler.is_database_transport_error(e)
or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e)
or PrismaDBExceptionHandler.is_deadlock_error(e)
)
async def _create_spend_logs_with_poison_isolation(
@ -7824,7 +8106,7 @@ async def _create_spend_logs_with_poison_isolation(
await repo.table.create_many(data=rows, skip_duplicates=True)
return failure_budget
except Exception as e:
if not PrismaDBExceptionHandler.is_prisma_data_error(e):
if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)):
raise
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
raise

39
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -14,7 +14,7 @@ from litellm.responses.litellm_completion_transformation.custom_tools import (
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
PART_UNION_TYPES,
@ -1259,6 +1259,43 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
return reasoning_aligned
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
"""
The response.incomplete event to bill when the client went away mid-stream.
Built from the chat completion chunks relayed so far, so the usage is what
stream_chunk_builder counts for them.
"""
if self.completed_response is not None or not self.collected_chat_completion_chunks:
return None
built: Final = stream_chunk_builder(
chunks=self.collected_chat_completion_chunks,
messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
input=self.request_input,
responses_api_request=dict(self.responses_api_request),
),
logging_obj=self.litellm_logging_obj,
)
if not isinstance(built, ModelResponse):
return None
responses_api_response: Final = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
request_input=self.request_input,
chat_completion_response=built,
responses_api_request=self.responses_api_request,
)
)
partial: Final = self._with_encoded_response_id(
responses_api_response.model_copy(
update={
"id": self._cached_response_id or responses_api_response.id,
"status": "incomplete",
"output": list(self._output_with_streamed_item_ids(responses_api_response)),
}
)
)
stamp_responses_usage_cost(partial, self.litellm_logging_obj)
return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
def _emit_terminal_response_event(
self, litellm_model_response: ModelResponse
) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:

16
litellm/responses/main.py
View file

@ -547,6 +547,22 @@ def _will_bridge_to_chat_completions(
)
def provider_stores_the_conversation(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
"""Whether the provider, not a LiteLLM spend log, holds this turn for a `previous_response_id` follow-up.
A bridged turn is replayed from its spend log, so the bridge needs that row in the DB, and a bridged
first MCP round runs as an internal call that writes no spend log at all.
"""
api_base: Final = _api_base_kwarg(kwargs)
return not _will_bridge_to_chat_completions(
model,
custom_llm_provider or litellm.get_llm_provider(model=model, api_base=api_base)[1],
bool(kwargs.get("use_chat_completions_api")),
kwargs.get("model_info"),
api_base,
)
@contextmanager
def _prompt_management_sees_a_provisional_message_list(
kwargs: dict[str, object], # mutable-ok: the signal is read and popped out of the caller's own kwargs

72
litellm/responses/streaming_iterator.py
View file

@ -41,10 +41,12 @@ from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.openai import (
PART_UNION_TYPES,
ResponseAPIUsage,
ResponseIncompleteEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
)
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
from litellm.types.utils import CallTypes
from litellm.utils import async_post_call_success_deployment_hook
@ -289,6 +291,7 @@ class BaseResponsesAPIStreamingIterator:
self._yielded_first_chunk = False
self._generated_content = ""
self._generated_tool_arguments = ""
self._created_response: ResponsesAPIResponse | None = None
self._completed_response_cached = False
self._completed_response_logged = False
self._completed_response_cache_hit: bool | None = None
@ -450,6 +453,10 @@ class BaseResponsesAPIStreamingIterator:
# Store the completed response (also for incomplete/failed so logging still fires)
_chunk_type: Final = getattr(openai_responses_api_chunk, "type", None)
openai_types = _get_openai_response_types()
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED:
_created: Final[object] = getattr(openai_responses_api_chunk, "response", None)
if isinstance(_created, ResponsesAPIResponse):
self._created_response = _created
if openai_responses_api_chunk and _chunk_type in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
@ -481,7 +488,7 @@ class BaseResponsesAPIStreamingIterator:
else openai_responses_api_chunk.model_copy(update={"response": _billed_response})
)
self.completed_response = _terminal_chunk
_stamp_responses_usage_cost(_billed_response, self.logging_obj)
stamp_responses_usage_cost(_billed_response, self.logging_obj)
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
self._handle_logging_failed_response()
@ -874,6 +881,34 @@ class BaseResponsesAPIStreamingIterator:
except Exception:
pass
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
"""
The response.incomplete event to bill when the client went away mid-stream,
the same shape the success handler assembles for a natural end of stream.
Its response is the response.created payload (it carries the served
service_tier) plus the text and tool arguments relayed so far, with counted
usage since the provider never sent any. None when nothing was relayed yet
or the stream already reached its terminal event and billed itself.
"""
created: Final = self._created_response
generated: Final = self._generated_content + self._generated_tool_arguments
if self.completed_response is not None or created is None or not generated:
return None
usage: Final = _estimate_usage_safely(
self.model or "", self.request_data.get("input"), self.request_data, generated
)
if usage is None:
return None
partial: Final = created.model_copy(
update={
"status": "incomplete",
"usage": usage,
"output": _partial_output_items(self._generated_content),
}
)
stamp_responses_usage_cost(partial, self.logging_obj)
return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
async def call_post_streaming_hooks_for_testing(
iterator: object, chunk: ResponsesAPIStreamingResponse
@ -1347,6 +1382,37 @@ def _add_text_like_part_events(
)
@runtime_checkable
class _BillsPartialStreamOnDisconnect(Protocol):
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: ...
def partial_responses_incomplete_event_on_disconnect(stream: object) -> ResponseIncompleteEvent | None:
"""Ask the stream behind a proxy or router wrapper for the partial response to bill, if it can build one."""
if not isinstance(stream, _BillsPartialStreamOnDisconnect):
return None
try:
partial: Final = stream.partial_response_on_disconnect()
except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown
verbose_logger.debug("Failed to build partial Responses API response for disconnect billing: %s", e)
return None
return partial if isinstance(partial, ResponseIncompleteEvent) else None
def _partial_output_items(generated_text: str) -> Sequence[GenericResponseOutputItem]:
if not generated_text:
return []
return [
GenericResponseOutputItem(
type="message",
id=f"msg_{uuid.uuid4()}",
status="incomplete",
role="assistant",
content=[OutputText(type="output_text", text=generated_text, annotations=[])],
)
]
def _billed_terminal_response(
response_obj: object, estimate: Callable[[], ResponseAPIUsage | None] | None
) -> ResponsesAPIResponse | None:
@ -1453,7 +1519,7 @@ def _estimate_usage_safely(
return None
def _stamp_responses_usage_cost(
def stamp_responses_usage_cost(
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
) -> None:
if response_obj is None or logging_obj is None:
@ -1479,7 +1545,7 @@ def build_synthetic_response_events(
chunk_size: int,
) -> list[ResponsesAPIStreamingResponse]:
openai_types: Final = _get_openai_response_types()
_stamp_responses_usage_cost(transformed, logging_obj)
stamp_responses_usage_cost(transformed, logging_obj)
events: Final[list[ResponsesAPIStreamingResponse]] = [
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),

67
litellm/router.py
View file

@ -98,6 +98,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.sensitive_data_masker import (
@ -199,6 +200,7 @@ from litellm.router_utils.cooldown_handlers import (
is_background_response_cost_poll_not_found,
is_caller_timeout_408,
)
from litellm.router_utils.deployment_pricing import deployment_pricing_models
from litellm.router_utils.fallback_event_handlers import (
MID_STREAM_FALLBACK_CONTROLS_KEY,
AttemptedFallbackTargets,
@ -368,6 +370,7 @@ if TYPE_CHECKING:
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseIncompleteEvent,
ResponseInputParam,
ResponsesAPIResponse,
)
@ -647,6 +650,11 @@ class FallbackAwareAnthropicMessagesStream:
self._source_iterator = fallback_response
self.fallback_headers_adopted = True
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
return partial_responses_incomplete_event_on_disconnect(self._source_iterator)
def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream":
return self
@ -3260,6 +3268,7 @@ class Router:
from datetime import datetime
self._async_generator = async_generator
self._billing_source: object = source_iterator
# Mirror every attribute BaseResponsesAPIStreamingIterator.__init__
# would have set. The wrapper bypasses super().__init__ (it has no
# httpx.Response of its own and no provider config to drive), so
@ -3308,10 +3317,29 @@ class Router:
def adopt_fallback_headers(self, fallback_response: object) -> tuple[dict[str, object], dict[str, object]]:
prepared: Final = Router._prepare_fallback_hidden_params(fallback_response)
self._billing_source = fallback_response
self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} # mutable-ok: stream metadata
self.fallback_headers_adopted = True
return prepared
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
"""
The event to bill on a client disconnect. After a mid-stream hop the primary
stream's relayed tokens are folded into the fallback's event, the way
_combine_responses_fallback_usage does for a natural end of stream.
"""
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
current: Final = partial_responses_incomplete_event_on_disconnect(self._billing_source)
if self._billing_source is source_iterator:
return current
primary: Final = partial_responses_incomplete_event_on_disconnect(source_iterator)
if current is None or primary is None:
return current or primary
if primary.response.usage is not None:
Router._combine_responses_fallback_usage(current, primary.response.usage)
return current
def __aiter__(self):
return self
@ -4051,6 +4079,7 @@ class Router:
"""Helper to get stream timeout from kwargs or deployment params"""
return (
kwargs.get("stream_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier")
or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment
or self.stream_timeout # timeout set on router
or self.request_timeout # litellm_settings.request_timeout (per-attempt)
@ -4062,6 +4091,7 @@ class Router:
timeout: Final = (
kwargs.get("timeout", None) # the params dynamically set by user
or kwargs.get("request_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier")
or data.get("timeout", None) # timeout set on litellm_params for this deployment
or data.get("request_timeout", None) # timeout set on litellm_params for this deployment
or self.request_timeout # litellm_settings.request_timeout (per-attempt)
@ -8313,17 +8343,24 @@ class Router:
When a retry or fallback happens, record which model group, deployment and attempt just failed and why,
and count it toward the request-wide num_retries_per_request cap
"""
from litellm.litellm_core_utils.get_litellm_params import _get_base_model_from_litellm_call_metadata
from litellm.types.router import RetryAttemptRecord
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var]
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
model_group: Final = kwargs.get("model")
model_info: Final = request_metadata.get("model_info")
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
base_model: Final = _get_base_model_from_litellm_call_metadata(
metadata=request_metadata
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
deployment_model: Final = request_metadata.get("deployment")
attempted_retries: Final = request_metadata.get("attempted_retries")
attempt_record: Final[RetryAttemptRecord] = {
"model_group": model_group if isinstance(model_group, str) else None,
"deployment_id": deployment_id if isinstance(deployment_id, str) else None,
"model": deployment_model if isinstance(deployment_model, str) else None,
"base_model": base_model if isinstance(base_model, str) else None,
"exception_type": type(e).__name__,
"exception_string": str(e),
"attempted_retries": attempted_retries if type(attempted_retries) is int else None,
@ -8341,6 +8378,18 @@ class Router:
kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap
return kwargs
def _deployment_params_base_model(self, deployment_id: str | None) -> str | None:
"""The ``base_model`` a deployment declares under ``litellm_params`` (the Azure named-deployment
shape), the same key the success path's ``get_base_model_from_metadata`` reads."""
deployment: Final = self.get_deployment(model_id=deployment_id) if deployment_id is not None else None
if deployment is None:
return None
extra_params: Final = cast( # cast-ok: pydantic extras are untyped, base_model is isinstance-checked below
"Mapping[str, object]", deployment.litellm_params.model_extra or {}
)
base_model: Final = extra_params.get("base_model")
return base_model if isinstance(base_model, str) else None
def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int:
"""
Update deployment rpm for that minute
@ -10335,19 +10384,9 @@ class Router:
)
for deployment in deployments
)
params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments)
# base_model resolution mirrors get_router_model_info: unset or blank means the
# deployment's own model name is the cost-map key.
cost_map_keys: Final = tuple(
dict.fromkeys( # deduplicates while preserving config order
key
for key in (
model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
for model_info, litellm_params in zip(model_infos, params)
)
if isinstance(key, str) and key
)
)
raw_deployments: Final[tuple[object, ...]] = deployments
pricing_keys: Final = (deployment_pricing_models(deployment) for deployment in raw_deployments)
cost_map_keys: Final = tuple(dict.fromkeys(keys[0] for keys in pricing_keys if keys))
return DeploymentModelListingInfo(
cost_map_keys=cost_map_keys,
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),

33
litellm/router_strategy/lowest_cost.py
View file

@ -1,5 +1,6 @@
#### What this does ####
# picks based on response time (for streaming, this is time to first token)
from collections.abc import Mapping
from datetime import datetime
from typing import Final
@ -9,6 +10,30 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
"""Input and output per-token prices from the first cost-map row among the deployment's
pricing models that carries one; (None, None) when no row does, so the caller falls back to
its unmapped default."""
rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment))
row: Final = next((row for row in rows if _declares_token_price(row)), None)
if row is None:
return None, None
return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token"))
def _cost_map() -> Mapping[str, object]:
return litellm.model_cost
def _declares_token_price(row: Mapping[str, object]) -> bool:
return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None
def _as_rate(value: object) -> float | None:
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
class LowestCostLoggingHandler(CustomLogger):
@ -239,8 +264,8 @@ class LowestCostLoggingHandler(CustomLogger):
or _deployment.get("model_info", {}).get("rpm", None)
or float("inf")
)
item_litellm_model_name = _deployment.get("litellm_params", {}).get("model")
item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {})
deployment_view: object = _deployment
map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view)
# check if user provided input_cost_per_token and output_cost_per_token in litellm_params
item_input_cost = None
@ -252,10 +277,10 @@ class LowestCostLoggingHandler(CustomLogger):
item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token")
if item_input_cost is None:
item_input_cost = item_litellm_model_cost_map.get("input_cost_per_token", 5.0)
item_input_cost = map_input_cost if map_input_cost is not None else 5.0
if item_output_cost is None:
item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0)
item_output_cost = map_output_cost if map_output_cost is not None else 5.0
# if litellm["model"] is not in model_cost map -> use item_cost = $10

37
litellm/router_utils/deployment_pricing.py Normal file
View file

@ -0,0 +1,37 @@
from collections.abc import Iterator, Mapping
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter, ValidationError
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_STR_KEYED: Final = TypeAdapter(dict[str, object])
def pricing_models(base_model: object, model: object) -> tuple[str, ...]:
"""Cost-map keys to price a deployment under, ``model_info.base_model`` first, then the
deployment's own model, blanks and duplicates dropped. Every spend and routing surface that
prices a deployment walks this order, so a ``base_model`` the cost map does not know falls
through to the deployment model instead of billing or ranking at the unmapped default."""
return tuple(dict.fromkeys(_named_models(base_model, model)))
def deployment_pricing_models(deployment: object) -> tuple[str, ...]:
"""``pricing_models`` read off a raw deployment dict (``model_info.base_model``, then
``litellm_params.base_model``, then ``litellm_params.model``); malformed blocks count as unset."""
view: Final = str_keyed(deployment)
params: Final = str_keyed(view.get("litellm_params"))
info: Final = str_keyed(view.get("model_info"))
return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model"))
def _named_models(*candidates: object) -> Iterator[str]:
return (candidate for candidate in candidates if isinstance(candidate, str) and candidate)
def str_keyed(value: object) -> Mapping[str, object]:
"""``value`` as a read-only str-keyed mapping, empty when it is not a dict."""
try:
return MappingProxyType(_STR_KEYED.validate_python(value))
except ValidationError:
return _EMPTY

5
litellm/systemone/main.py
View file

@ -8,6 +8,7 @@ from litellm.decisions.call import (
SystemOneQuestions,
SystemOneRequestFields,
asend,
log_caller_request,
prepare_call,
reject_other_format,
send,
@ -29,10 +30,12 @@ def _prepare_systemone_call(
extra_headers: Mapping[str, str] | None,
kwargs: Mapping[str, object],
) -> DecisionsCall:
fields: Final = SystemOneRequestFields(state=state, questions=questions)
log_caller_request(kwargs, fields)
reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions")
return prepare_call(
model=model,
fields=SystemOneRequestFields(state=state, questions=questions),
fields=fields,
safety_identifier=safety_identifier,
api_key=api_key,
api_base=api_base,

2
litellm/types/integrations/anthropic_cache_control_hook.py
View file

@ -22,7 +22,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
class CacheControlToolConfigInjectionPoint(TypedDict):
"""Type for tool_config-level injection points (Bedrock)."""
"""Type for tool_config-level injection points (Anthropic and Bedrock)."""
location: Literal["tool_config"]
control: ChatCompletionCachedContent | None

2
litellm/types/litellm_params.py
View file

@ -72,6 +72,8 @@ class ProviderConnection:
request_timeout: float | None = None
force_timeout: float | None = None
stream_timeout: float | str | None = None
timeout_by_service_tier: Mapping[str, float] | None = None
stream_timeout_by_service_tier: Mapping[str, float] | None = None
max_retries: int | None = None
tenant_id: str | None = None
client_id: str | None = None

7
litellm/types/router.py
View file

@ -389,6 +389,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
otpm: int | None = None
timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/
stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/
# per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key
timeout_by_service_tier: Mapping[str, float] | None = None
stream_timeout_by_service_tier: Mapping[str, float] | None = None
max_retries: int | None = None
drop_params: bool | str | None = None
organization: str | None = None # for openai orgs
@ -573,6 +576,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
api_version: str | None
timeout: float | str | httpx.Timeout | None
stream_timeout: float | str | None
timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
max_retries: int | None
organization: list | str | None # for openai orgs
provider_affinity_header: ReadOnly[str | None]
@ -996,6 +1001,8 @@ class RouterModelGroupAliasItem(TypedDict):
class RetryAttemptRecord(TypedDict):
model_group: ReadOnly[str | None]
deployment_id: ReadOnly[str | None]
model: ReadOnly[str | None]
base_model: ReadOnly[str | None]
exception_type: ReadOnly[str]
exception_string: ReadOnly[str]
attempted_retries: ReadOnly[int | None]

7
litellm/types/utils.py
View file

@ -223,6 +223,12 @@ class SearchContextCostPerQuery(TypedDict, total=False):
search_context_size_high: float
class SearchFreeRequests(TypedDict):
count: ReadOnly[int]
period: ReadOnly[Literal["day", "month"]]
shared_group: ReadOnly[str]
class AgenticLoopParams(TypedDict, total=False):
"""
Parameters passed to agentic loop hooks (e.g., WebSearch interception).
@ -399,6 +405,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
web_search_billing_unit: (
Literal["per_query", "per_prompt"] | None
) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x)
search_free_requests: ReadOnly[SearchFreeRequests | None]
google_maps_grounding_cost_per_query: ReadOnly[float | None]
citation_cost_per_token: float | None # Cost per citation token for Perplexity
tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope

15
litellm/utils.py
View file

@ -954,10 +954,6 @@ def _ocr_document_summary(document: object) -> str:
# Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc.
_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset(
(CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value)
)
def function_setup(
original_function: str,
@ -1207,9 +1203,6 @@ def function_setup(
if isinstance(search_query, list)
else search_query
)
elif call_type in _DECISION_CALL_TYPES:
decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or ""
messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str)
elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value):
messages = args[1] if len(args) > 1 else kwargs.get("prompt")
elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value):
@ -1372,6 +1365,7 @@ def _schedule_async_success_logging(
"""
def _enqueue_async_logging() -> None:
logging_obj.mark_async_success_log_enqueued()
asyncio.create_task(
_client_async_logging_helper(
logging_obj=logging_obj,
@ -5165,6 +5159,9 @@ def get_optional_params(
EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"})
LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",))
def add_provider_specific_params_to_optional_params(
optional_params: dict,
passed_params: dict,
@ -5180,8 +5177,9 @@ def add_provider_specific_params_to_optional_params(
# for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46
if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False:
extra_body: Final = dict(passed_params.pop("extra_body", None) or {})
withheld_params: Final = frozenset[str]() if custom_llm_provider == "litellm_proxy" else LITELLM_ONLY_PARAMS
for k in passed_params:
if k not in openai_params and passed_params[k] is not None:
if k not in openai_params and k not in withheld_params and passed_params[k] is not None:
extra_body[k] = passed_params[k]
if not isinstance(optional_params.get("extra_body"), dict):
optional_params["extra_body"] = {}
@ -6446,6 +6444,7 @@ def _get_model_info_helper(
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
search_free_requests=_model_info.get("search_free_requests", None),
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),

1
tests/code_coverage_tests/router_code_coverage.py
View file

@ -87,6 +87,7 @@ ignored_function_names = [
"_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"_deployment_params_base_model", # Tested through log_retry in test_router_helper_utils.py
]

41
tests/router_unit_tests/test_router_helper_utils.py
View file

@ -637,7 +637,12 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) ->
"model": "gpt-3.5-turbo",
"api_key": "sk-must-not-be-recorded",
"messages": [{"role": "user", "content": "hi"}],
metadata_key: {"model_info": {"id": "deployment-1"}, "attempted_retries": 2, "user_api_key": "sk-proxy"},
metadata_key: {
"model_info": {"id": "deployment-1", "base_model": "azure/gpt-3.5-turbo"},
"deployment": "azure/my-35-deployment",
"attempted_retries": 2,
"user_api_key": "sk-proxy",
},
},
e=rate_limit_error,
)
@ -645,6 +650,8 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) ->
{
"model_group": "gpt-3.5-turbo",
"deployment_id": "deployment-1",
"model": "azure/my-35-deployment",
"base_model": "azure/gpt-3.5-turbo",
"exception_type": "RateLimitError",
"exception_string": "litellm.RateLimitError: slow down",
"attempted_retries": 2,
@ -656,6 +663,38 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) ->
assert router.log_retry(kwargs=planted_kwargs, e=rate_limit_error)[metadata_key]["request_retry_count"] == 1
def test_log_retry_reads_base_model_from_the_deployments_litellm_params() -> None:
"""An Azure named deployment declares ``base_model`` under ``litellm_params``, not ``model_info``, and
that is the model the failure row is priced at, so the retry record has to carry it too"""
router = Router(
model_list=[
{
"model_name": "azure-group",
"litellm_params": {
"model": "azure/my-named-deployment",
"api_key": "sk-fake",
"api_base": "https://example.openai.azure.com",
"base_model": "azure/gpt-4o-mini",
},
"model_info": {"id": "dep-azure-named"},
}
]
)
new_kwargs = router.log_retry(
kwargs={
"model": "azure-group",
"metadata": {
"model_info": {"id": "dep-azure-named"},
"deployment": "azure/my-named-deployment",
"attempted_retries": 0,
},
},
e=litellm.Timeout(message="slow", llm_provider="azure", model="azure/my-named-deployment"),
)
record = new_kwargs["metadata"]["previous_models"][0]
assert (record["model"], record["base_model"]) == ("azure/my-named-deployment", "azure/gpt-4o-mini")
def test_update_usage(model_list):
"""Test if the '_update_usage' function is working correctly"""
router = Router(model_list=model_list)

36
tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py
View file

@ -1210,3 +1210,39 @@ class TestCoerceNumericFormFields:
numeric_fields=self.numeric_fields,
)
assert result == {"n": 3, "temperature": None, "image": buffer}
@pytest.mark.asyncio
async def test_form_data_metadata_with_lone_surrogate_escape_is_rejected_like_a_json_body():
"""``json.loads`` accepts ``"\\ud800"`` and yields a lone surrogate that no
provider or database can utf-8 encode. The JSON body path already answers
400, so the multipart ``metadata`` field must too, instead of letting the
value through to the spend log writers."""
mock_request = MagicMock()
mock_request.form = AsyncMock(
return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["poison-\\ud800-tag"]}'})
)
mock_request.headers = {"content-type": "multipart/form-data"}
mock_request.scope = {}
mock_request.state._cached_headers = None
with pytest.raises(ProxyException) as exc_info:
await _read_request_body(mock_request)
assert exc_info.value.code == "400"
assert "metadata" in exc_info.value.message
@pytest.mark.asyncio
async def test_form_data_metadata_keeps_non_ascii_and_valid_surrogate_pairs():
mock_request = MagicMock()
mock_request.form = AsyncMock(
return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["caf\\u00e9", "\\ud83d\\ude00"]}'})
)
mock_request.headers = {"content-type": "multipart/form-data"}
mock_request.scope = {}
mock_request.state._cached_headers = None
result = await _read_request_body(mock_request)
assert result["metadata"]["tags"] == ["café", "😀"]

51
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -3843,28 +3843,57 @@ async def test_commit_spend_updates_retries_deadlock_on_every_entity_path(monkey
proxy_logging.failure_handler.assert_not_called()
def _responses_router() -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "native",
"litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test"},
"model_info": {"id": "dep-native"},
},
{
"model_name": "bridged",
"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"},
"model_info": {"id": "dep-bridged"},
},
{
"model_name": "forced-bridge",
"litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test", "use_chat_completions_api": True},
"model_info": {"id": "dep-forced-bridge"},
},
]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"call_type, expects_flush",
[("aresponses", True), ("responses", True), ("acompletion", False)],
"call_type, model_id, expects_flush",
[
("aresponses", "dep-bridged", True),
("responses", "dep-bridged", True),
("aresponses", "dep-forced-bridge", True),
("aresponses", None, True),
("aresponses", "dep-native", False),
("responses", "dep-native", False),
("acompletion", "dep-bridged", False),
],
)
async def test_insert_spend_log_asks_for_an_immediate_flush_on_rows_other_workers_read_back(
call_type: str, expects_flush: bool
async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bridge_reads_back(
call_type: str, model_id: str | None, expects_flush: bool
):
"""
A `previous_response_id` chained straight off the previous turn reads the DB, so a
Responses row cannot sit in this worker's queue until the monitor's next poll.
A `previous_response_id` chained straight off a bridged turn reads that turn's row from the
DB, so it cannot sit in this worker's queue until the monitor's next poll. A native Responses
provider holds the conversation itself, so its row waits for the batch write like a chat row.
"""
db_writer = DBSpendUpdateWriter()
prisma = _tool_usage_prisma()
prisma.spend_log_flush_requested = asyncio.Event()
payload = {"request_id": "req-1", "call_type": call_type, "model_id": model_id}
await db_writer._insert_spend_log_to_db(
payload={"request_id": "req-1", "call_type": call_type},
prisma_client=prisma,
)
await db_writer._insert_spend_log_to_db(payload=payload, prisma_client=prisma, llm_router=_responses_router())
assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}]
assert prisma.spend_log_transactions == [payload]
assert prisma.spend_log_flush_requested.is_set() is expects_flush

153
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -9,6 +9,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth
@ -28,7 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
@pytest.mark.asyncio
@ -94,6 +96,65 @@ async def test_async_post_call_failure_hook():
assert metadata["original_key"] == "original_value"
@pytest.mark.asyncio
async def test_each_earlier_billed_attempt_gets_its_own_spend_row():
"""A timed-out request that the router retried writes one row per billed attempt: the final
attempt under the request id, every earlier Timeout attempt under ``<id>-attempt-<n>`` with its
own cost, attempt number and deployment."""
from litellm.proxy.utils import BilledFailedAttempt
logger = _ProxyDBLogger()
user_api_key_dict = UserAPIKeyAuth(api_key="test_api_key", user_id="u1", team_id="t1")
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
request_data = {
"model": "gpt-4.1",
"litellm_call_id": "call-1",
"standard_logging_object": {"id": "call-1", "model_id": "dep-final"},
"metadata": {
"attempted_retries": 2,
"model_info": {"id": "dep-final", "input_cost_per_token": 0.5},
"previous_models": ({"exception_type": "Timeout"}, {"exception_type": "Timeout"}),
},
"combined_usage_object": usage,
"response_cost": 0.5,
"billed_failed_attempts": (
BilledFailedAttempt(attempted_retries=0, deployment_id="dep-a", usage=usage, response_cost=0.25),
BilledFailedAttempt(attempted_retries=0, deployment_id="dep-final", usage=usage, response_cost=0.5),
),
}
with patch(
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
new_callable=AsyncMock,
) as mock_update_database:
await logger.async_post_call_failure_hook(
request_data=request_data,
original_exception=litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai"),
user_api_key_dict=user_api_key_dict,
)
rows = [call.kwargs for call in mock_update_database.call_args_list]
assert [row["response_cost"] for row in rows] == [0.5, 0.25, 0.5]
final_row, attempt_0, attempt_1 = (row["kwargs"] for row in rows)
assert final_row["litellm_call_id"] == "call-1"
assert final_row["litellm_params"]["metadata"]["attempted_retries"] == 2
assert attempt_0["litellm_call_id"] == "call-1-attempt-0"
assert attempt_0["standard_logging_object"]["id"] == "call-1-attempt-0"
assert attempt_0["litellm_params"]["metadata"]["attempted_retries"] == 0
assert attempt_0["litellm_params"]["metadata"]["model_info"]["id"] == "dep-a"
assert "previous_models" not in attempt_0["litellm_params"]["metadata"]
assert "billed_failed_attempts" not in attempt_0
assert attempt_0["combined_usage_object"] is usage
assert attempt_1["litellm_call_id"] == "call-1-attempt-1"
assert attempt_1["litellm_params"]["metadata"]["attempted_retries"] == 0
assert attempt_1["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final"
assert final_row["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final"
for row in rows:
assert row["token"] == "test_api_key"
assert row["user_id"] == "u1"
assert row["team_id"] == "t1"
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata():
"""
@ -2785,3 +2846,93 @@ async def test_batch_accounting_does_not_retry_after_settling_a_budget_reservati
)
assert reservation["finalized"] is True
assert outcome.retryable_failure is False
class _SharedCounter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
return self.totals[key]
def _grounded_gemini_kwargs(response_cost: float) -> dict:
return {
"model": "vertex_ai/gemini-test",
"call_type": "acompletion",
"response_cost": response_cost,
"litellm_params": {"metadata": {"user_api_key": "sk-test", "user_api_key_user_id": "user-1"}},
"standard_logging_object": {
"response_cost": response_cost,
"request_tags": None,
"cache_hit": None,
"custom_llm_provider": "vertex_ai",
"cost_breakdown": {
"input_cost": 0.0,
"output_cost": 0.0,
"tool_usage_cost": 0.5,
"total_cost": response_cost,
},
"model_map_information": {
"model_map_key": "vertex_ai/gemini-test",
"model_map_value": {
"litellm_provider": "vertex_ai",
"mode": "chat",
"web_search_billing_unit": "per_query",
"search_context_cost_per_query": {"search_context_size_medium": 0.1},
"search_free_requests": {"count": 100, "period": "month", "shared_group": "test"},
},
},
},
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.parametrize("opted_in", [True, False])
async def test_track_cost_callback_frees_grounding_under_the_allowance_only_when_opted_in(monkeypatch, opted_in):
monkeypatch.setattr(litellm, "apply_search_free_tier", opted_in, raising=False)
counter = _SharedCounter()
kwargs = _grounded_gemini_kwargs(response_cost=1.0)
response = ModelResponse(
usage=Usage(
prompt_tokens=10,
completion_tokens=5,
total_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=5),
)
)
with (
patch( # test-quality-ok: increment_spend_counters is a proxy_server global the callback reads lazily, no seam
"litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock
),
patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam
"litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock
),
patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam
"litellm.proxy.proxy_server.proxy_logging_obj"
) as mock_proxy_logging,
):
mock_proxy_logging.failed_tracking_alert = AsyncMock()
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True)
mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
mock_proxy_logging.internal_usage_cache.dual_cache.redis_cache = counter
await _ProxyDBLogger()._PROXY_track_cost_callback(
kwargs=kwargs, completion_response=response, start_time=datetime.now(), end_time=datetime.now()
)
await asyncio.sleep(0)
mock_proxy_logging.failed_tracking_alert.assert_not_called()
mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once()
breakdown = kwargs["standard_logging_object"]["cost_breakdown"]
if opted_in:
assert kwargs["response_cost"] == pytest.approx(1.0 - 5 * 0.1)
assert breakdown["original_cost"] == 1.0 and breakdown["discount_amount"] == pytest.approx(5 * 0.1)
assert sum(counter.totals.values()) == 5
else:
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in breakdown
assert counter.totals == {}

26
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -1249,6 +1249,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value(
assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected
def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier():
litellm_params = {
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
}
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params)
== 20.0
)
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params)
== 950.0
)
assert (
resolve_llm_passthrough_timeout(
kwargs={"service_tier": "flex", "stream": True},
litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}},
)
== 30.0
)
def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner():
with pytest.raises(ValidationError):
resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)})

112
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -7637,6 +7637,118 @@ def _apply_stream_usage_tracking(
)
async def _start_partial_responses_stream(self, recorder: _RecordingSuccessLogger, *, relay_text: bool = True):
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
mocked: Final = await litellm.aresponses(
model="gpt-4o-mini",
input="tell me a story",
mock_response="The codename is AZURE-FALCON-42 and the story is long.",
stream=True,
api_key="test-key",
)
logging_obj: Final = mocked.logging_obj
logging_obj.dynamic_async_success_callbacks = [recorder]
stream: Final = ResponsesAPIStreamingIterator(
response=httpx.Response(200),
model="gpt-4o-mini",
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=logging_obj,
request_data={"model": "gpt-4o-mini", "input": "tell me a story"},
custom_llm_provider="openai",
)
created: Final = {
"type": "response.created",
"sequence_number": 0,
"response": {
"id": "resp_partial",
"object": "response",
"created_at": 1700000000.0,
"model": "gpt-4o-mini",
"status": "in_progress",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"service_tier": "priority",
},
}
delta: Final = {
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_partial",
"output_index": 0,
"content_index": 0,
"delta": "The codename is AZURE-FALCON-42",
}
for event in (created, delta) if relay_text else (created,):
stream._process_chunk(json.dumps(event))
return stream
async def _bill_and_wait(self, recorder: _RecordingSuccessLogger, streamed: object, logging_obj: object) -> bool:
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed)
for _ in range(50):
if recorder.success_events:
break
await asyncio.sleep(0.1)
await asyncio.sleep(0.5)
return billed
@pytest.mark.asyncio
async def test_disconnect_bills_partial_native_responses_stream_with_counted_usage(self):
from litellm.types.llms.openai import ResponsesAPIResponse
recorder: Final = _RecordingSuccessLogger()
stream: Final = await self._start_partial_responses_stream(recorder)
billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert isinstance(partial_response, ResponsesAPIResponse)
assert partial_response.status == "incomplete"
assert getattr(partial_response, "service_tier") == "priority"
assert partial_response.usage.output_tokens == litellm.token_counter(
model="gpt-4o-mini", text="The codename is AZURE-FALCON-42"
)
assert partial_response.usage.input_tokens > 0
@pytest.mark.asyncio
async def test_disconnect_bills_partial_default_route_messages_stream_through_the_responses_adapter(self):
from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import (
AnthropicResponsesStreamWrapper,
)
from litellm.router import FallbackAwareAnthropicMessagesStream
from litellm.types.llms.openai import ResponsesAPIResponse
async def _sse_frames() -> AsyncGenerator[bytes, None]:
yield b"event: message_start\n\n"
recorder: Final = _RecordingSuccessLogger()
stream: Final = await self._start_partial_responses_stream(recorder)
wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=stream, model="gpt-4o-mini")
streamed: Final = FallbackAwareAnthropicMessagesStream(_sse_frames(), wrapper.async_anthropic_sse_wrapper())
billed: Final = await self._bill_and_wait(recorder, streamed, stream.logging_obj)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert isinstance(partial_response, ResponsesAPIResponse)
assert partial_response.usage.output_tokens == litellm.token_counter(
model="gpt-4o-mini", text="The codename is AZURE-FALCON-42"
)
@pytest.mark.asyncio
async def test_disconnect_before_any_responses_output_bills_nothing(self):
recorder: Final = _RecordingSuccessLogger()
stream: Final = await self._start_partial_responses_stream(recorder, relay_text=False)
billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj)
assert billed is False
assert recorder.success_events == []
class TestApplyStreamUsageTracking:
def test_default_injects_usage_and_marks_strip_for_chat_completions(self):
data = {"stream": True, "model": "gpt-5.4-nano"}

2
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -7764,6 +7764,7 @@ _PLANTED_STAMPS = {
"attempted_fallbacks": 99,
"original_model_group": "spoofed-group",
"request_retry_count": -100,
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
"_client_output_ceiling": {"api_base": "https://attacker.example"},
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9,
"client_key": "client_value",
@ -7798,6 +7799,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
assert "original_model_group" not in updated["metadata"]
assert "_client_output_ceiling" not in updated["metadata"]
assert "request_retry_count" not in updated["metadata"]
assert "previous_models" not in updated["metadata"]
assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"]
assert updated["metadata"]["client_key"] == "client_value"

22
tests/test_litellm/proxy/test_pricing_field_strip.py
View file

@ -58,9 +58,7 @@ class TestStripClientPricingOverrides:
# The strip set is built from the model so additions are picked up
# automatically — this test guards against the model and the strip
# set drifting apart if someone replaces the auto-derivation later.
assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset(
CustomPricingLiteLLMParams.model_fields.keys()
)
assert _CLIENT_PRICING_CONTROL_FIELDS >= frozenset(CustomPricingLiteLLMParams.model_fields.keys())
# Sanity: the obvious top-level pricing fields are in the set.
for field in (
"input_cost_per_token",
@ -84,6 +82,22 @@ class TestStripClientPricingOverrides:
"messages": [{"role": "user", "content": "hi"}],
}
def test_root_base_model_dropped(self):
# ``base_model`` picks the cost-map entry a spend row is priced with
# (success rows via the standard logging payload, billed failure rows
# via the failure lift), so a caller naming a cheaper model lowers
# their own recorded spend.
data = {
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hi"}],
"base_model": "gpt-4o-mini",
}
_strip_client_pricing_overrides(data)
assert data == {
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hi"}],
}
def test_metadata_model_info_dropped(self):
data = {
"model": "gpt-4",
@ -264,6 +278,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():
"model": "gpt-4",
"messages": [{"role": "user", "content": "hi"}],
"input_cost_per_token": 0.0001,
"base_model": "gpt-4o-mini",
"metadata": {"model_info": {"output_cost_per_token": 0.0002}},
}
@ -279,6 +294,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():
assert updated["input_cost_per_token"] == 0.0001
assert updated["metadata"]["model_info"] == {"output_cost_per_token": 0.0002}
assert updated["base_model"] == "gpt-4o-mini"
@pytest.mark.asyncio

669
tests/test_litellm/proxy/test_proxy_utils.py
View file

@ -2,6 +2,7 @@ import datetime as real_datetime
import smtplib
from typing import Final
import httpx
import pytest
from fastapi import HTTPException
@ -443,6 +444,47 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime:
assert "litellm_logging_obj" not in request_data
class TestPostCallFailureHookDoesNotInventAProviderHandoff:
"""A failure the proxy raised before any provider call (here an auth error) is logged through
_handle_logging_proxy_only_error, which fires pre_call for the callbacks. That must not stamp a
handoff instant onto the request, or every downstream reader (preprocessing latency, the
dispatched check behind the failure-row estimate) sees a provider call that never happened."""
@pytest.mark.asyncio
async def test_proxy_only_failure_lifts_no_first_api_call_start_time(self):
from datetime import datetime
from unittest.mock import AsyncMock, patch
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth
logging_obj = Logging(
model="gpt-4.1",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="acompletion",
start_time=datetime.now(),
litellm_call_id="pre-handoff-auth-failure",
function_id="pre-handoff",
)
request_data: Final[dict[str, object]] = {"litellm_logging_obj": logging_obj, "model": "gpt-4.1", "metadata": {}}
proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
with (
patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()),
patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()),
):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=Exception("Authentication Error, invalid key"),
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"),
error_type=ProxyErrorTypes.auth_error,
)
assert request_data.get("first_api_call_start_time") is None
assert "api_call_start_time" not in logging_obj.model_call_details
class TestPostCallFailureHookLiftsRecoveredPartialSpend:
"""A stream that broke mid-flight still billed the provider for the chunks
already delivered. The streaming handler stashes that recovered usage and
@ -687,11 +729,13 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
"""A non-stream request that failed after dispatch (timeout, provider
error) consumed provider-billed input tokens but recovered no usage.
post_call_failure_hook must estimate the input side onto request_data so
the spend log's failure row records what was sent instead of zero, while
never charging spend for the failure (LIT-5690).
the spend log's failure row records what was sent instead of zero
(LIT-5690). On a timeout that estimate is priced at the model's input
rate since the provider bills the prompt it accepted (LIT-145); other
post-dispatch failures keep zero spend.
"""
async def _run(self, request_data):
async def _run(self, request_data, original_exception: Exception | None = None):
from unittest.mock import AsyncMock, patch
from litellm.proxy._types import UserAPIKeyAuth
@ -701,17 +745,275 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=Exception("boom"),
original_exception=original_exception or Exception("boom"),
user_api_key_dict=UserAPIKeyAuth(),
)
def _timeout(self, model: str, cause: Exception) -> Exception:
try:
try:
raise cause
except Exception as inner:
raise litellm.Timeout("Request timed out.", model=model, llm_provider="openai") from inner
except litellm.Timeout as timed_out:
return timed_out
def _logging_obj(self, model_call_details):
logging_obj = MagicMock()
logging_obj.model_call_details = model_call_details
return logging_obj
@pytest.mark.asyncio
async def test_dispatched_failure_estimates_input_tokens_with_zero_cost(self):
async def test_timed_out_request_estimates_input_tokens_priced_at_input_rate(self):
from datetime import datetime
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime.now(),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
),
"metadata": {},
"response_cost": 123.0,
}
await self._run(
request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ReadTimeout("no bytes in time"))
)
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
assert estimated.prompt_tokens > 0
assert estimated.completion_tokens == 0
assert estimated.total_tokens == estimated.prompt_tokens
expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens)
assert expected_input_cost > 0
assert request_data["response_cost"] == pytest.approx(expected_input_cost)
@pytest.mark.asyncio
async def test_client_disconnect_after_dispatch_is_priced_at_input_rate(self):
from datetime import datetime
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime.now(),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True,
}
),
"metadata": {},
}
await self._run(
request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
)
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens)
assert expected_input_cost > 0
assert request_data["response_cost"] == pytest.approx(expected_input_cost)
@pytest.mark.asyncio
async def test_upstream_499_error_is_a_plain_failure_not_a_priced_disconnect(self):
"""A provider answering HTTP 499 surfaces as litellm.APIError with status_code 499. Only the
proxy's own disconnect HTTPException means the caller hung up mid-generation, so the upstream
error keeps the estimated usage at zero cost like any other post-dispatch provider error."""
from datetime import datetime
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 0),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
),
"metadata": {},
}
await self._run(
request_data,
original_exception=litellm.APIError(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
message="client closed request",
llm_provider="openai",
model="gpt-3.5-turbo",
),
)
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
assert estimated.prompt_tokens > 0
assert request_data["response_cost"] == 0.0
@pytest.mark.asyncio
async def test_client_disconnect_after_the_call_already_completed_is_not_priced_again(self):
"""The LLM task finished and claimed its success log (which carries the real usage and cost)
while a during_call guardrail task was still pending, then the caller hung up. The 499 raised
for that disconnect must not price the prompt a second time."""
from datetime import datetime
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "count these input tokens please"}],
stream=False,
call_type="acompletion",
start_time=datetime(2026, 1, 1, 12, 0, 0),
litellm_call_id="completed-before-disconnect",
function_id="fn",
)
logging_obj.model_call_details.update(
{
"first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
)
logging_obj.mark_async_success_log_enqueued()
request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}
await self._run(
request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
)
assert request_data.get("response_cost", 0.0) == 0.0
@pytest.mark.asyncio
async def test_client_disconnect_with_a_claimed_but_deferred_success_log_is_still_priced(self):
"""With a post_call guardrail registered the success log is claimed but parked in the deferred slot,
and a 499 raised before the flush means it never runs. Nothing else bills that request, so the
failure row keeps the estimated input cost."""
from datetime import datetime
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "count these input tokens please"}],
stream=False,
call_type="acompletion",
start_time=datetime(2026, 1, 1, 12, 0, 0),
litellm_call_id="deferred-success-never-flushed",
function_id="fn",
)
logging_obj.model_call_details.update(
{
"first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
)
logging_obj._defer_async_logging = True
logging_obj._enqueue_deferred_logging = lambda: None
request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}
await self._run(
request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
)
assert request_data["response_cost"] > 0.0
@pytest.mark.asyncio
async def test_client_disconnect_before_provider_handoff_is_not_priced(self):
"""A caller that hung up while the request was still in pre-call hooks never reached the
provider, so there is no handoff instant on the logging object and the 499 row is not priced."""
from datetime import datetime
from unittest.mock import AsyncMock, patch
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy._types import UserAPIKeyAuth
logging_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "count these input tokens please"}],
stream=False,
call_type="acompletion",
start_time=datetime.now(),
litellm_call_id="lit145-pre-handoff",
function_id="lit145",
)
assert logging_obj.model_call_details.get("first_api_call_start_time") is None
request_data = {"litellm_logging_obj": logging_obj, "model": "gpt-3.5-turbo", "metadata": {}}
proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
with (
patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()),
patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()),
):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"),
)
assert request_data.get("first_api_call_start_time") is None
assert "combined_usage_object" not in request_data
assert request_data.get("response_cost", 0.0) == 0.0
@pytest.mark.asyncio
async def test_connect_timeout_never_reached_the_provider_stays_zero_cost(self):
from datetime import datetime
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime.now(),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
),
"metadata": {},
}
await self._run(
request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ConnectTimeout("no route"))
)
assert isinstance(request_data["combined_usage_object"], Usage)
assert request_data["response_cost"] == 0.0
@pytest.mark.asyncio
async def test_dispatched_non_timeout_failure_estimates_input_tokens_with_zero_cost(self):
from datetime import datetime
from litellm.types.utils import Usage
@ -733,8 +1035,6 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
assert estimated.prompt_tokens > 0
assert estimated.completion_tokens == 0
assert estimated.total_tokens == estimated.prompt_tokens
assert request_data["response_cost"] == 0.0
@pytest.mark.asyncio
@ -1901,7 +2201,355 @@ def test_usage_the_stream_already_recovered_beats_an_estimate():
assert lifted == (recovered, 0.25)
def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost():
def test_a_timed_out_request_on_an_unmapped_model_keeps_its_row_at_zero_cost():
from litellm.proxy.utils import _failure_usage_to_lift
lifted = _failure_usage_to_lift(
model_call_details={
"call_type": "acompletion",
"model": "my-unmapped-model",
"custom_llm_provider": "openai",
"messages": ONE_USER_MESSAGE,
},
request_body={},
dispatched=True,
provider_billed=True,
)
assert lifted is not None
usage, response_cost = lifted
assert usage.prompt_tokens > 0
assert response_cost == 0.0
def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the_deployment(monkeypatch):
"""Router._create_deployment registers a deployment's own ``input_cost_per_token`` under its
``model_info.id``; the failure row prices with that entry through the same
``deployment_pricing_model_info`` the success path uses."""
from litellm.proxy.utils import _failure_usage_to_lift
monkeypatch.setitem(
litellm.model_cost,
"dep-lit145",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0},
)
lifted = _failure_usage_to_lift(
model_call_details={
"call_type": "acompletion",
"model": FAILURE_USAGE_MODEL,
"messages": ONE_USER_MESSAGE,
},
request_body={},
dispatched=True,
provider_billed=True,
deployment_id="dep-lit145",
)
assert lifted is not None
usage, response_cost = lifted
assert usage.prompt_tokens > 0
assert response_cost == pytest.approx(usage.prompt_tokens * 0.25)
@pytest.mark.parametrize(("prompt_tokens", "expected_rate"), [(1_000, 1e-7), (250_000, 5e-7)])
def test_a_timed_out_request_is_priced_at_the_deployments_long_context_tier(monkeypatch, prompt_tokens, expected_rate):
"""A deployment that declares ``input_cost_per_token_above_200k_tokens`` next to its standard rate
bills a 250k-token prompt at the tier on the success path; the failure row must not flatten it to
the standard rate."""
from litellm.proxy.utils import _estimated_input_cost
monkeypatch.setitem(
litellm.model_cost,
"dep-lit-f9-tiered",
{
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-7,
"input_cost_per_token_above_200k_tokens": 5e-7,
"output_cost_per_token": 4e-7,
},
)
response_cost = _estimated_input_cost(
model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"},
prompt_tokens=prompt_tokens,
deployment_id="dep-lit-f9-tiered",
base_model=None,
)
assert response_cost == pytest.approx(prompt_tokens * expected_rate)
def test_every_earlier_timed_out_attempt_is_billed_at_its_own_deployment_rate(monkeypatch):
"""The router leaves one RetryAttemptRecord per failed attempt in metadata.previous_models, the
final attempt included; the earlier Timeout ones are billed (each at the deployment that served
it), a 429 attempt is not, and the final attempt is left to the failure row itself."""
from litellm.proxy.utils import _billed_earlier_attempts
monkeypatch.setitem(
litellm.model_cost,
"dep-lit145-b",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0},
)
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
public_rate, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=24)
attempts = _billed_earlier_attempts(
model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"},
litellm_params={
"metadata": {
"attempted_retries": 3,
"previous_models": (
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0},
{"deployment_id": "dep-lit145-b", "exception_type": "RateLimitError", "attempted_retries": 1},
{"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 2},
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 3},
),
}
},
estimated_usage=usage,
base_model=None,
deployment_id="dep-lit145-a",
)
assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [(0, "dep-lit145-a"), (2, "dep-lit145-b")]
assert attempts[0].response_cost == pytest.approx(public_rate)
assert attempts[0].response_cost > 0
assert attempts[1].response_cost == pytest.approx(24 * 0.25)
assert all(a.usage is usage for a in attempts)
def test_an_earlier_attempt_on_another_model_is_priced_at_that_models_rate_not_the_final_models(monkeypatch):
"""After a fallback hop the final attempt runs on another model group, so the records of the group
that timed out first name their own deployment model (and its ``base_model`` when the deployment
name is not in the cost map). Each is billed at that model's input rate, not at the final model's."""
from litellm.proxy.utils import _billed_earlier_attempts
for pricing_model, provider, input_rate in (
("lit145-final-model", "openai", 0.5),
("lit145-first-model", "openai", 0.02),
("azure/lit145-first-model", "azure", 0.02),
):
monkeypatch.setitem(
litellm.model_cost,
pricing_model,
{"litellm_provider": provider, "mode": "chat", "input_cost_per_token": input_rate, "output_cost_per_token": 1.0},
)
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
attempts = _billed_earlier_attempts(
model_call_details={"model": "openai/lit145-final-model", "custom_llm_provider": "openai"},
litellm_params={
"metadata": {
"attempted_retries": 1,
"model_info": {"id": "dep-lit145-final"},
"previous_models": (
{
"deployment_id": "dep-lit145-first",
"model": "openai/lit145-first-model",
"base_model": None,
"exception_type": "Timeout",
"attempted_retries": 0,
},
{
"deployment_id": "dep-lit145-first-azure",
"model": "azure/lit145-first-deployment",
"base_model": "azure/lit145-first-model",
"exception_type": "Timeout",
"attempted_retries": 1,
},
{
"deployment_id": "dep-lit145-final",
"model": "openai/lit145-final-model",
"base_model": None,
"exception_type": "Timeout",
"attempted_retries": 0,
},
{
"deployment_id": "dep-lit145-final",
"model": "openai/lit145-final-model",
"base_model": None,
"exception_type": "Timeout",
"attempted_retries": 1,
},
),
}
},
estimated_usage=usage,
base_model=None,
deployment_id="dep-lit145-final",
)
assert [(a.attempted_retries, a.deployment_id, a.response_cost) for a in attempts] == [
(0, "dep-lit145-first", pytest.approx(24 * 0.02)),
(1, "dep-lit145-first-azure", pytest.approx(24 * 0.02)),
(0, "dep-lit145-final", pytest.approx(24 * 0.5)),
]
def test_earlier_attempts_are_priced_from_the_prompt_estimate_not_the_final_streams_recovered_usage():
"""A final attempt whose stream broke mid-flight lifts its recovered usage (prompt plus the chunks
delivered); the earlier timed-out attempts never produced output, so they are priced from the
input-only estimate and not from that recovered usage."""
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.proxy.utils import _estimated_prompt_usage, _failure_fields_to_lift
logging_obj = _dispatched_failure_logging_obj(
FAILURE_USAGE_MODEL,
get_litellm_params(
metadata={
"attempted_retries": 1,
"model_info": {"id": "dep-lit145-final"},
"previous_models": (
{"deployment_id": "dep-lit145-final", "exception_type": "Timeout", "attempted_retries": 0},
),
}
),
)
recovered = litellm.Usage(prompt_tokens=500, completion_tokens=40, total_tokens=540)
logging_obj.model_call_details["combined_usage_object"] = recovered
logging_obj.model_call_details["response_cost"] = 0.01
lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
estimate = _estimated_prompt_usage(logging_obj.model_call_details, {})
assert lifted["combined_usage_object"] is recovered
assert [(a.usage.prompt_tokens, a.usage.completion_tokens) for a in lifted["billed_failed_attempts"]] == [
(estimate.prompt_tokens, 0)
]
assert estimate.prompt_tokens < recovered.prompt_tokens
def test_a_timed_out_attempt_from_the_group_a_fallback_left_is_still_billed():
"""After a fallback hop the router restarts ``attempted_retries`` at 0 for the new model group and
re-logs the attempt that exhausted the previous group, so the final attempt (dep-b, retry 1) must be
told from the previous group's attempt with the same count (dep-a, retry 1), and the re-logged
record must bill once."""
from litellm.proxy.utils import _billed_earlier_attempts
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
attempts = _billed_earlier_attempts(
model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"},
litellm_params={
"metadata": {
"attempted_retries": 1,
"model_info": {"id": "dep-lit145-b"},
"previous_models": (
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0},
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1},
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1},
{"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 0},
{"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 1},
),
}
},
estimated_usage=usage,
base_model=None,
deployment_id="dep-lit145-b",
)
assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [
(0, "dep-lit145-a"),
(1, "dep-lit145-a"),
(0, "dep-lit145-b"),
]
assert all(a.response_cost > 0 for a in attempts)
def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object:
"""A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params()
shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff
instant, so the lift treats the request as dispatched."""
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model=model,
messages=ONE_USER_MESSAGE,
stream=False,
call_type="acompletion",
start_time=real_datetime.datetime.now(),
litellm_call_id="call-lit145",
function_id="fn-lit145",
)
logging_obj.update_environment_variables(model=model, user="", optional_params={}, litellm_params=litellm_params)
logging_obj.model_call_details["first_api_call_start_time"] = real_datetime.datetime.now()
logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE
logging_obj.model_call_details["call_type"] = "acompletion"
return logging_obj
def test_the_deployment_rate_is_found_on_the_logging_objects_real_litellm_params(monkeypatch):
"""``get_litellm_params()`` always carries a ``litellm_metadata`` key (None on chat
completions) next to the router-stamped ``metadata.model_info.id``; the failure row must
still price at the deployment's own rate, like ``Logging.get_router_model_id`` on the
success path."""
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.proxy.utils import _failure_fields_to_lift
monkeypatch.setitem(
litellm.model_cost,
"dep-lit145-real-shape",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 0.25},
)
litellm_params = get_litellm_params(metadata={"model_info": {"id": "dep-lit145-real-shape"}})
assert "litellm_metadata" in litellm_params and litellm_params["litellm_metadata"] is None
logging_obj = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, litellm_params)
lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
assert lifted["response_cost"] == pytest.approx(lifted["combined_usage_object"].prompt_tokens * 0.25)
def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path():
"""An Azure deployment name is not in the cost map; the success path prices it through
``model_info.base_model`` (``get_base_model_from_metadata``) and so must the failure row."""
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.proxy.utils import _failure_fields_to_lift
logging_obj = _dispatched_failure_logging_obj(
"azure/lit145-not-in-cost-map",
get_litellm_params(metadata={"model_info": {"base_model": "azure/gpt-4.1"}}),
)
logging_obj.model_call_details["custom_llm_provider"] = "azure"
lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
expected_cost, _ = litellm.cost_per_token(
model="azure/gpt-4.1", prompt_tokens=lifted["combined_usage_object"].prompt_tokens
)
assert expected_cost > 0
assert lifted["response_cost"] == pytest.approx(expected_cost)
def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price():
"""``model_call_details`` is flattened with the request's optional_params, so a body key
named ``litellm_params`` can land there; the deployment rate must come from the logging
object's own ``litellm_params`` instead."""
from litellm.proxy.utils import _failure_usage_to_lift
lifted = _failure_usage_to_lift(
model_call_details={
"call_type": "acompletion",
"model": FAILURE_USAGE_MODEL,
"messages": ONE_USER_MESSAGE,
"litellm_params": {"input_cost_per_token": 0},
},
request_body={},
dispatched=True,
provider_billed=True,
deployment_id=None,
)
assert lifted is not None
usage, response_cost = lifted
expected_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens)
assert response_cost == pytest.approx(expected_cost)
assert response_cost > 0
def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rate():
from litellm.proxy.utils import _failure_usage_to_lift
lifted = _failure_usage_to_lift(
@ -1912,13 +2560,16 @@ def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost():
},
request_body={},
dispatched=True,
provider_billed=True,
)
assert lifted is not None
usage, response_cost = lifted
assert usage.prompt_tokens > 0
assert usage.completion_tokens == 0
assert response_cost == 0.0
expected_input_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens)
assert expected_input_cost > 0
assert response_cost == pytest.approx(expected_input_cost)
@pytest.mark.parametrize(

50
tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py
View file

@ -917,3 +917,53 @@ async def test_update_spend_logs_parks_failed_batch_in_redis_with_wire_safe_date
parked = await buffer.get_spend_logs_from_redis_buffer(limit=10)
assert mock_prisma_client.spend_log_transactions == []
assert [(row["request_id"], row["startTime"]) for row in parked] == [("a", started.isoformat())]
async def _create_many_while_postgres_is_restarting(**_: object) -> int:
from prisma.engine.utils import handle_response_errors
handle_response_errors(
MagicMock(),
[
{
"error": "Error in connector: Can't reach database server at `127.0.0.1:5432`",
"user_facing_error": {"error_code": "P1001", "message": "Can't reach database server", "meta": None},
}
],
)
return 0
@pytest.mark.asyncio
async def test_update_spend_logs_retries_and_requeues_batch_when_postgres_restart_tears_down_prisma_engine(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
"""While Postgres restarts the prisma engine answers the write with a P1001
payload whose ``meta`` is ``None``, which prisma's own response parser trips
over before it can build a typed error. That outage must be retried like
any other, and once retries exhaust the dequeued batch goes back to the head
of the queue so the rows land when Postgres is back.
"""
async def _fake_sleep(_: float) -> None:
return None
monkeypatch.setattr(utils_mod.asyncio, "sleep", _fake_sleep)
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
side_effect=_create_many_while_postgres_is_restarting
)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="c")]
with pytest.raises(AttributeError):
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=2,
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
logs_to_process=[make_spend_log_row(request_id="a"), make_spend_log_row(request_id="b")],
)
assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["a", "b", "c"]
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 3

36
tests/unit/batches/test_batch_utils.py
View file

@ -641,7 +641,7 @@ async def test_calculate_vertex_disable_transform_path(monkeypatch):
monkeypatch.setattr(
bu,
"calculate_vertex_ai_batch_cost_and_usage",
lambda content, model, model_info=None: bu.BatchCostUsageResult(
lambda content, model, model_info=None, vertex_location=None: bu.BatchCostUsageResult(
cost=9.9,
usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3),
models=["gemini-2.0-flash-001"],
@ -672,7 +672,7 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch):
monkeypatch.setattr(
bu,
"calculate_vertex_ai_batch_cost_and_usage",
lambda content, model, model_info=None: pytest.fail("raw vertex path should not run"),
lambda content, model, model_info=None, vertex_location=None: pytest.fail("raw vertex path should not run"),
)
result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=[], custom_llm_provider="vertex_ai")
@ -1113,6 +1113,36 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk
assert result.failed_requests == 0
@pytest.mark.asyncio
async def test_handle_completed_vertex_batch_regional_location_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm.files.main as files_main
rows = [_vertex_openai_row("request-1", "gemini-3.6-flash", 10, 5)]
async def fake_afile_content(**kw):
return type("R", (), {"content": _vertex_jsonl(rows)})()
monkeypatch.setattr(files_main, "afile_content", fake_afile_content)
global_result = await bu._handle_completed_batch(
_batch("gs://litellm-bucket/output/predictions.jsonl"),
custom_llm_provider="vertex_ai",
litellm_params={"vertex_project": "proj-1", "vertex_location": "global"},
vertex_location="global",
)
regional_result = await bu._handle_completed_batch(
_batch("gs://litellm-bucket/output/predictions.jsonl"),
custom_llm_provider="vertex_ai",
litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"},
vertex_location="us-central1",
)
uplift = litellm.model_cost["vertex_ai/gemini-3.6-flash"]["regional_endpoint_uplift_multiplier"]
assert global_result.cost > 0
assert regional_result.cost == pytest.approx(global_result.cost * uplift, rel=1e-9)
assert regional_result.usage == global_result.usage
@pytest.mark.asyncio
async def test_output_file_content_no_output_file_id_raises():
with pytest.raises(ValueError, match="Output file id is None"):
@ -1342,7 +1372,7 @@ async def test_handle_completed_batch_vertex_disable_transform_path(monkeypatch)
monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", True, raising=False)
seen: dict = {}
def fake_vertex_calc(content, model, model_info=None):
def fake_vertex_calc(content, model, model_info=None, vertex_location=None):
seen["content"] = content
seen["model"] = model
return bu.BatchCostUsageResult(

120
tests/unit/decisions/test_call.py Normal file
View file

@ -0,0 +1,120 @@
import base64
import json
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final
import pytest
import litellm
from litellm.decisions.main import _prepare_decisions_call
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.systemone.main import _prepare_systemone_call
QUESTION_TEXT: Final = "Is this about a billing problem?"
CONNECTION: Final = {
"model": "hosted_vllm/jev-fake",
"safety_identifier": None,
"api_key": "fake-provider-key",
"api_base": "http://127.0.0.1:4010",
"timeout": None,
"custom_llm_provider": None,
"extra_headers": None,
}
def _logging_obj(call_type: str) -> Logging:
return Logging(
model="hosted_vllm/jev-fake",
messages="placeholder",
stream=False,
call_type=call_type,
start_time=datetime.now(timezone.utc),
litellm_call_id="lit-9659",
function_id="lit-9659",
)
def _logged_request(logging_obj: Logging) -> Mapping[str, object]:
messages: Final = logging_obj.model_call_details["messages"]
assert len(messages) == 1 and messages[0]["role"] == "user"
return json.loads(messages[0]["content"])
def test_systemone_logs_every_request_field_the_caller_sent():
logging_obj: Final = _logging_obj("asystemone")
_prepare_systemone_call(
**CONNECTION,
state={"message": "My card was charged twice."},
questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}},
kwargs={"litellm_logging_obj": logging_obj},
)
logged: Final = _logged_request(logging_obj)
assert logged["state"] == {"message": "My card was charged twice."}
assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT
assert set(logged) == {"state", "questions"}
def test_decisions_logs_every_request_field_the_caller_sent():
logging_obj: Final = _logging_obj("adecisions")
_prepare_decisions_call(
**CONNECTION,
decision_input="My card was charged twice.",
questions=[{"type": "predicate", "instructions": QUESTION_TEXT}],
kwargs={"litellm_logging_obj": logging_obj},
)
logged: Final = _logged_request(logging_obj)
assert logged["input"] == "My card was charged twice."
assert logged["questions"][0]["instructions"] == QUESTION_TEXT
assert set(logged) == {"input", "questions"}
def test_request_is_logged_even_when_it_is_rejected_before_the_provider_call():
logging_obj: Final = _logging_obj("asystemone")
with pytest.raises(litellm.BadRequestError):
_prepare_systemone_call(
**CONNECTION,
state={"message": "My card was charged twice."},
questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}},
kwargs={"litellm_logging_obj": logging_obj, "input": "sent to the wrong function"},
)
logged: Final = _logged_request(logging_obj)
assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT
def test_decisions_request_is_logged_even_when_its_questions_fail_validation():
logging_obj: Final = _logging_obj("adecisions")
with pytest.raises(litellm.BadRequestError):
_prepare_decisions_call(
**CONNECTION,
decision_input="My card was charged twice.",
questions=[{"type": "not-a-question-type", "instructions": QUESTION_TEXT}],
kwargs={"litellm_logging_obj": logging_obj},
)
logged: Final = _logged_request(logging_obj)
assert logged["questions"][0]["instructions"] == QUESTION_TEXT
def test_non_utf8_bytes_in_the_request_are_logged_and_rejected_with_a_400():
logging_obj: Final = _logging_obj("adecisions")
raw_input: Final = b"\xff\xfe not utf-8"
with pytest.raises(litellm.BadRequestError):
_prepare_decisions_call(
**CONNECTION,
decision_input=raw_input,
questions=[{"type": "predicate", "instructions": QUESTION_TEXT}],
kwargs={"litellm_logging_obj": logging_obj},
)
logged: Final = _logged_request(logging_obj)
assert base64.urlsafe_b64decode(logged["input"]) == raw_input
assert logged["questions"][0]["instructions"] == QUESTION_TEXT

85
tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -99,6 +99,31 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re
assert completion_cost == pytest.approx(380 * 9.7e-7)
@pytest.mark.parametrize(
("model_info", "expected_prompt_cost"),
[
({"input_cost_per_token": 1e-7, "input_cost_per_character": 2.5e-8, "output_cost_per_token": 0.0}, 35 * 2.5e-8),
({"input_cost_per_token": 1.5e-7, "output_cost_per_token": 0.0}, 11 * 1.5e-7),
],
ids=["character_priced_model_bills_characters_instead_of_tokens", "token_priced_model_ignores_the_character_count"],
)
def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit(
model_info: ModelInfo, expected_prompt_cost: float
) -> None:
usage = Usage(
prompt_tokens=11,
completion_tokens=0,
total_tokens=11,
prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35),
)
prompt_cost, completion_cost = generic_cost_per_token(
model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info
)
assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0)
def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None:
openai_info: ModelInfo = {"input_cost_per_token": 2e-6}
fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6}
@ -450,6 +475,66 @@ def test_video_tokens_fallback_to_base_cost():
assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12)
def test_prompt_audio_and_video_tokens_without_modality_rates_bill_at_tiered_input_rate():
"""Prompt audio/video tokens with no modality rate are prompt tokens, so a >200k prompt bills them above-200k."""
from unittest.mock import patch
mock_model_info = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_above_200k_tokens": 4e-6,
"output_cost_per_token": 1.2e-5,
"output_cost_per_token_above_200k_tokens": 1.8e-5,
}
usage = Usage(
prompt_tokens=816032,
completion_tokens=4559,
total_tokens=820591,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=9033, audio_tokens=14999, video_tokens=792000),
completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=4559),
)
with patch(
"litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info",
return_value=mock_model_info,
):
prompt_cost, completion_cost = generic_cost_per_token(
model="test-model", usage=usage, custom_llm_provider="vertex_ai"
)
assert round(prompt_cost, 9) == round(816032 * 4e-6, 9)
assert round(completion_cost, 9) == round(4559 * 1.8e-5, 9)
def test_prompt_modality_rates_still_win_over_tiered_input_rate():
"""An explicit input_cost_per_audio_token / input_cost_per_video_token beats the fallback, even above 200k."""
from unittest.mock import patch
mock_model_info = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_above_200k_tokens": 4e-6,
"input_cost_per_audio_token": 3e-6,
"input_cost_per_video_token": 1e-6,
"output_cost_per_token": 1.2e-5,
}
usage = Usage(
prompt_tokens=300000,
completion_tokens=10,
total_tokens=300010,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100000, audio_tokens=100000, video_tokens=100000),
completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=10),
)
with patch(
"litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info",
return_value=mock_model_info,
):
prompt_cost, _ = generic_cost_per_token(model="test-model", usage=usage, custom_llm_provider="vertex_ai")
assert round(prompt_cost, 9) == round(100000 * 4e-6 + 100000 * 3e-6 + 100000 * 1e-6, 9)
def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map):
# gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing
model = "gemini-2.5-pro"

36
tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py
View file

@ -17,6 +17,7 @@ import pytest
import litellm
import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod
import litellm.proxy.common_request_processing as common_request_processing_mod
from litellm.cost_calculator import _select_model_name_for_cost_calc
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
ResponseMetadata,
@ -651,3 +652,38 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_
assert result._response_ms == pytest.approx(2000)
assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2)
def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model():
start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0)
logging_obj: Final = Logging(
model="azure/ptu-deployment",
messages=[{"role": "user", "content": "Hello"}],
stream=False,
call_type="completion",
start_time=start_time,
litellm_call_id="provider-model-response-metadata",
function_id="f",
)
logging_obj.update_environment_variables(
model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure"
)
result: Final = ModelResponse(
model="gpt-5.4-nano-2026-03-17",
usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18),
)
update_response_metadata(
result=result,
logging_obj=logging_obj,
model="azure/ptu-deployment",
kwargs={},
start_time=start_time,
end_time=start_time + datetime.timedelta(seconds=2),
)
result.model = "model-group-alias"
assert (
_select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure")
== "azure/gpt-5.4-nano-2026-03-17"
)

157
tests/unit/litellm_core_utils/test_litellm_logging.py
View file

@ -20,14 +20,14 @@ from openai._legacy_response import HttpxBinaryResponseContent
import litellm
from litellm._logging import session_id_var, trace_id_var
from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.cost_calculator import ocr_batch_cost
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.litellm_logging import (
_extract_response_obj_and_hidden_params,
get_standard_logging_object_payload,
_get_status_fields,
get_standard_logging_object_payload,
set_callbacks,
)
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
@ -335,6 +335,20 @@ def test_get_masked_api_base(logging_obj):
assert type(masked_api_base) == str
@pytest.mark.parametrize("no_upstream_call", [False, True])
def test_pre_call_stamps_the_provider_handoff_only_when_a_provider_call_is_made(logging_obj, no_upstream_call):
"""The proxy fires pre_call for a failure it raised itself (auth, rate limit, client disconnect)
so callbacks see it; that call never reached a provider, so neither handoff instant may be set."""
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
if no_upstream_call:
logging_obj.model_call_details[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True
logging_obj.pre_call(input="", api_key="")
assert ("api_call_start_time" in logging_obj.model_call_details) is not no_upstream_call
assert (logging_obj.model_call_details.get("first_api_call_start_time") is not None) is not no_upstream_call
def test_pre_call_tolerates_missing_api_base(logging_obj):
"""Presigned batch retrieves (Mistral, Bedrock) build their own URL and pass api_base=None
to pre_call; masking must not raise or the request's pre-call logging is silently lost."""
@ -1318,6 +1332,56 @@ class TestRetrieveBatchCostPassesModelIdentity:
assert captured["model_info"] is not None
assert captured["model_info"]["input_cost_per_token"] == 0.0
@pytest.mark.asyncio
async def test_forwards_vertex_location_of_the_deployment(self, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.litellm_core_utils import litellm_logging as logging_module
captured: dict[str, object] = {}
from litellm.batches.batch_utils import BatchCostUsageResult
from litellm.types.utils import LiteLLMBatch, Usage
async def fake_handle_completed_batch(**kwargs: object) -> BatchCostUsageResult:
captured.update(kwargs)
return BatchCostUsageResult(
cost=1.25,
usage=Usage(prompt_tokens=1800, completion_tokens=1000, total_tokens=2800),
models=["gemini-3.5-flash"],
successful_requests=1,
failed_requests=0,
)
monkeypatch.setattr(logging_module, "_handle_completed_batch", fake_handle_completed_batch)
obj = LitellmLogging(
model="vertex_ai/gemini-3.5-flash",
messages=[{"role": "user", "content": "Hey"}],
stream=False,
call_type="aretrieve_batch",
start_time=time.time(),
litellm_call_id="batch-call-2",
function_id="f",
)
obj.custom_llm_provider = "vertex_ai"
obj.litellm_params = {"vertex_project": "proj-1", "vertex_location": "us-central1"}
batch = LiteLLMBatch(
id="batch_vertex",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id="file-in",
object="batch",
status="completed",
output_file_id="file-out",
)
with contextlib.suppress(Exception):
await obj._async_success_handler_body(result=batch, start_time=None, end_time=None)
assert captured, "handle_completed_batch was never called"
assert captured["vertex_location"] == "us-central1"
class TestRetrieveBatchPricesOnlyFinalBatches:
"""Regression (LIT-7048): retrieving a provider-id batch priced it on every poll.
@ -6783,10 +6847,10 @@ async def test_streaming_success_callbacks_survive_guardrail_logging_hook_failur
def _resolve(custom_llm_provider, litellm_params, optional_params, model):
from litellm.litellm_core_utils.litellm_logging import (
_resolve_vertex_location_for_cost,
resolve_vertex_location_for_cost,
)
return _resolve_vertex_location_for_cost(
return resolve_vertex_location_for_cost(
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
optional_params=optional_params,
@ -8325,6 +8389,7 @@ class TestAzurePTUSpilloverCost:
ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id"
SERVED_MODEL: Final = "azure/spill-served-model-ptu"
PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment"
PTU_MODEL_INFO: Final = {
"id": ROUTER_MODEL_ID,
"team_id": "team-1",
@ -8360,7 +8425,9 @@ class TestAzurePTUSpilloverCost:
litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None)
litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None)
def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging:
def _logging_obj(
self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None
) -> LitellmLogging:
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag)
obj = LitellmLogging(
model=self.SERVED_MODEL,
@ -8377,7 +8444,7 @@ class TestAzurePTUSpilloverCost:
optional_params={},
litellm_params={
"api_base": "",
"metadata": {"model_info": model_info},
"metadata": {"model_info": model_info, "deployment": deployment},
"input_cost_per_token": litellm_rate,
"output_cost_per_token": litellm_rate,
},
@ -8447,6 +8514,84 @@ class TestAzurePTUSpilloverCost:
finally:
self._unregister_models()
def test_spillover_payload_reports_the_pay_as_you_go_entry_billing_used(self, monkeypatch) -> None:
self._register_models()
try:
obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
response_cost: Final = obj._response_cost_calculator(result=response)
payload: Final = get_standard_logging_object_payload(
kwargs={
"model": self.SERVED_MODEL,
"custom_llm_provider": "azure",
"litellm_params": obj.litellm_params,
"response_cost": response_cost,
},
init_response_obj=response,
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
logging_obj=obj,
status="success",
)
assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST)
assert payload is not None
model_map_value: Final = payload["model_map_information"]["model_map_value"]
assert model_map_value is not None
assert model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"]
assert model_map_value["output_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"]
assert response_cost == pytest.approx(
100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"]
)
finally:
self._unregister_models()
def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None:
self._register_models()
litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"}
try:
obj = self._logging_obj(
dict(self.PTU_MODEL_INFO),
flag="True",
litellm_rate=0.0,
monkeypatch=monkeypatch,
deployment=self.PTU_DEPLOYMENT,
)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
response._hidden_params["provider_response_model"] = response.model
response_cost: Final = obj._response_cost_calculator(result=response)
response.model = "spill-model-group-alias"
payload: Final = get_standard_logging_object_payload(
kwargs={
"model": self.SERVED_MODEL,
"custom_llm_provider": "azure",
"litellm_params": obj.litellm_params,
"response_cost": response_cost,
},
init_response_obj=response,
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
logging_obj=obj,
status="success",
)
assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST)
assert payload is not None
model_map_information: Final = payload["model_map_information"]
assert model_map_information["model_map_key"] != self.PTU_DEPLOYMENT
model_map_value: Final = model_map_information["model_map_value"]
assert model_map_value is not None
assert response_cost == pytest.approx(
100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"]
)
finally:
litellm.model_cost.pop(self.PTU_DEPLOYMENT, None)
self._unregister_models()
def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None:
self._register_models()
custom_model_id: Final = "non-ptu-custom-router-model-id"

90
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -29,7 +29,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transfor
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import (
VertexAIAnthropicConfig,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
from litellm.types.utils import ServerToolUse, Usage
@ -6798,3 +6798,91 @@ def test_chat_dummy_tool_result_for_an_orphaned_tool_call_replays_a_byte_identic
_assert_prefix_stable(requests)
assert [m["role"] for m in requests[0]["messages"]] == ["user", "assistant", "user"]
assert requests[0]["messages"][2]["content"][0]["type"] == "tool_result"
def _make_transform_request(optional_params: dict, litellm_params: dict) -> dict:
return AnthropicConfig().transform_request(
model="claude-3-5-sonnet-20241022",
messages=[{"role": "user", "content": "hi"}],
optional_params=optional_params,
litellm_params=litellm_params,
headers={},
)
def _two_function_tools() -> list[dict]:
return [
{
"type": "function",
"function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}},
}
for i in range(2)
]
@pytest.mark.parametrize(
"control, expected_control",
[
(None, {"type": "ephemeral"}),
({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}),
],
)
def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control):
"""LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which
Anthropic rejected with 400 Extra inputs are not permitted."""
tools = _two_function_tools()
tools_before = copy.deepcopy(tools)
point = {"location": "tool_config", **({"control": control} if control is not None else {})}
litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}}
data = _make_transform_request(
optional_params={"tools": tools, "cache_control_injection_points": [point]},
litellm_params=litellm_params,
)
assert "cache_control_injection_points" not in data
assert data["tools"][-1]["cache_control"] == expected_control
assert "cache_control" not in data["tools"][0]
assert tools == tools_before
assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1"
def test_transform_request_tool_config_point_keeps_the_clients_tool_marker():
tools = _two_function_tools()
tools[0]["cache_control"] = {"type": "ephemeral"}
data = _make_transform_request(
optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]},
litellm_params={},
)
assert "cache_control_injection_points" not in data
assert data["tools"][0]["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in data["tools"][1]
def test_transform_request_tool_config_point_skips_tool_search_tools():
tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES))
tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}]
data = _make_transform_request(
optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]},
litellm_params={},
)
assert data["tools"][1]["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in data["tools"][2]
assert "cache_control" not in data["tools"][0]
def test_transform_request_tool_config_point_without_tools_drops_the_key():
litellm_params = {"metadata": {}}
data = _make_transform_request(
optional_params={"cache_control_injection_points": [{"location": "tool_config"}]},
litellm_params=litellm_params,
)
assert "cache_control_injection_points" not in data
assert "tools" not in data
assert litellm_params["metadata"] == {}

49
tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -1,4 +1,5 @@
import asyncio
import copy
import json
import os
import uuid
@ -1480,6 +1481,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul
assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"]
@pytest.mark.asyncio
async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic():
"""Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21."""
@ -1594,7 +1596,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict
return safeguards, safeguard_results
def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler:
def _upstream_answering_with(
safeguard_results: list[dict[str, object]], captured: dict[str, object]
) -> AsyncHTTPHandler:
def upstream_records_the_request(request: httpx.Request) -> httpx.Response:
captured["body"] = json.loads(request.content)
captured["anthropic-beta"] = request.headers.get("anthropic-beta")
@ -1620,7 +1624,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture
_CLIENT_BETA_HEADERS: Final = (
pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"),
pytest.param(
{"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"
),
pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"),
pytest.param({}, id="client_sends_no_beta_header"),
)
@ -1685,3 +1691,42 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet
assert "anthropic_beta" not in captured["body"]
assert captured["anthropic-beta"].split(",").count("dangerous-tool-use-2026-09-03") == 1
assert response["safeguard_results"] == safeguard_results
def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch):
"""LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the
native path dropped it silently, so the tools went out without any cache breakpoint."""
from litellm.llms.anthropic.experimental_pass_through.messages import handler
captured = {}
def fake_native(**kwargs):
captured["params"] = kwargs["anthropic_messages_optional_request_params"]
captured["kwargs"] = kwargs["kwargs"]
captured["messages"] = kwargs["messages"]
return "native-passthrough"
monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native)
tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)]
tools_before = copy.deepcopy(tools)
result = handler.anthropic_messages_handler(
max_tokens=100,
messages=[{"role": "user", "content": "Hello"}],
model="anthropic/claude-opus-4-6",
api_key="sk-test",
tools=tools,
litellm_metadata={},
cache_control_injection_points=[
{"location": "message", "role": "user"},
{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}},
],
)
assert result == "native-passthrough"
assert "cache_control_injection_points" not in captured["params"]
assert "cache_control_injection_points" not in captured["kwargs"]
assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
assert "cache_control" not in captured["params"]["tools"][0]
assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"}
assert tools == tools_before

4
tests/unit/llms/custom_httpx/test_llm_http_handler.py
View file

@ -2900,7 +2900,7 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati
from litellm.litellm_core_utils.litellm_logging import (
Logging,
_resolve_vertex_location_for_cost,
resolve_vertex_location_for_cost,
)
monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5")
@ -2949,7 +2949,7 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati
global_deployment = await logging_obj_after_handler(GenericLiteLLMParams(vertex_location="global"))
assert global_deployment.litellm_params["vertex_location"] == "global"
assert (
_resolve_vertex_location_for_cost(
resolve_vertex_location_for_cost(
custom_llm_provider="vertex_ai",
litellm_params=global_deployment.litellm_params,
optional_params=global_deployment.optional_params,

0
tests/unit/llms/vertex_ai/vertex_embeddings/__init__.py Normal file
View file

30
tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py Normal file
View file

@ -0,0 +1,30 @@
from typing import Final
import pytest
from litellm.llms.vertex_ai.vertex_embeddings.transformation import VertexAITextEmbeddingConfig
from litellm.types.utils import EmbeddingResponse
def _gecko_prediction(values: list[float], token_count: int) -> dict[str, object]:
return {"embeddings": {"values": values, "statistics": {"token_count": token_count, "truncated": False}}}
@pytest.mark.parametrize(
("metadata", "character_count"),
[({"metadata": {"billableCharacterCount": 10}}, 10), ({}, None)],
ids=["billable_character_count_reported", "no_metadata"],
)
def test_vertex_embedding_usage_carries_the_billable_character_count_google_prices_on(
metadata: dict[str, object], character_count: int | None
) -> None:
payload: Final = {"predictions": [_gecko_prediction([0.1, 0.2, 0.3], 3), _gecko_prediction([1, 2.0, 3], 5)], **metadata}
response: Final = VertexAITextEmbeddingConfig().transform_vertex_response_to_openai(
response=payload, model="text-embedding-005", model_response=EmbeddingResponse()
)
assert (response.usage.prompt_tokens, getattr(response.usage.prompt_tokens_details, "character_count", None)) == (
8,
character_count,
)

210
tests/unit/proxy/hooks/test_search_free_tier.py Normal file
View file

@ -0,0 +1,210 @@
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from litellm.proxy.hooks.search_free_tier import (
apply_search_free_tier,
freed_units,
search_free_tier_key,
search_free_tier_ttl,
)
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc)
RATE: Final = 0.01
class _Counter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
self.ttls: dict[str, int | None] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
self.ttls[key] = ttl
return self.totals[key]
class _BrokenCounter:
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
raise ConnectionError("redis is down")
def _model_info(billing_unit: str, count: int, period: str = "month", provider: str = "vertex_ai") -> dict:
return {
"litellm_provider": provider,
"mode": "chat",
"web_search_billing_unit": billing_unit,
"search_context_cost_per_query": {"search_context_size_medium": RATE},
"search_free_requests": {"count": count, "period": period, "shared_group": "test-group"},
}
def _response(queries: int | None) -> ModelResponse:
details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None
return ModelResponse(
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details)
)
def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]:
sl_object = {
"response_cost": response_cost,
"cache_hit": cache_hit,
"custom_llm_provider": model_info["litellm_provider"],
"cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost},
"model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info},
}
return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object
@pytest.mark.parametrize(
("allowance", "units", "total_after", "expected"),
[
(5000, 5, 5, 5),
(5000, 5, 5000, 5),
(5000, 10, 5005, 5),
(5000, 5, 5010, 0),
(2, 5, 5, 2),
(0, 3, 3, 0),
],
)
def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected):
assert freed_units(allowance, units, total_after) == expected
def test_counter_key_is_per_provider_group_and_utc_period():
assert search_free_tier_key("gemini", {"count": 1, "period": "month", "shared_group": "g3"}, NOW) == (
"litellm:search_free_tier:gemini:g3:2026-10"
)
assert search_free_tier_key("vertex_ai", {"count": 1, "period": "day", "shared_group": "g2"}, NOW) == (
"litellm:search_free_tier:vertex_ai:g2:2026-10-20"
)
@pytest.mark.parametrize(
("period", "boundary"),
[("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))],
)
def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary):
expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW))
assert boundary < expiry < boundary + timedelta(hours=2)
@pytest.mark.asyncio
async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call():
counter = _Counter()
model_info = _model_info("per_query", count=7)
reduced = []
for _ in range(3):
kwargs, sl_object = _call(model_info, response_cost=1.0)
cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"]))
(c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced
assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE)
assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE)
assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5
assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE)
assert b2["discount_amount"] == pytest.approx(2 * RATE)
assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 15}
assert counter.ttls["litellm:search_free_tier:vertex_ai:test-group:2026-10"] > 0
@pytest.mark.asyncio
async def test_gemini_api_and_vertex_ai_grounding_draw_on_separate_allowances():
counter = _Counter()
gemini_api = _model_info("per_query", count=3, provider="gemini")
vertex_ai = _model_info("per_query", count=3, provider="vertex_ai")
kwargs, sl_object = _call(gemini_api, response_cost=1.0)
await apply_search_free_tier(kwargs, sl_object, 1.0, _response(2), counter, NOW)
kwargs, sl_object = _call(vertex_ai, response_cost=1.0)
vertex_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(4), counter, NOW)
kwargs, sl_object = _call(gemini_api, response_cost=1.0)
gemini_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(1), counter, NOW)
assert vertex_cost == pytest.approx(1.0 - 3 * RATE), "vertex ai had its whole allowance left"
assert gemini_cost == pytest.approx(1.0 - RATE), "gemini api had one free query left"
@pytest.mark.asyncio
async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count():
counter = _Counter()
model_info = _model_info("per_prompt", count=1, period="day")
kwargs, sl_object = _call(model_info, response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
kwargs, sl_object = _call(model_info, response_cost=1.0)
second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
assert first == pytest.approx(1.0 - RATE)
assert second == 1.0
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10-20": 2}
@pytest.mark.asyncio
async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
response = {
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
"prompt_tokens_details": {"web_search_requests": 3},
}
}
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", [None, _BrokenCounter()])
async def test_full_price_when_no_counter_is_available(counter):
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()])
async def test_requests_without_grounding_never_touch_the_counter(response):
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_models_without_an_allowance_never_touch_the_counter():
counter = _Counter()
model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"}
kwargs, sl_object = _call(model_info, response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_a_second_callback_run_on_the_same_request_counts_it_once():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW)
assert first == second == pytest.approx(1.0 - 5 * RATE)
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5}
@pytest.mark.asyncio
async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True)
assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0
assert counter.totals == {}
assert "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
async def test_freed_cost_never_exceeds_the_billed_cost():
counter = _Counter()
billed = 3 * RATE
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed)
assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0
assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed)
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5}

28
tests/unit/proxy/test_update_spend.py
View file

@ -321,3 +321,31 @@ async def test_update_spend_logs_multiple_batches_with_failure():
# Verify all logs were cleared from transactions
assert len(prisma_client.spend_log_transactions) == 0
@pytest.mark.asyncio
async def test_update_spend_logs_drops_row_prisma_cannot_encode_and_keeps_the_rest():
"""A row the Prisma engine cannot utf-8 encode (a lone surrogate in a string
column) is permanent, so it must be dropped like any other poison row instead
of being requeued at the head of the queue where it would block every tenant."""
prisma_client = MockPrismaClient()
proxy_logging_obj = create_mock_proxy_logging()
prisma_client.spend_log_transactions = [
{"request_id": "good-1", "request_tags": '["a"]'},
{"request_id": "poison", "request_tags": "\ud800"},
{"request_id": "good-2", "request_tags": '["b"]'},
]
written = []
async def create_many(data, skip_duplicates):
if any("\ud800" in row["request_tags"] for row in data):
raise UnicodeEncodeError("utf-8", "\ud800", 0, 1, "surrogates not allowed")
written.extend(row["request_id"] for row in data)
prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=create_many)
await update_spend(prisma_client, None, proxy_logging_obj)
assert sorted(written) == ["good-1", "good-2"]
assert prisma_client.spend_log_transactions == []
proxy_logging_obj.failure_handler.assert_not_called()

37
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -5311,3 +5311,40 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool
)
assert exc_info.value.status_code == 400
assert "tool_choice={'type': 'file_search'}" in str(exc_info.value)
@pytest.mark.asyncio
async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chunks():
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
chat_stream: Final = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "tell me a story"}],
mock_response="The codename is AZURE-FALCON-42 and the story is long.",
stream=True,
api_key="test-key",
)
iterator: Final = LiteLLMCompletionStreamingIterator(
model="gpt-4o-mini",
litellm_custom_stream_wrapper=chat_stream,
request_input="tell me a story",
responses_api_request={},
custom_llm_provider="openai",
)
assert iterator.partial_response_on_disconnect() is None, "nothing relayed yet, nothing to bill"
events: Final = [await iterator.__anext__() for _ in range(6)]
relayed_text: Final = "".join(
getattr(event, "delta", "") for event in events if event.type == "response.output_text.delta"
)
assert relayed_text, "the bridge should have relayed some text by now"
incomplete_event: Final = iterator.partial_response_on_disconnect()
assert incomplete_event is not None
assert incomplete_event.type == "response.incomplete"
assert incomplete_event.response.status == "incomplete"
assert incomplete_event.response.usage is not None
assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text)
assert incomplete_event.response.usage.input_tokens > 0

99
tests/unit/responses/test_responses_streaming_iterator.py
View file

@ -7,7 +7,10 @@ crashed with exit 139 whenever any CustomLogger was registered).
"""
import asyncio
import json
import time
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
@ -17,6 +20,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module
from litellm.responses import streaming_iterator as responses_streaming_iterator_module
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.types.llms.openai import ResponsesAPIResponse
@ -156,3 +160,98 @@ async def test_sync_callbacks_run_only_after_async_handler_completes(recording_e
submit_times = recording_executor.submit_times_for(logging_obj)
assert len(submit_times) == 1
assert submit_times[0] >= recorder.async_hook_finished
_CREATED_RESPONSE_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = {
"id": "resp_partial",
"object": "response",
"created_at": 1700000000.0,
"model": "gpt-5.4-nano",
"status": "in_progress",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"service_tier": "priority",
}
_CREATED_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = {
"type": "response.created",
"sequence_number": 0,
"response": _CREATED_RESPONSE_WITH_PRIORITY_TIER,
}
def _text_delta(text: str, sequence_number: int) -> Mapping[str, object]:
return {
"type": "response.output_text.delta",
"sequence_number": sequence_number,
"item_id": "msg_partial",
"output_index": 0,
"content_index": 0,
"delta": text,
}
def _completed_after(text: str) -> Mapping[str, object]:
return {
"type": "response.completed",
"sequence_number": 9,
"response": {
**_CREATED_RESPONSE_WITH_PRIORITY_TIER,
"status": "completed",
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
},
}
def _live_iterator(logging_obj: LitellmLogging) -> ResponsesAPIStreamingIterator:
return ResponsesAPIStreamingIterator(
response=httpx.Response(200),
model="gpt-5.4-nano",
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=logging_obj,
request_data={"model": "gpt-5.4-nano", "input": "tell me a story"},
custom_llm_provider="openai",
)
def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier():
iterator: Final = _live_iterator(_make_logging_obj())
for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time ", 1), _text_delta("a fox slept.", 2)):
iterator._process_chunk(json.dumps(event))
incomplete_event: Final = iterator.partial_response_on_disconnect()
assert incomplete_event is not None
assert incomplete_event.type == "response.incomplete"
partial: Final = incomplete_event.response
assert partial.status == "incomplete"
assert getattr(partial, "service_tier") == "priority"
assert partial.output[0].content[0].text == "Once upon a time a fox slept."
assert partial.usage.output_tokens == litellm.token_counter(
model="gpt-5.4-nano", text="Once upon a time a fox slept."
)
assert partial.usage.input_tokens == litellm.token_counter(
model="gpt-5.4-nano", messages=[{"role": "user", "content": "tell me a story"}]
)
assert partial.usage.total_tokens == partial.usage.input_tokens + partial.usage.output_tokens
@pytest.mark.parametrize(
"events",
[
(),
(_CREATED_WITH_PRIORITY_TIER,),
(_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1), _completed_after("Once upon a time")),
],
ids=["nothing_relayed", "created_only", "already_completed"],
)
@pytest.mark.asyncio
async def test_partial_response_on_disconnect_is_none_when_there_is_nothing_to_bill_or_the_stream_billed_itself(
events: tuple[Mapping[str, object], ...],
):
iterator: Final = _live_iterator(_make_logging_obj())
for event in events:
iterator._process_chunk(json.dumps(event))
assert iterator.partial_response_on_disconnect() is None

20
tests/unit/responses/test_streaming_iterator.py
View file

@ -437,26 +437,26 @@ def _responses_api_response_with_usage() -> ResponsesAPIResponse:
def test_stamp_responses_usage_cost_stamps_computed_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.return_value = 0.000704
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
logging_obj._response_cost_calculator.assert_called_once_with(result=response)
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
setattr(response.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
logging_obj._response_cost_calculator.assert_not_called()
@ -476,7 +476,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage(
{
"input_tokens": 29,
@ -488,7 +488,7 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
)
logging_obj = Mock(spec=LiteLLMLoggingObj)
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(3e-05)
@ -497,12 +497,12 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149})
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.return_value = 0.000704
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(0.000704)
@ -510,13 +510,13 @@ def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
def test_stamp_responses_usage_cost_survives_calculator_failure():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) is None

46
tests/unit/router_strategy/test_lowest_cost.py
View file

@ -101,3 +101,49 @@ async def test_async_log_success_event_counts_a_response_with_no_completion_toke
)
assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
_TWO_DOLLARS_PER_TOKEN = {"input_cost_per_token": 1.0, "output_cost_per_token": 1.0}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"azure_litellm_params, base_model",
[
({"model": "azure/my-gpt4-deploy"}, "azure/gpt-4.1"),
({"model": "azure/gpt-4.1"}, "azure/not-a-real-model"),
],
ids=["base_model_priced", "unmapped_base_model_falls_back_to_deployment_model"],
)
async def test_cost_based_routing_prices_an_azure_deployment_through_its_base_model(
azure_litellm_params: dict, base_model: str
):
handler = LowestCostLoggingHandler(router_cache=DualCache())
healthy_deployments = [
{
"model_name": "gpt-4.1",
"litellm_params": {"model": "openai/gpt-4.1", **_TWO_DOLLARS_PER_TOKEN},
"model_info": {"id": "openai-two-dollars"},
},
{
"model_name": "gpt-4.1",
"litellm_params": azure_litellm_params,
"model_info": {"id": "azure-cost-map-priced", "base_model": base_model},
},
]
picked = await handler.async_get_available_deployments(
model_group="gpt-4.1", healthy_deployments=healthy_deployments
)
assert picked is not None and picked["model_info"]["id"] == "azure-cost-map-priced", (
"a cost-map priced Azure deployment must beat a $2 per token deployment, "
"it loses only when it is scored at the $10 unmapped default"
)

53
tests/unit/router_utils/test_deployment_pricing.py Normal file
View file

@ -0,0 +1,53 @@
import pytest
from litellm.router_utils.deployment_pricing import deployment_pricing_models, pricing_models
@pytest.mark.parametrize(
"base_model, model, expected",
[
("azure/gpt-4.1", "azure/my-gpt4-deploy", ("azure/gpt-4.1", "azure/my-gpt4-deploy")),
(None, "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)),
("", "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)),
("azure/gpt-4.1", "azure/gpt-4.1", ("azure/gpt-4.1",)),
(None, None, ()),
],
ids=["base_model_first", "no_base_model", "blank_base_model", "same_name_once", "nothing_named"],
)
def test_pricing_models_walks_base_model_then_deployment_model(base_model, model, expected):
assert pricing_models(base_model, model) == expected
@pytest.mark.parametrize(
"deployment, expected",
[
(
{"litellm_params": {"model": "azure/my-gpt4-deploy"}, "model_info": {"base_model": "azure/gpt-4.1"}},
("azure/gpt-4.1", "azure/my-gpt4-deploy"),
),
(
{"litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4.1"}},
("azure/gpt-4.1", "azure/my-gpt4-deploy"),
),
(
{
"litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4o"},
"model_info": {"base_model": "azure/gpt-4.1"},
},
("azure/gpt-4.1", "azure/my-gpt4-deploy"),
),
({"litellm_params": {"model": "openai/gpt-4.1"}, "model_info": {}}, ("openai/gpt-4.1",)),
({"litellm_params": "not-a-dict", "model_info": None}, ()),
("not-a-deployment", ()),
],
ids=[
"model_info_base_model",
"params_base_model",
"model_info_wins",
"no_base_model",
"malformed_blocks",
"not_a_dict",
],
)
def test_deployment_pricing_models_reads_the_deployment_dict(deployment, expected):
assert deployment_pricing_models(deployment) == expected

24
tests/unit/test_cost_calculator.py
View file

@ -12,6 +12,7 @@ from pydantic import BaseModel
import litellm
from litellm.cost_calculator import (
batch_cost_calculator,
BaseTokenUsageProcessor,
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
@ -3223,6 +3224,29 @@ def _batch_cache_usage() -> Usage:
)
def test_batch_cost_calculator_vertex_regional_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None:
"""
Regression for https://github.com/BerriAI/litellm/issues/46880: a Vertex batch
on a regional location must cost the global batch times the model's
regional_endpoint_uplift_multiplier, the same invariant the online path keeps.
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000)
global_prompt, global_completion = batch_cost_calculator(
usage=usage, model="gemini-3.5-flash", custom_llm_provider="vertex_ai", vertex_location="global"
)
regional_prompt, regional_completion = batch_cost_calculator(
usage=usage, model="gemini-3.5-flash", custom_llm_provider="vertex_ai", vertex_location="us-central1"
)
uplift = litellm.get_model_info("vertex_ai/gemini-3.5-flash")["regional_endpoint_uplift_multiplier"]
assert global_prompt + global_completion > 0
assert regional_prompt == pytest.approx(global_prompt * uplift, rel=1e-9)
assert regional_completion == pytest.approx(global_completion * uplift, rel=1e-9)
def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates():
from litellm.cost_calculator import batch_cost_calculator

29
tests/unit/test_model_prices_schema.py
View file

@ -10,11 +10,13 @@ from typing import Final
import jsonschema
import pytest
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
from litellm.types.utils import SearchFreeRequests
REPO_ROOT = Path(__file__).parents[2]
GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py"
@ -591,3 +593,30 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
"unknown_root-new_family_models",
"vertex_ai-new_family_models",
]
def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]:
prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
return MappingProxyType(
{key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row}
)
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None:
rows = _search_free_requests_rows(path)
assert rows
allowances = {
key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items()
}
for key, row in rows.items():
assert row.get("supports_web_search") is True, key
assert row.get("search_context_cost_per_query"), key
assert allowances[key]["count"] > 0, key
for group in {allowance["shared_group"] for allowance in allowances.values()}:
same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group}
assert len(same_group) == 1, (group, same_group)
def test_search_free_requests_backup_rows_match_the_main_catalog() -> None:
assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH)

67
tests/unit/test_router/test_router.py
View file

@ -11833,7 +11833,7 @@ async def test_cyclic_fallback_graph_does_not_amplify_one_request():
_FLAT_ATTEMPT_RECORD_KEYS = frozenset(
{"model_group", "deployment_id", "exception_type", "exception_string", "attempted_retries"}
{"model_group", "deployment_id", "model", "base_model", "exception_type", "exception_string", "attempted_retries"}
)
_BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doNotShip"
@ -18143,3 +18143,68 @@ def test_bare_model_group_served_by_wildcard_deployment_has_provider_prefixed_co
assert router._has_content_policy_fallback("claude-sonnet-4-6", {}) is True
assert router._has_content_policy_fallback("claude-haiku-4-5", {}) is False
@pytest.mark.asyncio
async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens():
"""After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone."""
import json
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
def _relayed_iterator(text: str) -> ResponsesAPIStreamingIterator:
logging_obj: Final = MagicMock(spec=LiteLLMLogging)
logging_obj.model_call_details = {"litellm_params": {}}
logging_obj.completion_start_time = None
iterator: Final = ResponsesAPIStreamingIterator(
response=httpx.Response(200),
model="gpt-5.4",
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=logging_obj,
request_data={"model": "gpt-5.4", "input": "Hi"},
custom_llm_provider="openai",
)
created: Final = {
"id": "resp_1",
"object": "response",
"created_at": 1700000000.0,
"model": "gpt-5.4",
"status": "in_progress",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
}
iterator._process_chunk(json.dumps({"type": "response.created", "sequence_number": 0, "response": created}))
iterator._process_chunk(
json.dumps(
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_1",
"output_index": 0,
"content_index": 0,
"delta": text,
}
)
)
return iterator
router: Final = Router(
model_list=[{"model_name": "primary", "litellm_params": {"model": "openai/gpt-5.4", "api_key": "k"}}]
)
primary: Final = _relayed_iterator("Once upon a time there was a lighthouse keeper")
fallback: Final = _relayed_iterator("who lit the lamp")
wrapped: Final = await router._aresponses_streaming_iterator(
response=primary,
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": AsyncMock()},
)
primary_tokens: Final = primary.partial_response_on_disconnect().response.usage.output_tokens
fallback_tokens: Final = fallback.partial_response_on_disconnect().response.usage.output_tokens
assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens
wrapped.adopt_fallback_headers(fallback)
assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens

61
tests/unit/test_router/test_router_timeout.py Normal file
View file

@ -0,0 +1,61 @@
from typing import Final
import pytest
from litellm import Router
@pytest.mark.parametrize(
"request_kwargs, expected_timeout",
[
({"service_tier": "flex"}, 900.0),
({"service_tier": "default"}, 610.0),
({}, 610.0),
({"service_tier": "flex", "timeout": 20}, 20),
({"service_tier": "flex", "stream": True}, 950.0),
({"service_tier": "default", "stream": True}, 610.0),
({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30),
],
)
def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout(
request_kwargs: dict[str, object], expected_timeout: float
) -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 610,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout
def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0
assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0

66
tests/unit/test_utils.py
View file

@ -1068,16 +1068,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "object",
"additionalProperties": {"type": "number"},
},
"search_free_requests": {
"type": "object",
"properties": {
"count": {"type": "integer", "minimum": 1},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string"},
},
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
},
"search_context_cost_per_query": {
"type": "object",
"properties": {
@ -1091,6 +1081,16 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "string",
"enum": ["per_prompt", "per_query"],
},
"search_free_requests": {
"type": "object",
"properties": {
"count": {"type": "integer", "minimum": 1},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string"},
},
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
},
"citation_cost_per_token": {"type": "number"},
"supported_modalities": {
"type": "array",
@ -3187,9 +3187,7 @@ class TestExtraBodyCannotOverrideModel:
}
]
untouched = litellm.get_optional_params(
model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()
)
untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools())
assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched
result = litellm.get_optional_params(
@ -4581,7 +4579,13 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = (
pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"),
pytest.param(
ResponsesAPIResponse(
id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6"
id="resp_abc",
created_at=1,
output=[],
parallel_tool_calls=False,
tool_choice="auto",
tools=[],
model="gpt-5.6",
),
CallTypes.aresponses,
id="responses",
@ -4609,7 +4613,9 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo
assert result is response
assert second_hook.seen_responses == (response,)
failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message)
failure_logs: Final = tuple(
r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message
)
assert len(failure_logs) == 1
assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message
assert str(call_type) in failure_logs[0].message
@ -6448,3 +6454,33 @@ def test_registered_fields_reset_when_model_cost_is_replaced(monkeypatch: pytest
monkeypatch.setattr(litellm, "model_cost", {})
assert litellm.utils._registered_fields_for_current_map() == {}
def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body():
"""LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no
step that consumes it, so it must not ride along in extra_body and 400 at OpenAI."""
optional_params = litellm.utils.get_optional_params(
model="gpt-4.1",
custom_llm_provider="openai",
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}],
extra_body={"kept": True},
)
assert optional_params["extra_body"] == {"kept": True}
assert "cache_control_injection_points" not in optional_params
def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy():
"""A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy
applies cache_control_injection_points itself, so the point must ride along in extra_body."""
point: Final = {"location": "tool_config", "control": {"type": "ephemeral"}}
optional_params: Final[dict[str, object]] = litellm.utils.get_optional_params(
model="claude-sonnet-4-5",
custom_llm_provider="litellm_proxy",
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
cache_control_injection_points=[point],
extra_body={"kept": True},
)
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}

2
tests/unit/types/test_litellm_params.py
View file

@ -74,6 +74,8 @@ CONNECTION_NAMES: Final = (
"request_timeout",
"force_timeout",
"stream_timeout",
"timeout_by_service_tier",
"stream_timeout_by_service_tier",
"max_retries",
"tenant_id",
"client_id",

16
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -32938,6 +32938,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -32950,6 +32954,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */
@ -46554,6 +46562,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -46566,6 +46578,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */