scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

Merge stable/1.103.x (#47743, #47763) into litellm_backport_p0_batch_translation_cost_stable_1_103_x

Browse source 
# Conflicts:
#	litellm/model_prices_and_context_window_backup.json
#	litellm/proxy/hooks/proxy_track_cost_callback.py
#	litellm/responses/litellm_completion_transformation/streaming_iterator.py
#	model_prices_and_context_window.json
#	tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
#	tests/test_litellm/test_utils.py
#	uv.lock
This commit is contained in:
Yuneng Jiang 2026-10-21 00:55:49 -07:00
commit d7b606ac91
No known key found for this signature in database
69 changed files with 5252 additions and 305 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/__init__.py
View file

@ -555,6 +555,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
apply_search_free_tier: bool = False
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:

1
litellm/constants.py
View file

@ -1840,6 +1840,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"apply_search_free_tier",
"cost_margin_config",
"block_requests_for_models_without_pricing",
"budget_exceeded_throttle_percentage",

83
litellm/litellm_core_utils/litellm_logging.py
View file

@ -42,6 +42,7 @@ from litellm.constants import (
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
EMPTY_MAPPING,
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
PROVIDER_REQUEST_ID_HEADERS,
SENTRY_DENYLIST,
SENTRY_PII_DENYLIST,
@ -651,6 +652,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._defer_async_logging: bool = False
self._enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
self._async_success_enqueued: bool = False
self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
@ -1342,15 +1344,20 @@ class Logging(LiteLLMLoggingBaseClass):
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
)
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = self.model_call_details["api_call_start_time"]
# A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks
# without any provider handoff, so it must not stamp the handoff instants.
if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details:
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = self.model_call_details[
"api_call_start_time"
]
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
for callback in callbacks:
@ -1779,18 +1786,7 @@ class Logging(LiteLLMLoggingBaseClass):
router_model_id = self.get_router_model_id()
## RESPONSE COST ##
spilled_over: Final = is_spilled_over_ptu_request(
model_info=_deployment_model_info(self.litellm_params if hasattr(self, "litellm_params") else None),
response_headers=self.model_call_details.get("response_headers"),
additional_headers=result_additional_headers,
)
custom_pricing: Final = (
False
if spilled_over
else use_custom_pricing_for_model(
litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None)
)
)
custom_pricing: Final = self.custom_pricing_for(result)
prompt = self._prompt_for_cost_calculation()
@ -1861,6 +1857,24 @@ class Logging(LiteLLMLoggingBaseClass):
return None
def spilled_over_ptu_for(self, result: object) -> bool:
result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({})
additional_headers: Final = (
result_hidden_params.get("additional_headers")
if isinstance(result_hidden_params, dict)
else getattr(result_hidden_params, "additional_headers", None)
)
return is_spilled_over_ptu_request(
model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
response_headers=self.model_call_details.get("response_headers"),
additional_headers=additional_headers,
)
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False
return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
def _prompt_for_cost_calculation(self) -> str:
"""
The raw input string is only priced directly for text-to-speech, which bills per character.
@ -2066,6 +2080,16 @@ class Logging(LiteLLMLoggingBaseClass):
except Exception:
return True
@property
def async_success_log_enqueued(self) -> bool:
"""True once the success log task was actually created, not merely claimed, so a later failure raised
around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second
time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""
return self._async_success_enqueued
def mark_async_success_log_enqueued(self) -> None:
self._async_success_enqueued = True
def claim_async_success_log(self) -> bool:
"""One async success log per request. The innermost @client wrapper always exits first,
since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
@ -6349,13 +6373,16 @@ def get_standard_logging_object_payload(
)
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
# The router overrides completion_response.model to the model-group alias before
# this payload is built, so cost-map lookup via that alias always misses.
# Fall back to the actual deployment model set by the router in metadata.
if base_model is None:
base_model = metadata.get("deployment")
custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params)
configured_base_model: Final = _get_base_model_from_metadata(model_call_details=kwargs)
spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
# The router overrides completion_response.model to the model-group alias before this payload is
# built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
# A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
# which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
base_model: Final = (
metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
)
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))

60
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -942,6 +942,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
)
def _modality_token_cost(
model_info: ModelInfo,
modality_cost_key: str,
tokens: int,
prompt_base_cost: float,
service_tier: str | None,
) -> float:
"""
Audio, image and video prompt tokens bill at their modality rate when the model has one.
Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
`_get_token_base_cost` has already resolved for the service tier and long-context tier.
"""
tiered_key: Final = _get_service_tier_cost_key(modality_cost_key, service_tier)
cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
if model_info.get(cost_key) is None:
return float(tokens) * prompt_base_cost
return calculate_cost_component(model_info, cost_key, tokens)
def _calculate_input_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
@ -953,8 +972,14 @@ def _calculate_input_cost(
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
A model priced per character (`input_cost_per_character`) bills the reported character count
in place of its text tokens, so the same prompt is never charged under both units.
"""
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
bills_characters: Final = (
prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None
)
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
@ -972,28 +997,37 @@ def _calculate_input_cost(
if prompt_tokens_details["audio_tokens"] and not (
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
):
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_audio_token",
prompt_tokens_details["audio_tokens"],
prompt_base_cost,
service_tier,
)
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"] and not (
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
):
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
if model_info.get(image_token_cost_key) is None:
image_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_image_token",
prompt_tokens_details["image_tokens"],
prompt_base_cost,
service_tier,
)
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"] and not (
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
):
video_token_cost_key = "input_cost_per_video_token"
if model_info.get(video_token_cost_key) is None:
video_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_video_token",
prompt_tokens_details["video_tokens"],
prompt_base_cost,
service_tier,
)
### CACHE WRITING COST - Now uses tiered pricing
if (

29
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -4,6 +4,7 @@ from functools import reduce
from typing import Any, Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
@ -101,6 +102,32 @@ def response_timing_metrics(
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
try:
return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
except ValidationError:
return {}
def _existing_provider_response_model(result: object) -> str | None:
"""The pricing model a response already carries, like the decisions path sets, wins over the response model."""
hidden_params: Final[object] = getattr(result, "_hidden_params", None)
carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
hidden_params, "provider_response_model", None
)
return carried if isinstance(carried, str) and carried else None
def _provider_response_model(result: object) -> str | None:
"""The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
return model if isinstance(model, str) and model else None
class ResponseMetadata:
"""
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@ -133,6 +160,8 @@ class ResponseMetadata:
preserve_litellm_internal_headers=True,
),
"litellm_model_name": model,
"provider_response_model": _existing_provider_response_model(self.result)
or _provider_response_model(self.result),
}
self._update_hidden_params(new_params)

23
litellm/litellm_core_utils/request_timeout_resolver.py
View file

@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
from __future__ import annotations
from typing import Final
from collections.abc import Mapping
from typing import Final, Literal, Protocol
from pydantic import TypeAdapter
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
def get_configured_request_timeout() -> float | None:
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
return timeout
return None
class _DeploymentParams(Protocol):
def get(self, key: str, /) -> object: ...
def get_service_tier_timeout(
service_tier: object,
litellm_params: _DeploymentParams,
key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
) -> float | None:
"""Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
if not isinstance(service_tier, str):
return None
timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
return None if timeouts is None else timeouts.get(service_tier)

15
litellm/llms/anthropic/chat/transformation.py
View file

@ -97,6 +97,7 @@ from ..common_utils import (
eager_input_streaming_flag,
process_anthropic_headers,
strip_advisor_blocks_from_messages,
tools_marked_for_a_tool_config_point,
)
if TYPE_CHECKING:
@ -2026,6 +2027,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Remove internal LiteLLM parameters that should not be sent to Anthropic API
optional_params.pop("is_vertex_request", None)
optional_params.pop("client_metadata", None)
request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature
dict[str, object], optional_params
)
marked_tools: Final = tools_marked_for_a_tool_config_point(
request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
)
if marked_tools is not None:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature
Mapping[str, object], litellm_params
)
AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1)
# ``top_k`` is a provider-specific kwarg that bypasses
# ``map_openai_params``; gate it here, the single boundary shared by
@ -2045,6 +2059,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"model": model,
"messages": anthropic_messages,
**optional_params,
**({} if marked_tools is None else {"tools": list(marked_tools)}),
}
self._apply_output_config(data=data, model=model, optional_params=optional_params)

49
litellm/llms/anthropic/common_utils.py
View file

@ -33,6 +33,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_OAUTH_BETA_HEADER,
ANTHROPIC_OAUTH_TOKEN_PREFIX,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesToolChoice,
@ -71,7 +72,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
_DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@ -92,22 +93,56 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None
)
def _tool_config_point_control(points: object) -> Mapping[object, object] | None:
point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ())
tool_config_point: Final = next(
(fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None
)
if tool_config_point is None:
return None
control: Final = tool_config_point.get("control")
return {"type": "ephemeral"} if control is None else validated_object_mapping(control)
def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None:
"""Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches.
Anthropic caches tools as one prefix ending at the marked tool, so the last one carries
the mark, skipping tool-search tools whose transform drops cache markers. A client that
already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked.
"""
control: Final = _tool_config_point_control(points)
tool_list: Final = validated_object_list(tools) or ()
tool_fields: Final = tuple(validated_object_mapping(tool) for tool in tool_list)
if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields):
return None
cacheable: Final = tuple(
(position, fields)
for position, fields in enumerate(tool_fields)
if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
)
if not cacheable:
return None
position, fields = cacheable[-1]
return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :])
def is_claude_code_user_agent(user_agent: str) -> bool:
"""Claude Code sends its API calls through the Anthropic SDK as `claude-cli/<version>` and its own
fetches, such as gateway model discovery, as `claude-code/<version>`"""
return user_agent.startswith(_CLAUDE_CODE_USER_AGENT_PREFIXES)
def _validated_claude_code_mapping(value: object) -> dict[object, object] | None:
def validated_object_mapping(value: object) -> dict[object, object] | None:
try:
return _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER.validate_python(value)
except ValidationError:
return None
def _validated_claude_code_list(value: object) -> list[object] | None:
def validated_object_list(value: object) -> Sequence[object] | None:
try:
return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value)
return OBJECT_LIST_ADAPTER.validate_python(value)
except ValidationError:
return None
@ -134,10 +169,10 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None
def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None:
if isinstance(system, str):
return (system,)
blocks: Final = _validated_claude_code_list(system)
blocks: Final = validated_object_list(system)
if blocks is None:
return None
block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks)
block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks)
if any(block is None for block in block_mappings):
return None
text_values: Final = tuple(
@ -170,7 +205,7 @@ def is_claude_code_one_shot_subagent_request(
tools: object,
user_agent: str | None,
) -> bool:
only_message: Final = _validated_claude_code_mapping(messages[0]) if len(messages) == 1 else None
only_message: Final = validated_object_mapping(messages[0]) if len(messages) == 1 else None
return (
user_agent is not None
and is_claude_code_user_agent(user_agent)

10
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,7 @@ from functools import partial
from typing import Any, Final, cast
import litellm
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
@ -19,6 +20,7 @@ from litellm.llms.anthropic.common_utils import (
sanitize_tool_use_ids_in_anthropic_messages,
strip_empty_content_blocks_from_anthropic_messages,
strip_provider_specific_fields_from_anthropic_messages,
tools_marked_for_a_tool_config_point,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
@ -634,7 +636,13 @@ def anthropic_messages_handler(
f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}"
)
local_vars.update(kwargs)
request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped
marked_tools: Final = tools_marked_for_a_tool_config_point(
local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None)
)
if marked_tools is not None:
AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1)
local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})})
anthropic_messages_optional_request_params: Final = (
AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
params=local_vars,

39
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -4,7 +4,7 @@ import asyncio
import json
import traceback
from collections import deque
from collections.abc import AsyncIterator, Iterator, Mapping
from collections.abc import AsyncGenerator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
from litellm import verbose_logger
@ -23,8 +23,11 @@ from .transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.types.llms.openai import ResponseIncompleteEvent
class AnthropicResponsesStreamWrapper:
@ -391,8 +394,11 @@ class AnthropicResponsesStreamWrapper:
raise StopAsyncIteration
async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]:
"""Yield SSE-encoded bytes for each Anthropic event chunk."""
def async_anthropic_sse_wrapper(self) -> "AnthropicResponsesSSEByteStream":
"""SSE-encoded bytes for each Anthropic event chunk."""
return AnthropicResponsesSSEByteStream(self)
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
async for chunk in self:
if isinstance(chunk, dict):
event_type: str = str(chunk.get("type", "message"))
@ -400,3 +406,30 @@ class AnthropicResponsesStreamWrapper:
yield payload.encode()
else:
yield chunk
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
return partial_responses_incomplete_event_on_disconnect(self.responses_stream)
class AnthropicResponsesSSEByteStream:
"""
The byte stream the proxy relays for a default-route /v1/messages request.
Keeps the Responses API iterator reachable so a client disconnect can bill
the partial output the way native /v1/responses streams do.
"""
def __init__(self, wrapper: AnthropicResponsesStreamWrapper) -> None:
self._wrapper: Final = wrapper
self._frames: Final = wrapper._sse_frames() # pyright: ignore[reportPrivateUsage] # the wrapper's own byte stream
def __aiter__(self) -> "AnthropicResponsesSSEByteStream":
return self
async def __anext__(self) -> bytes:
return await self._frames.__anext__()
async def aclose(self) -> None:
await self._frames.aclose()
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
return self._wrapper.partial_response_on_disconnect()

186
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
View file

@ -2,27 +2,102 @@
Bedrock Token Counter implementation using the CountTokens API.
"""
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any, Final
from pydantic import JsonValue
from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.types.utils import LlmProviders, TokenCountResponse
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import LiteLLMPydanticObjectBase, LlmProviders, TokenCountResponse
RUNTIME_TOKENIZER_TYPE: Final = "bedrock_api"
MANTLE_TOKENIZER_TYPE: Final = "bedrock_mantle_api"
_RUNTIME_FALLBACK_STATUSES: Final = frozenset({400, 404})
class _CountTokensReply(LiteLLMPydanticObjectBase):
input_tokens: int
@dataclass(frozen=True, slots=True)
class CountedTokens:
input_tokens: int
original_response: Mapping[str, JsonValue]
tokenizer_type: str
@dataclass(frozen=True, slots=True)
class CountTokensFailure:
status_code: int
message: str
tokenizer_type: str
CountTokensOutcome = CountedTokens | CountTokensFailure
def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool:
return (
isinstance(outcome, CountTokensFailure)
and outcome.status_code in _RUNTIME_FALLBACK_STATUSES
and is_mantle_claude_model(resolved_model)
)
class BedrockTokenCounter(BaseTokenCounter):
"""Token counter implementation for AWS Bedrock provider using the CountTokens API."""
"""Token counter for AWS Bedrock: bedrock-runtime CountTokens first, and Anthropic's count_tokens
on bedrock-mantle for Claude models bedrock-runtime answers 400 or 404 for"""
def __init__(
self,
runtime_handler: BedrockCountTokensHandler | None = None,
mantle_handler: BedrockMantleCountTokensHandler | None = None,
client: AsyncHTTPHandler | None = None,
) -> None:
self._runtime_handler: Final = runtime_handler or BedrockCountTokensHandler()
self._mantle_handler: Final = mantle_handler or BedrockMantleCountTokensHandler()
self._client: Final = client
def should_use_token_counting_api(
self,
custom_llm_provider: str | None = None,
) -> bool:
"""
Returns True if we should use the Bedrock CountTokens API for token counting.
"""
return custom_llm_provider == LlmProviders.BEDROCK.value
async def _count_with(
self,
handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler,
tokenizer_type: str,
request_data: dict[str, object],
litellm_params: dict[str, object],
resolved_model: str,
) -> CountTokensOutcome:
try:
result: Final = await handler.handle_count_tokens_request(
request_data=request_data,
litellm_params=litellm_params,
resolved_model=resolved_model,
client=self._client,
)
reply: Final = _CountTokensReply.model_validate(result)
except BedrockError as e:
verbose_logger.debug(
"%s CountTokens API error: status=%s, message=%s", tokenizer_type, e.status_code, e.message
)
return CountTokensFailure(status_code=e.status_code, message=e.message, tokenizer_type=tokenizer_type)
except Exception as e:
verbose_logger.debug("Error calling %s CountTokens API: %s", tokenizer_type, e)
return CountTokensFailure(status_code=500, message=str(e), tokenizer_type=tokenizer_type)
return CountedTokens(input_tokens=reply.input_tokens, original_response=result, tokenizer_type=tokenizer_type)
async def count_tokens(
self,
model_to_use: str,
@ -33,81 +108,52 @@ class BedrockTokenCounter(BaseTokenCounter):
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using AWS Bedrock's CountTokens API.
This method calls the existing BedrockCountTokensHandler to make an API call
to Bedrock's token counting endpoint, bypassing the local tiktoken-based counting.
Args:
model_to_use: The model identifier
messages: The messages to count tokens for
contents: Alternative content format (not used for Bedrock)
deployment: Deployment configuration containing litellm_params
request_model: The original request model name
Returns:
TokenCountResponse with token count, or None if counting fails
"""
if not messages:
return None
deployment = deployment or {}
litellm_params: Final = deployment.get("litellm_params", {})
# Build request data in the format expected by BedrockCountTokensHandler
request_data: Final[dict[str, Any]] = {
litellm_params: Final = (deployment or {}).get("litellm_params", {})
request_data: Final[dict[str, object]] = {
"model": model_to_use,
"messages": messages,
**({"tools": tools} if tools else {}),
**({"system": system} if system else {}),
}
if tools:
request_data["tools"] = tools
if system:
request_data["system"] = system
# Get the resolved model (strip prefixes like bedrock/, converse/, etc.)
resolved_model: Final = get_bedrock_base_model(model_to_use)
try:
handler: Final = BedrockCountTokensHandler()
result: Final = await handler.handle_count_tokens_request(
request_data=request_data,
litellm_params=litellm_params,
resolved_model=resolved_model,
runtime: Final = await self._count_with(
self._runtime_handler, RUNTIME_TOKENIZER_TYPE, request_data, litellm_params, resolved_model
)
outcome: Final = (
await self._count_with(
self._mantle_handler, MANTLE_TOKENIZER_TYPE, request_data, litellm_params, resolved_model
)
# Transform response to TokenCountResponse
if result is not None:
if _runtime_rejected_claude_model(runtime, resolved_model)
else runtime
)
match outcome:
case CountedTokens():
return TokenCountResponse(
total_tokens=result.get("input_tokens", 0),
total_tokens=outcome.input_tokens,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="bedrock_api",
original_response=result,
tokenizer_type=outcome.tokenizer_type,
original_response=dict(outcome.original_response),
)
except BedrockError as e:
verbose_logger.warning("Bedrock CountTokens API error: status=%s, message=%s", e.status_code, e.message)
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="bedrock_api",
error=True,
error_message=e.message,
status_code=e.status_code,
)
except Exception as e:
verbose_logger.warning("Error calling Bedrock CountTokens API: %s", e)
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="bedrock_api",
error=True,
error_message=str(e),
status_code=500,
)
return None
case CountTokensFailure():
verbose_logger.warning(
"%s CountTokens API error: status=%s, message=%s",
outcome.tokenizer_type,
outcome.status_code,
outcome.message,
)
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type=outcome.tokenizer_type,
error=True,
error_message=outcome.message,
status_code=outcome.status_code,
)
case _:
assert_never(outcome)

2
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -125,7 +125,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
raise
except httpx.HTTPStatusError as e:
# HTTP errors - preserve the actual status code
verbose_logger.error("HTTP error in CountTokens handler: %s", e)
verbose_logger.debug("HTTP error in CountTokens handler: %s", e)
raise BedrockError(
status_code=e.response.status_code,
message=e.response.text,

99
litellm/llms/bedrock/count_tokens/mantle_handler.py Normal file
View file

@ -0,0 +1,99 @@
from typing import Final
import httpx
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountTokensConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
MANTLE_COUNT_TOKENS_SUFFIX: Final = "/count_tokens"
MANTLE_ANTHROPIC_VERSION: Final = "2023-06-01"
class MantleCountTokensRequest(TypedDict):
messages: ReadOnly[list[dict[str, JsonValue]]]
system: ReadOnly[NotRequired[JsonValue]]
tools: ReadOnly[NotRequired[list[dict[str, JsonValue]]]]
_COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
class BedrockMantleCountTokensHandler(BaseAWSLLM):
"""Counts tokens through Anthropic's count_tokens on the bedrock-mantle endpoint.
Claude models that Bedrock offers only through cross-region inference answer 400 on
bedrock-runtime's CountTokens; AWS documents Mantle's /anthropic/v1/messages/count_tokens
as the way to count them, with the base model id and the deployment's AWS credentials
"""
def __init__(self, anthropic_config: AnthropicCountTokensConfig | None = None) -> None:
super().__init__()
self._anthropic_config: Final = anthropic_config or AnthropicCountTokensConfig()
def get_mantle_count_tokens_endpoint(self, aws_region_name: str) -> str:
messages_url: Final = build_mantle_messages_url(
api_base=None, aws_bedrock_runtime_endpoint=None, region=aws_region_name
)
return f"{messages_url}{MANTLE_COUNT_TOKENS_SUFFIX}"
async def handle_count_tokens_request(
self,
request_data: dict[str, object],
litellm_params: dict[str, object],
resolved_model: str,
client: AsyncHTTPHandler | None = None,
) -> dict[str, JsonValue]:
try:
request: Final = _COUNT_REQUEST.validate_python(request_data)
aws_region_name: Final = self._get_aws_region_name(
optional_params=litellm_params, model=resolved_model, model_id=None
)
endpoint_url: Final = self.get_mantle_count_tokens_endpoint(aws_region_name)
body: Final = self._anthropic_config.transform_request_to_count_tokens(
model=resolved_model,
messages=request["messages"],
tools=request.get("tools"),
system=request.get("system"),
)
verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
api_key: Final = litellm_params.get("api_key")
signed_headers, signed_body = await run_aws_signing(
self._sign_request,
service_name="bedrock",
headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
optional_params=litellm_params,
request_data=body,
api_base=endpoint_url,
model=resolved_model,
api_key=api_key if isinstance(api_key, str) else None,
)
async_client: Final = client or get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK)
response: Final = await async_client.post(
endpoint_url, headers=signed_headers, data=signed_body, timeout=30.0
)
if response.status_code != 200:
raise BedrockError(
status_code=response.status_code,
message=response.text,
headers=response.headers,
response=response,
)
return _COUNT_RESPONSE.validate_json(response.content)
except BedrockError:
raise
except httpx.HTTPStatusError as e:
raise BedrockError(
status_code=e.response.status_code,
message=e.response.text,
headers=e.response.headers,
response=e.response,
)
except Exception as e:
raise BedrockError(status_code=500, message=f"bedrock-mantle count_tokens processing error: {e}")

4
litellm/llms/bedrock_mantle/common_utils.py
View file

@ -45,6 +45,10 @@ def split_mantle_region_prefix(model: str) -> tuple[str | None, str]:
return None, model
def is_mantle_claude_model(model: str) -> bool:
return "claude" in model.lower()
def resolve_mantle_region(params: Mapping[str, object]) -> str:
region: Final = params.get("aws_region_name")
if isinstance(region, str) and region:

34
litellm/llms/vertex_ai/vertex_embeddings/transformation.py
View file

@ -1,14 +1,34 @@
import types
from typing import Final, Literal
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
from litellm.types.utils import EmbeddingResponse, Usage
from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage
from .types import *
class _PredictMetadata(TypedDict, total=False):
billableCharacterCount: ReadOnly[int]
class _PredictResponseMetadata(TypedDict, total=False):
metadata: ReadOnly[_PredictMetadata]
_PREDICT_RESPONSE_METADATA: Final = TypeAdapter(_PredictResponseMetadata)
def billable_character_count(response: _PredictResponseMetadata) -> int | None:
"""
`metadata.billableCharacterCount` of a Vertex text embedding predict response, the count Google
bills per-character embedding models (text-embedding-005, text-multilingual-embedding-002) on.
"""
return response.get("metadata", {}).get("billableCharacterCount")
class VertexAITextEmbeddingConfig(BaseModel):
"""
Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#TextEmbeddingInput
@ -248,7 +268,15 @@ class VertexAITextEmbeddingConfig(BaseModel):
model_response.object = "list"
model_response.data = embedding_response
model_response.model = model
usage: Final = Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens)
billable_characters: Final = billable_character_count(_PREDICT_RESPONSE_METADATA.validate_python(response))
usage: Final = Usage(
prompt_tokens=input_tokens,
completion_tokens=0,
total_tokens=input_tokens,
prompt_tokens_details=(
None if billable_characters is None else PromptTokensDetailsWrapper(character_count=billable_characters)
),
)
setattr(model_response, "usage", usage)
return model_response

22
litellm/passthrough/timeout_utils.py
View file

@ -5,6 +5,11 @@ from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
_SECONDS: Final = TypeAdapter(float)
@ -48,21 +53,27 @@ def resolve_llm_passthrough_timeout(
Anthropic /v1/messages).
Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout
-> 600s default.
timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout,
when explicitly set) -> general_settings.pass_through_request_timeout -> 600s default.
Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the
non-streaming chain above.
kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] ->
litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above.
Only the first set value is validated as seconds, so a value in a lower-precedence
field never fails the call.
"""
request: Final = kwargs if kwargs is not None else _NO_PARAMS
deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
service_tier: Final = request.get("service_tier")
stream_candidates: Final = (
(request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout)
(
request.get("stream_timeout"),
get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"),
deployment.get("stream_timeout"),
router_stream_timeout,
)
if request.get("stream")
else ()
)
@ -70,6 +81,7 @@ def resolve_llm_passthrough_timeout(
*stream_candidates,
request.get("timeout"),
request.get("request_timeout"),
get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"),
deployment.get("timeout"),
deployment.get("request_timeout"),
router_timeout,

12
litellm/proxy/common_request_processing.py
View file

@ -95,6 +95,7 @@ from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
from litellm.router import Router
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.common_utils import resolve_model_group_alias
@ -396,6 +397,13 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
# A natural end-of-stream success event already fired and released the
# slot; do not bill again, and let the caller skip the slot release.
return True
partial_responses_event: Final = partial_responses_incomplete_event_on_disconnect(response)
if partial_responses_event is not None:
verbose_proxy_logger.debug(
"Billing partial streamed Responses API spend after client disconnect, litellm_call_id=%s",
logging_obj.litellm_call_id,
)
return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event)
chunks: Final[object] = getattr(response, "chunks", None)
if not isinstance(chunks, list) or not chunks:
return False
@ -425,6 +433,10 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
partial_usage: Final = getattr(partial_response, "usage", None)
if isinstance(partial_usage, Usage):
backfill_missing_cache_usage_fields(partial_usage)
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
try:
await logging_obj.dispatch_success_handlers(
partial_response,

27
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -115,6 +115,23 @@ def coerce_numeric_form_fields(
}
def _parse_form_metadata(raw_metadata: str) -> object:
"""The JSON body path rejects a lone surrogate escape (``"\\ud800"``) with a
400 because no provider or database can encode it. ``json.loads`` accepts
the escape, so the multipart ``metadata`` field gets the same check."""
parsed: Final[object] = json.loads(raw_metadata)
try:
json.dumps(parsed, ensure_ascii=False).encode("utf-8")
except UnicodeEncodeError as e:
raise ProxyException(
message=f"Invalid form payload: metadata {e}",
type="invalid_request_error",
param="metadata",
code=status.HTTP_400_BAD_REQUEST,
) from e
return parsed
async def _read_request_body(request: Request | None) -> dict:
"""
Safely read the request body and parse it as JSON.
@ -154,9 +171,13 @@ async def _read_request_body(request: Request | None) -> dict:
param="request_body",
code=status.HTTP_400_BAD_REQUEST,
)
parsed_body = dict(form_data)
if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str):
parsed_body["metadata"] = json.loads(parsed_body["metadata"])
form_body: Final = dict(form_data)
raw_metadata: Final = form_body.get("metadata")
parsed_body = (
{**form_body, "metadata": _parse_form_metadata(raw_metadata)}
if isinstance(raw_metadata, str)
else form_body
)
else:
# Read the request body
body: Final = await request.body()

48
litellm/proxy/db/db_spend_update_writer.py
View file

@ -87,6 +87,7 @@ if TYPE_CHECKING:
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
else:
PrismaClient = Any
ProxyLogging = Any
@ -96,6 +97,34 @@ RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, Call
_SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _turn_chains_through_the_db(payload: Mapping[str, object], llm_router: "Router | None") -> bool:
"""A Responses turn the chat-completions bridge replays from its spend log on the next `previous_response_id`.
Native Responses providers hold the conversation themselves, so their rows can wait for the batch write.
A row with no router deployment behind it is treated as bridged, which keeps today's immediate flush.
"""
if payload.get("call_type") not in RESPONSES_SESSION_CALL_TYPES:
return False
model_id: Final = payload.get("model_id")
deployment: Final = (
llm_router.get_deployment(model_id=model_id) if llm_router is not None and isinstance(model_id, str) else None
)
if deployment is None:
return True
from litellm.responses.main import provider_stores_the_conversation
litellm_params: Final = deployment.litellm_params
return not provider_stores_the_conversation(
litellm_params.model,
litellm_params.custom_llm_provider,
{
"use_chat_completions_api": litellm_params.use_chat_completions_api,
"model_info": deployment.model_info.model_dump(),
"api_base": litellm_params.api_base,
},
)
def _org_member_transaction_key(org_id: str, user_id: str) -> str:
return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
@ -376,12 +405,13 @@ class DBSpendUpdateWriter:
get_request_model_access_groups,
)
llm_router: Final = get_llm_router()
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=completion_response,
start_time=start_time,
end_time=end_time,
llm_router=get_llm_router(),
llm_router=llm_router,
)
payload["spend"] = response_cost or 0.0
if isinstance(payload["startTime"], datetime):
@ -396,7 +426,10 @@ class DBSpendUpdateWriter:
payload["team_id"] = team_id
if not await self._record_spend_log(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
payload=payload,
prisma_client=prisma_client,
disable_spend_logs=disable_spend_logs,
llm_router=llm_router,
):
return False
@ -457,14 +490,18 @@ class DBSpendUpdateWriter:
return True
async def _record_spend_log(
self, payload: SpendLogsPayload, prisma_client: "PrismaClient | None", disable_spend_logs: bool
self,
payload: SpendLogsPayload,
prisma_client: "PrismaClient | None",
disable_spend_logs: bool,
llm_router: "Router | None" = None,
) -> bool:
if prisma_client is not None and _is_batch_cost_row(payload):
return await self._claim_batch_cost_spend_log(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
)
if disable_spend_logs is False:
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client)
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router)
return True
async def _claim_batch_cost_spend_log(
@ -1311,6 +1348,7 @@ class DBSpendUpdateWriter:
payload: dict | SpendLogsPayload,
prisma_client: PrismaClient | None = None,
spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"),
llm_router: "Router | None" = None,
) -> PrismaClient | None:
verbose_proxy_logger.debug(
"Writing spend log to db - request_id: {}, spend: {}".format(
@ -1321,7 +1359,7 @@ class DBSpendUpdateWriter:
from litellm.proxy.utils import enqueue_spend_logs, request_spend_log_flush
await enqueue_spend_logs(prisma_client, (payload,))
if payload.get("call_type") in RESPONSES_SESSION_CALL_TYPES:
if _turn_chains_through_the_db(_SPEND_METADATA_ADAPTER.validate_python(payload), llm_router):
request_spend_log_flush(prisma_client)
else:
verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.")

24
litellm/proxy/google_endpoints/endpoints.py
View file

@ -188,26 +188,12 @@ async def google_count_tokens(request: Request, model_name: str):
request=token_request,
call_endpoint=True,
)
if token_response is not None:
# cast the response to the well known format
original_response: Final[dict] = token_response.original_response or {}
if original_response:
return TokenCountDetailsResponse(
totalTokens=original_response.get("totalTokens", 0),
promptTokensDetails=original_response.get("promptTokensDetails", []),
)
else:
return TokenCountDetailsResponse(
totalTokens=token_response.total_tokens or 0,
promptTokensDetails=[],
)
#########################################################
# Return the response in the well known format
#########################################################
if token_response is None:
return TokenCountDetailsResponse(totalTokens=0, promptTokensDetails=[])
original_response: Final[dict] = token_response.original_response or {}
return TokenCountDetailsResponse(
totalTokens=0,
promptTokensDetails=[],
totalTokens=original_response.get("totalTokens") or token_response.total_tokens or 0,
promptTokensDetails=original_response.get("promptTokensDetails", []),
)

133
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -3,7 +3,7 @@ import traceback
from collections.abc import Callable, Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import TypeAdapter
@ -31,6 +31,7 @@ from litellm.proxy.db.db_spend_update_writer import (
debitable_model_access_groups,
get_llm_router,
)
from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.spend_tracking.spend_event import (
ObjectMapping,
@ -51,7 +52,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
get_request_model_access_groups,
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend
from litellm.types.utils import (
CallTypes,
LiteLLMBatch,
@ -124,6 +125,77 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
return proxy_logging_obj.db_spend_update_writer
def _str_keyed_fields(value: object) -> Mapping[str, object]:
return value if isinstance(value, Mapping) and all(isinstance(key, str) for key in value) else {}
def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
attempts: Final = request_data.get("billed_failed_attempts")
if not isinstance(attempts, tuple):
return ()
lifted: Final = cast("tuple[object, ...]", attempts) # cast-ok: lifted by _failure_fields_to_lift, checked below
return tuple(attempt for attempt in lifted if isinstance(attempt, BilledFailedAttempt))
def _failure_spend_rows(
request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object
) -> tuple[tuple[float, Mapping[str, object]], ...]:
"""One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the
request id, then every earlier attempt the provider also billed under its own row."""
final_row: Final[Mapping[str, object]] = (
request_data
if late_guardrails_snapshot is None
else {**request_data, "standard_logging_object": late_guardrails_snapshot}
)
return (
(final_cost, final_row),
*(
(attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))
for position, attempt in enumerate(_billed_failed_attempts(request_data))
),
)
def _attempt_spend_row_kwargs(
request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int
) -> Mapping[str, object]:
"""The final attempt's row kwargs re-keyed for one earlier billed attempt: its own
``request_id`` (``<id>-attempt-<position>``, since ``request_id`` is the spend-log primary key
and the retry count restarts after a fallback hop), its estimated usage and cost, its retry
count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""
suffix: Final = f"-attempt-{position}"
raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict
litellm_params: Final = dict(raw_params)
metadata: Final = {
**cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict
"attempted_retries": attempt.attempted_retries,
}
metadata.pop("previous_models", None)
if attempt.deployment_id is not None:
metadata["model_info"] = {
**cast("Mapping[str, object]", metadata.get("model_info") or {}), # cast-ok: router dict
"id": attempt.deployment_id,
}
litellm_params["metadata"] = metadata
row: Final = {
**request_data,
"litellm_params": litellm_params,
"combined_usage_object": attempt.usage,
"response_cost": attempt.response_cost,
}
row.pop("billed_failed_attempts", None)
call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
if isinstance(call_id, str):
row["litellm_call_id"] = f"{call_id}{suffix}"
standard_logging_object: Final = request_data.get("standard_logging_object")
if isinstance(standard_logging_object, dict):
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
if isinstance(payload.get("id"), str):
row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}
return row
class _ProxyDBLogger(CustomLogger):
def __init__(
self,
@ -291,20 +363,40 @@ class _ProxyDBLogger(CustomLogger):
existing_metadata.get("standard_logging_guardrail_information")
)
await self._spend_writer().update_database(
token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
response_cost=recovered_response_cost,
user_id=user_api_key_dict.user_id,
end_user_id=user_api_key_dict.end_user_id,
team_id=user_api_key_dict.team_id,
kwargs=request_data,
completion_response=original_exception,
await self._write_failure_spend_rows(
user_api_key_dict=user_api_key_dict,
original_exception=original_exception,
start_time=actual_start_time,
end_time=datetime.now(),
org_id=user_api_key_dict.org_id,
project_id=user_api_key_dict.project_id,
spend_rows=_failure_spend_rows(
cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below
recovered_response_cost,
None,
),
)
async def _write_failure_spend_rows(
self,
user_api_key_dict: UserAPIKeyAuth,
original_exception: Exception,
start_time: datetime,
spend_rows: Sequence[tuple[float, Mapping[str, object]]],
) -> None:
"""One spend row (and one key/team/user spend deduction) per ``(response_cost, kwargs)`` pair."""
for row_cost, row_kwargs in spend_rows:
await self._spend_writer().update_database(
token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
response_cost=row_cost,
user_id=user_api_key_dict.user_id,
end_user_id=user_api_key_dict.end_user_id,
team_id=user_api_key_dict.team_id,
kwargs=dict(row_kwargs),
completion_response=original_exception,
start_time=start_time,
end_time=datetime.now(),
org_id=user_api_key_dict.org_id,
project_id=user_api_key_dict.project_id,
)
@log_db_metrics
async def _PROXY_track_cost_callback(
self,
@ -369,8 +461,21 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
raw_cost: Final = _str_keyed_fields(
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
).get("response_cost")
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
response_cost: Final = (
sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
await apply_search_free_tier(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists
response_cost=priced_cost,
completion_response=completion_response,
counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache,
now=datetime.now(timezone.utc),
)
if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None
else priced_cost
)
tags: Final = _get_request_tags_for_cost_tracking(
sl_object=sl_object,

123
litellm/proxy/hooks/search_free_tier.py Normal file
View file

@ -0,0 +1,123 @@
from collections.abc import MutableMapping
from datetime import datetime
from types import MappingProxyType
from typing import Final, Protocol
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage
_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"})
_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"})
_TTL_GRACE_SECONDS: Final = 3600
_COUNTED_KWARG: Final = "search_free_tier_counted"
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
class SharedCounter(Protocol):
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ...
class _UsageCarrier(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
usage: Usage | None = None
def freed_units(allowance: int, units: int, total_after: int) -> int:
"""Units of this request that landed under the allowance, given the shared total after they were added."""
return max(0, min(units, allowance - (total_after - units)))
def search_free_tier_key(provider: str, allowance: SearchFreeRequests, now: datetime) -> str:
period_stamp: Final = now.strftime(_PERIOD_STAMP[allowance["period"]])
return f"litellm:search_free_tier:{provider}:{allowance['shared_group']}:{period_stamp}"
def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int:
reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC")
return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS
def _usage_of(completion_response: object) -> Usage | None:
try:
return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage
except ValidationError:
return None
def _grounding_queries(usage: Usage) -> int:
details: Final = usage.prompt_tokens_details
from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None
return from_details or get_web_search_requests_from_usage(usage) or 0
def _billable_units(usage: Usage, model_info: ModelInfo) -> int:
queries: Final = _grounding_queries(usage)
if queries == 0:
return 0
return queries if model_info.get("web_search_billing_unit") == "per_query" else 1
def _record_freed_cost(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
original_cost: float,
freed_cost: float,
) -> float:
reduced_cost: Final = original_cost - freed_cost
breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost)
sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload
{
**breakdown,
"total_cost": reduced_cost,
"original_cost": breakdown.get("original_cost", original_cost),
"discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost,
}
)
sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload
kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"]
return reduced_cost
async def apply_search_free_tier(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
response_cost: float,
completion_response: object,
counter: SharedCounter | None,
now: datetime,
) -> float:
"""Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged."""
model_info: Final = sl_object["model_map_information"]["model_map_value"]
allowance: Final = model_info.get("search_free_requests") if model_info is not None else None
usage: Final = _usage_of(completion_response)
if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]:
return response_cost
units: Final = _billable_units(usage, model_info)
if units == 0 or kwargs.get(_COUNTED_KWARG):
return response_cost
if counter is None:
verbose_proxy_logger.warning(
"apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price"
)
return response_cost
provider: Final = sl_object["custom_llm_provider"] or model_info["litellm_provider"]
key: Final = search_free_tier_key(provider, allowance, now)
try:
total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now))
except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row
verbose_proxy_logger.warning(
"search free tier counter %s is unavailable, grounding is billed at full price", key
)
return response_cost
kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once
freed: Final = freed_units(allowance["count"], units, int(total_after))
if freed == 0:
return response_cost
freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units)
return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost)

6
litellm/proxy/litellm_pre_call_utils.py
View file

@ -358,8 +358,9 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess
# ``litellm.model_cost`` map. Both effects belong to deployment configuration,
# not to user-supplied request bodies, so the proxy strips them before they
# reach the call path. Built from the Pydantic model so newly-added pricing
# fields are covered automatically.
_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
# fields are covered automatically. ``base_model`` selects which cost-map entry
# prices the row, so a caller naming a cheaper model lowers their own spend.
_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"}
# ``model_info`` carries the same pricing fields when read by
# ``use_custom_pricing_for_model``; strip from metadata for the same reason.
# ``standard_logging_guardrail_information`` is proxy-written telemetry summed
@ -373,6 +374,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
{
"attempted_fallbacks",
"original_model_group",
"previous_models",
"request_retry_count",
CLIENT_OUTPUT_CEILING_METADATA_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,

306
litellm/proxy/utils.py
View file

@ -44,11 +44,13 @@ from typing import (
overload,
)
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm import _custom_logger_compatible_callbacks_literal
from litellm.constants import (
DEFAULT_MODEL_CREATED_AT_TIME,
LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
MAX_TEAM_LIST_LIMIT,
SPEND_LOG_QUEUE_MAX_BYTES,
@ -133,7 +135,8 @@ from litellm.litellm_core_utils.core_helpers import (
independent_snapshot,
is_expected_client_error,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.token_counter import offload_token_count
@ -221,6 +224,7 @@ from litellm.repositories.verification_token_repository import (
VerificationTokenRepository,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.deployment_pricing import pricing_models
from litellm.secret_managers.main import str_to_bool
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
from litellm.types.llms.openai import ResponsesAPIResponse
@ -233,6 +237,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionRe
from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams
from litellm.utils import (
_add_custom_logger_callback_to_specific_event, # pyright: ignore[reportPrivateUsage] # only string-to-logger helper
_get_base_model_from_metadata, # pyright: ignore[reportPrivateUsage] # line predates the public name
)
if TYPE_CHECKING:
@ -921,20 +926,48 @@ def _failure_usage_to_lift(
model_call_details: Mapping[str, object],
request_body: Mapping[str, object],
dispatched: bool,
provider_billed: bool = False,
deployment_id: str | None = None,
base_model: str | None = None,
) -> tuple[object, object] | None:
"""A stream that broke mid-flight still billed the provider for the chunks
already delivered; the streaming handler stashes that recovered usage and
cost in model_call_details, so prefer it. Otherwise a request that was
dispatched to a provider and failed without upstream usage gets an
estimated input-side Usage with zero cost. The raw request body backfills
the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a
chat-completions provider) without filling optional_params. Returns the
estimated input-side Usage. When the failure is one the provider bills
anyway (``provider_billed``: a read timeout or a client disconnect, where
the provider had the prompt and kept working after LiteLLM hung up) that
usage is priced at the model's input rate, the way the success path prices it:
at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``)
when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure
deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch
failure stays at zero cost. Returns the
(combined_usage_object, response_cost) pair to lift, or None."""
recovered_usage: Final = model_call_details.get("combined_usage_object")
if recovered_usage is not None:
return recovered_usage, model_call_details.get("response_cost")
if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL):
if not dispatched:
return None
if model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL) and not provider_billed:
# The proxy-only error handler stamps this flag on a client disconnect too, even though the
# request had already reached the provider, so a billed failure keeps its estimate.
return None
estimated_usage: Final = _estimated_prompt_usage(model_call_details, request_body)
if estimated_usage is None:
return None
if not provider_billed:
return estimated_usage, 0.0
return estimated_usage, _estimated_input_cost(
model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model
)
def _estimated_prompt_usage(
model_call_details: Mapping[str, object], request_body: Mapping[str, object]
) -> Usage | None:
"""The input-only Usage estimate for the prompt this request sent, or None when the call type has no
estimable input. The raw request body backfills the system prompt when the SDK bridges an endpoint
(e.g. /v1/messages on a chat-completions provider) without filling optional_params."""
if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES:
return None
optional_params: Final = model_call_details.get("optional_params")
@ -944,14 +977,205 @@ def _failure_usage_to_lift(
else None
)
system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions")
estimated_usage: Final = _estimate_dispatched_failure_usage(
return _estimate_dispatched_failure_usage(
model=str(model_call_details.get("model") or ""),
request_input=model_call_details.get("messages"),
system_input=system_input,
)
if estimated_usage is None:
@dataclass(frozen=True)
class BilledFailedAttempt:
"""An earlier retry attempt the provider also billed, written as its own spend row."""
attempted_retries: int
deployment_id: str | None
usage: Usage
response_cost: float
_NO_RETRY_METADATA: Final[Mapping[str, object]] = MappingProxyType({})
def _retry_metadata_bucket(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]:
"""The request metadata bucket the router writes retry state into, ``litellm_metadata`` when the
route carries one (/messages, /responses) and ``metadata`` otherwise, same order as the deployment id."""
if litellm_params is None:
return _NO_RETRY_METADATA
for metadata_key in ("litellm_metadata", "metadata"):
metadata = litellm_params.get(metadata_key)
if isinstance(metadata, dict):
return cast("Mapping[str, object]", metadata) # cast-ok: untyped router dict, keys read with .get
return _NO_RETRY_METADATA
def _attempt_number(record: Mapping[str, object], index: int) -> int:
"""The attempt a retry record describes: its own ``attempted_retries`` count, else its position."""
recorded: Final = record.get("attempted_retries")
return recorded if isinstance(recorded, int) else index
def _retry_attempt_records(litellm_params: Mapping[str, object] | None) -> tuple[Mapping[str, object], ...]:
"""The router's ``RetryAttemptRecord`` breadcrumbs, one per failed attempt, read from the same bucket."""
if litellm_params is None:
return ()
for metadata_key in ("litellm_metadata", "metadata"):
metadata = litellm_params.get(metadata_key)
if not isinstance(metadata, dict):
continue
records = cast("Mapping[str, object]", metadata).get("previous_models") # cast-ok: untyped router dict
if isinstance(records, (list, tuple)):
return tuple(
cast("Mapping[str, object]", record) # cast-ok: RetryAttemptRecord TypedDict, read with .get
for record in cast("Sequence[object]", records) # cast-ok: untyped router tuple
if isinstance(record, dict)
)
return ()
def _record_deployment_id(record: Mapping[str, object]) -> str | None:
deployment_id: Final = record.get("deployment_id")
return deployment_id if isinstance(deployment_id, str) else None
def _attempt_identity(record: Mapping[str, object]) -> tuple[str | None, int | None]:
"""What tells one provider call from another across the request: the deployment that served it
and the retry count within its model group. Two records with the same identity describe the
same call (the fallback handler re-logs the attempt that exhausted the previous group)."""
attempted_retries: Final = record.get("attempted_retries")
return _record_deployment_id(record), attempted_retries if isinstance(attempted_retries, int) else None
def _timed_out_earlier_records(
litellm_params: Mapping[str, object] | None, final_identity: tuple[str | None, int | None]
) -> tuple[Mapping[str, object], ...]:
"""The ``Timeout`` breadcrumbs for calls other than the final attempt, one per provider call."""
by_identity: Final = {
_attempt_identity(record): record
for record in _retry_attempt_records(litellm_params)
if record.get("exception_type") == litellm.Timeout.__name__
}
return tuple(record for identity, record in by_identity.items() if identity != final_identity)
def _record_pricing(
model_call_details: Mapping[str, object], record: Mapping[str, object], base_model: str | None
) -> tuple[Mapping[str, object], str | None]:
"""The ``(model_call_details, base_model)`` to price one earlier attempt with: the record's own
deployment model and ``base_model`` when it served a different model than the final attempt
(the provider is then inferred from that model), else the final attempt's."""
record_model: Final = record.get("model")
if not isinstance(record_model, str) or record_model == model_call_details.get("model"):
return model_call_details, base_model
record_base_model: Final = record.get("base_model")
return (
{**model_call_details, "model": record_model, "custom_llm_provider": None},
record_base_model if isinstance(record_base_model, str) else None,
)
def _billed_earlier_attempt(
model_call_details: Mapping[str, object],
record: Mapping[str, object],
index: int,
estimated_usage: Usage,
base_model: str | None,
) -> BilledFailedAttempt:
pricing_details, pricing_base_model = _record_pricing(model_call_details, record, base_model)
return BilledFailedAttempt(
attempted_retries=_attempt_number(record, index),
deployment_id=_record_deployment_id(record),
usage=estimated_usage,
response_cost=_estimated_input_cost(
pricing_details,
estimated_usage.prompt_tokens,
deployment_id=_record_deployment_id(record),
base_model=pricing_base_model,
),
)
def _billed_earlier_attempts(
model_call_details: Mapping[str, object],
litellm_params: Mapping[str, object] | None,
estimated_usage: Usage,
base_model: str | None,
deployment_id: str | None,
) -> tuple[BilledFailedAttempt, ...]:
"""Every earlier retry attempt the provider also billed. The router records one
``RetryAttemptRecord`` per failed attempt in ``metadata.previous_models``, the final attempt
included, which the failure row itself bills. The final attempt is the record with the
request's own deployment and retry count (``attempted_retries`` restarts at 0 after a fallback
hop, so the count alone cannot tell the groups apart). An earlier attempt that ended in a
``Timeout`` had the same prompt accepted by its deployment, so it is billed like the final
attempt, at that deployment's input rate (else the cost map's rate for the record's own
``base_model`` or model, which a fallback hop can make a different model than the final one).
Only the exception class survives on the record, so an earlier connect timeout is billed too;
any other failure class is not."""
final_attempt: Final = _retry_metadata_bucket(litellm_params).get("attempted_retries")
final_identity: Final = (deployment_id, final_attempt if isinstance(final_attempt, int) else None)
return tuple(
_billed_earlier_attempt(model_call_details, record, index, estimated_usage, base_model)
for index, record in enumerate(_timed_out_earlier_records(litellm_params, final_identity))
)
def _deployment_priced_input_cost(
model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo
) -> float:
"""Input price for the estimated prompt at the deployment's registered rates, read through
``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success
path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``)
is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0."""
usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens)
try:
prompt_cost, _ = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""),
model_info=deployment_pricing,
)
except Exception as exc: # noqa: BLE001 # a malformed registered entry must not drop the failure row
verbose_proxy_logger.debug("failure spend estimate: could not price deployment of %s: %s", model, exc)
return 0.0
return prompt_cost
def _priced_input_cost(pricing_model: str, custom_llm_provider: str | None, prompt_tokens: int) -> float | None:
try:
estimated_input_cost, _ = litellm.cost_per_token(
model=pricing_model,
custom_llm_provider=custom_llm_provider,
prompt_tokens=prompt_tokens,
)
except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written
verbose_proxy_logger.debug("failure spend estimate: could not price %s: %s", pricing_model, exc)
return None
return estimated_usage, 0.0
return estimated_input_cost
def _estimated_input_cost(
model_call_details: Mapping[str, object],
prompt_tokens: int,
deployment_id: str | None,
base_model: str | None,
) -> float:
"""Input price for the estimated prompt: the deployment's registered rates when it has any (the id
is ``Logging.get_router_model_id()``, never the flattened ``model_call_details`` copy a request body
key can overwrite), else the cost map's rate for ``base_model``, else the deployment model, the same
candidate order ``completion_cost`` walks (an Azure deployment name is not in the cost map, its
``base_model`` is; a ``base_model`` the cost map does not know falls through to the deployment
model). A model no candidate can price keeps the row at $0 rather than dropping the failure row."""
model: Final = str(model_call_details.get("model") or "")
custom_llm_provider: Final = str(model_call_details.get("custom_llm_provider") or "") or None
deployment_pricing: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model)
if deployment_pricing is not None:
return _deployment_priced_input_cost(model, custom_llm_provider, prompt_tokens, deployment_pricing)
pricing_models_to_try: Final = pricing_models(base_model, model)
candidate_costs: Final = (
_priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens) for pricing_model in pricing_models_to_try
)
return next((cost for cost in candidate_costs if cost is not None), 0.0)
_EMPTY_LIFT: Final = MappingProxyType({})
@ -972,7 +1196,38 @@ def _call_type_for_route(route: str | None) -> str | None:
_PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException)
def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]:
_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16
def _exception_chain(exception: BaseException) -> tuple[BaseException, ...]:
chain: Final[list[BaseException]] = [exception] # mutable-ok: built once, returned frozen
for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH):
following = chain[-1].__cause__ or chain[-1].__context__
if following is None or following in chain:
break
chain.append(following)
return tuple(chain)
def _timed_out_reading_from_provider(exception: Exception) -> bool:
"""A read timeout means the request was sent and the provider was working on it; a connect
timeout (``httpx.ConnectTimeout``) never reached the provider, so it is not billed."""
return isinstance(exception, litellm.Timeout) and any(
isinstance(cause, httpx.ReadTimeout) for cause in _exception_chain(exception)
)
def _client_disconnected(exception: Exception) -> bool:
"""The 499 the proxy itself raises when the caller hung up while the provider was still working on the
request. A provider's own 499 arrives as a litellm exception and is a plain upstream failure."""
return isinstance(exception, HTTPException) and exception.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
def _provider_billed_the_failure(exception: Exception) -> bool:
return _timed_out_reading_from_provider(exception) or _client_disconnected(exception)
def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: bool = False) -> Mapping[str, object]:
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
request_data (it is not serialisable), so the caller merges these fields
onto request_data first: the request start and first-handoff instants for
@ -984,10 +1239,28 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str,
return _EMPTY_LIFT
_model_call_details: Final = getattr(_logging_obj, "model_call_details", {})
_first_handoff: Final = _model_call_details.get("first_api_call_start_time")
_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)
_base_model: Final = _get_base_model_from_metadata(model_call_details=_model_call_details)
_deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None
_success_already_logged: Final = isinstance(_logging_obj, Logging) and _logging_obj.async_success_log_enqueued
_billed: Final = provider_billed and not _success_already_logged
_usage_to_lift: Final = _failure_usage_to_lift(
model_call_details=_model_call_details,
request_body=request_data,
dispatched=_first_handoff is not None,
provider_billed=_billed,
deployment_id=_deployment_id,
base_model=_base_model,
)
_prompt_estimate: Final = (
_estimated_prompt_usage(_model_call_details, request_data)
if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
else None
)
_earlier_attempts: Final = (
_billed_earlier_attempts(_model_call_details, _litellm_params, _prompt_estimate, _base_model, _deployment_id)
if _prompt_estimate is not None
else ()
)
_entries: Final = (
("start_time", _model_call_details.get("start_time")),
@ -996,6 +1269,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str,
("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]),
("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)),
("standard_logging_object", _model_call_details.get("standard_logging_object")),
("billed_failed_attempts", _earlier_attempts or None),
)
return MappingProxyType({key: value for key, value in _entries if value is not None})
@ -3085,7 +3359,11 @@ class ProxyLogging:
original_exception=original_exception,
)
request_data.update(await offload_token_count(_failure_fields_to_lift)(request_data))
request_data.update(
await offload_token_count(_failure_fields_to_lift)(
request_data, provider_billed=_provider_billed_the_failure(original_exception)
)
)
# Remove before callbacks iterate — not serialisable
request_data.pop("litellm_logging_obj", None)
@ -7444,7 +7722,11 @@ MAX_SPEND_LOG_ISOLATION_FAILURES_PER_BATCH: Final = 256
def _is_transient_spend_log_write_error(e: Exception) -> bool:
return PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e)
return (
PrismaDBExceptionHandler.is_database_transport_error(e)
or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e)
or PrismaDBExceptionHandler.is_deadlock_error(e)
)
async def _create_spend_logs_with_poison_isolation(
@ -7478,7 +7760,7 @@ async def _create_spend_logs_with_poison_isolation(
await repo.table.create_many(data=rows, skip_duplicates=True)
return failure_budget
except Exception as e:
if not PrismaDBExceptionHandler.is_prisma_data_error(e):
if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)):
raise
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
raise

39
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -14,7 +14,7 @@ from litellm.responses.litellm_completion_transformation.custom_tools import (
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
PART_UNION_TYPES,
@ -1259,6 +1259,43 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
return reasoning_aligned
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
"""
The response.incomplete event to bill when the client went away mid-stream.
Built from the chat completion chunks relayed so far, so the usage is what
stream_chunk_builder counts for them.
"""
if self.completed_response is not None or not self.collected_chat_completion_chunks:
return None
built: Final = stream_chunk_builder(
chunks=self.collected_chat_completion_chunks,
messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
input=self.request_input,
responses_api_request=dict(self.responses_api_request),
),
logging_obj=self.litellm_logging_obj,
)
if not isinstance(built, ModelResponse):
return None
responses_api_response: Final = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
request_input=self.request_input,
chat_completion_response=built,
responses_api_request=self.responses_api_request,
)
)
partial: Final = self._with_encoded_response_id(
responses_api_response.model_copy(
update={
"id": self._cached_response_id or responses_api_response.id,
"status": "incomplete",
"output": list(self._output_with_streamed_item_ids(responses_api_response)),
}
)
)
stamp_responses_usage_cost(partial, self.litellm_logging_obj)
return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
def _emit_terminal_response_event(
self, litellm_model_response: ModelResponse
) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:

16
litellm/responses/main.py
View file

@ -544,6 +544,22 @@ def _will_bridge_to_chat_completions(
)
def provider_stores_the_conversation(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
"""Whether the provider, not a LiteLLM spend log, holds this turn for a `previous_response_id` follow-up.
A bridged turn is replayed from its spend log, so the bridge needs that row in the DB, and a bridged
first MCP round runs as an internal call that writes no spend log at all.
"""
api_base: Final = _api_base_kwarg(kwargs)
return not _will_bridge_to_chat_completions(
model,
custom_llm_provider or litellm.get_llm_provider(model=model, api_base=api_base)[1],
bool(kwargs.get("use_chat_completions_api")),
kwargs.get("model_info"),
api_base,
)
@contextmanager
def _prompt_management_sees_a_provisional_message_list(
kwargs: dict[str, Any], # mutable-ok: the signal is read and popped out of the caller's own kwargs

72
litellm/responses/streaming_iterator.py
View file

@ -41,10 +41,12 @@ from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.openai import (
PART_UNION_TYPES,
ResponseAPIUsage,
ResponseIncompleteEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
)
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
from litellm.types.utils import CallTypes
from litellm.utils import async_post_call_success_deployment_hook
@ -289,6 +291,7 @@ class BaseResponsesAPIStreamingIterator:
self._yielded_first_chunk = False
self._generated_content = ""
self._generated_tool_arguments = ""
self._created_response: ResponsesAPIResponse | None = None
self._completed_response_cached = False
self._completed_response_logged = False
self._completed_response_cache_hit: bool | None = None
@ -450,6 +453,10 @@ class BaseResponsesAPIStreamingIterator:
# Store the completed response (also for incomplete/failed so logging still fires)
_chunk_type: Final = getattr(openai_responses_api_chunk, "type", None)
openai_types = _get_openai_response_types()
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED:
_created: Final[object] = getattr(openai_responses_api_chunk, "response", None)
if isinstance(_created, ResponsesAPIResponse):
self._created_response = _created
if openai_responses_api_chunk and _chunk_type in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
@ -481,7 +488,7 @@ class BaseResponsesAPIStreamingIterator:
else openai_responses_api_chunk.model_copy(update={"response": _billed_response})
)
self.completed_response = _terminal_chunk
_stamp_responses_usage_cost(_billed_response, self.logging_obj)
stamp_responses_usage_cost(_billed_response, self.logging_obj)
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
self._handle_logging_failed_response()
@ -875,6 +882,34 @@ class BaseResponsesAPIStreamingIterator:
pass
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
"""
The response.incomplete event to bill when the client went away mid-stream,
the same shape the success handler assembles for a natural end of stream.
Its response is the response.created payload (it carries the served
service_tier) plus the text and tool arguments relayed so far, with counted
usage since the provider never sent any. None when nothing was relayed yet
or the stream already reached its terminal event and billed itself.
"""
created: Final = self._created_response
generated: Final = self._generated_content + self._generated_tool_arguments
if self.completed_response is not None or created is None or not generated:
return None
usage: Final = _estimate_usage_safely(
self.model or "", self.request_data.get("input"), self.request_data, generated
)
if usage is None:
return None
partial: Final = created.model_copy(
update={
"status": "incomplete",
"usage": usage,
"output": _partial_output_items(self._generated_content),
}
)
stamp_responses_usage_cost(partial, self.logging_obj)
return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
async def call_post_streaming_hooks_for_testing(
iterator: object, chunk: ResponsesAPIStreamingResponse
) -> ResponsesAPIStreamingResponse:
@ -1347,6 +1382,37 @@ def _add_text_like_part_events(
)
@runtime_checkable
class _BillsPartialStreamOnDisconnect(Protocol):
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: ...
def partial_responses_incomplete_event_on_disconnect(stream: object) -> ResponseIncompleteEvent | None:
"""Ask the stream behind a proxy or router wrapper for the partial response to bill, if it can build one."""
if not isinstance(stream, _BillsPartialStreamOnDisconnect):
return None
try:
partial: Final = stream.partial_response_on_disconnect()
except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown
verbose_logger.debug("Failed to build partial Responses API response for disconnect billing: %s", e)
return None
return partial if isinstance(partial, ResponseIncompleteEvent) else None
def _partial_output_items(generated_text: str) -> Sequence[GenericResponseOutputItem]:
if not generated_text:
return []
return [
GenericResponseOutputItem(
type="message",
id=f"msg_{uuid.uuid4()}",
status="incomplete",
role="assistant",
content=[OutputText(type="output_text", text=generated_text, annotations=[])],
)
]
def _billed_terminal_response(
response_obj: object, estimate: Callable[[], ResponseAPIUsage | None] | None
) -> ResponsesAPIResponse | None:
@ -1453,7 +1519,7 @@ def _estimate_usage_safely(
return None
def _stamp_responses_usage_cost(
def stamp_responses_usage_cost(
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
) -> None:
if response_obj is None or logging_obj is None:
@ -1479,7 +1545,7 @@ def build_synthetic_response_events(
chunk_size: int,
) -> list[ResponsesAPIStreamingResponse]:
openai_types: Final = _get_openai_response_types()
_stamp_responses_usage_cost(transformed, logging_obj)
stamp_responses_usage_cost(transformed, logging_obj)
events: Final[list[ResponsesAPIStreamingResponse]] = [
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),

77
litellm/router.py
View file

@ -97,6 +97,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.sensitive_data_masker import (
@ -132,6 +133,7 @@ from litellm.router_strategy.tag_based_routing import (
is_valid_deployment_tag,
)
from litellm.router_utils.add_retry_fallback_headers import (
HiddenParamsAsyncIteratorWrapper,
_HiddenParamsHost,
add_fallback_headers_to_response,
add_retry_headers_to_response,
@ -183,6 +185,7 @@ from litellm.router_utils.cooldown_handlers import (
is_advisor_orchestration_failure,
is_caller_timeout_408,
)
from litellm.router_utils.deployment_pricing import deployment_pricing_models
from litellm.router_utils.fallback_event_handlers import (
AttemptedFallbackTargets,
_check_non_standard_fallback_format,
@ -339,6 +342,7 @@ if TYPE_CHECKING:
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseIncompleteEvent,
ResponseInputParam,
ResponsesAPIResponse,
)
@ -591,6 +595,11 @@ def _anthropic_stream_commits_now(chunk: object, has_generated_content: bool, bu
return is_anthropic_content_delta_chunk(chunk) or buffered_chunk_count >= MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS
def _stream_behind_hidden_params_wrapper(stream: object) -> object:
"""set_response_headers wraps a bare byte stream before the fallback wrapper sees it; bill the stream inside."""
return stream._inner if isinstance(stream, HiddenParamsAsyncIteratorWrapper) else stream # pyright: ignore[reportPrivateUsage] # the wrapper exposes nothing else
class FallbackAwareAnthropicMessagesStream:
"""
Bare async generators can't carry the `_hidden_params` attribute the
@ -617,6 +626,13 @@ class FallbackAwareAnthropicMessagesStream:
self._source_iterator = fallback_response
self.fallback_headers_adopted = True
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
return partial_responses_incomplete_event_on_disconnect(
_stream_behind_hidden_params_wrapper(self._source_iterator)
)
def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream":
return self
@ -3209,6 +3225,7 @@ class Router:
from datetime import datetime
self._async_generator = async_generator
self._billing_source: object = source_iterator
# Mirror every attribute BaseResponsesAPIStreamingIterator.__init__
# would have set. The wrapper bypasses super().__init__ (it has no
# httpx.Response of its own and no provider config to drive), so
@ -3258,9 +3275,28 @@ class Router:
def adopt_fallback_headers(self, fallback_response: object) -> tuple[dict[str, object], dict[str, object]]:
prepared: Final = Router._prepare_fallback_hidden_params(fallback_response)
self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} # mutable-ok: stream metadata
self._billing_source = fallback_response
self.fallback_headers_adopted = True
return prepared
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
"""
The event to bill on a client disconnect. After a mid-stream hop the primary
stream's relayed tokens are folded into the fallback's event, the way
_combine_responses_fallback_usage does for a natural end of stream.
"""
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
current: Final = partial_responses_incomplete_event_on_disconnect(self._billing_source)
if self._billing_source is source_iterator:
return current
primary: Final = partial_responses_incomplete_event_on_disconnect(source_iterator)
if current is None or primary is None:
return current or primary
if primary.response.usage is not None:
Router._combine_responses_fallback_usage(current, primary.response.usage)
return current
def __aiter__(self):
return self
@ -4003,6 +4039,7 @@ class Router:
"""Helper to get stream timeout from kwargs or deployment params"""
return (
kwargs.get("stream_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier")
or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment
or self.stream_timeout # timeout set on router
or self.request_timeout # litellm_settings.request_timeout (per-attempt)
@ -4014,6 +4051,7 @@ class Router:
timeout: Final = (
kwargs.get("timeout", None) # the params dynamically set by user
or kwargs.get("request_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier")
or data.get("timeout", None) # timeout set on litellm_params for this deployment
or data.get("request_timeout", None) # timeout set on litellm_params for this deployment
or self.request_timeout # litellm_settings.request_timeout (per-attempt)
@ -8264,17 +8302,26 @@ class Router:
When a retry or fallback happens, record which model group, deployment and attempt just failed and why,
and count it toward the request-wide num_retries_per_request cap
"""
from litellm.litellm_core_utils.get_litellm_params import (
_get_base_model_from_litellm_call_metadata as get_base_model_from_litellm_call_metadata, # pyright: ignore[reportPrivateUsage] # line predates the public name
)
from litellm.types.router import RetryAttemptRecord
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var]
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
model_group: Final = kwargs.get("model")
model_info: Final = request_metadata.get("model_info")
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
base_model: Final = get_base_model_from_litellm_call_metadata(
metadata=request_metadata
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
deployment_model: Final = request_metadata.get("deployment")
attempted_retries: Final = request_metadata.get("attempted_retries")
attempt_record: Final[RetryAttemptRecord] = {
"model_group": model_group if isinstance(model_group, str) else None,
"deployment_id": deployment_id if isinstance(deployment_id, str) else None,
"model": deployment_model if isinstance(deployment_model, str) else None,
"base_model": base_model if isinstance(base_model, str) else None,
"exception_type": type(e).__name__,
"exception_string": str(e),
"attempted_retries": attempted_retries if type(attempted_retries) is int else None,
@ -8292,6 +8339,18 @@ class Router:
kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap
return kwargs
def _deployment_params_base_model(self, deployment_id: str | None) -> str | None:
"""The ``base_model`` a deployment declares under ``litellm_params`` (the Azure named-deployment
shape), the same key the success path's ``get_base_model_from_metadata`` reads."""
deployment: Final = self.get_deployment(model_id=deployment_id) if deployment_id is not None else None
if deployment is None:
return None
extra_params: Final = cast( # cast-ok: pydantic extras are untyped, base_model is isinstance-checked below
"Mapping[str, object]", deployment.litellm_params.model_extra or {}
)
base_model: Final = extra_params.get("base_model")
return base_model if isinstance(base_model, str) else None
def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int:
"""
Update deployment rpm for that minute
@ -10283,19 +10342,9 @@ class Router:
)
for deployment in deployments
)
params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments)
# base_model resolution mirrors get_router_model_info: unset or blank means the
# deployment's own model name is the cost-map key.
cost_map_keys: Final = tuple(
dict.fromkeys( # deduplicates while preserving config order
key
for key in (
model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
for model_info, litellm_params in zip(model_infos, params)
)
if isinstance(key, str) and key
)
)
raw_deployments: Final[tuple[object, ...]] = deployments
pricing_keys: Final = (deployment_pricing_models(deployment) for deployment in raw_deployments)
cost_map_keys: Final = tuple(dict.fromkeys(keys[0] for keys in pricing_keys if keys))
return DeploymentModelListingInfo(
cost_map_keys=cost_map_keys,
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),

33
litellm/router_strategy/lowest_cost.py
View file

@ -1,5 +1,6 @@
#### What this does ####
# picks based on response time (for streaming, this is time to first token)
from collections.abc import Mapping
from datetime import datetime
from typing import Final
@ -8,6 +9,30 @@ from litellm import ModelResponse, token_counter, verbose_logger
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
"""Input and output per-token prices from the first cost-map row among the deployment's
pricing models that carries one; (None, None) when no row does, so the caller falls back to
its unmapped default."""
rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment))
row: Final = next((row for row in rows if _declares_token_price(row)), None)
if row is None:
return None, None
return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token"))
def _cost_map() -> Mapping[str, object]:
return litellm.model_cost
def _declares_token_price(row: Mapping[str, object]) -> bool:
return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None
def _as_rate(value: object) -> float | None:
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
class LowestCostLoggingHandler(CustomLogger):
@ -234,8 +259,8 @@ class LowestCostLoggingHandler(CustomLogger):
or _deployment.get("model_info", {}).get("rpm", None)
or float("inf")
)
item_litellm_model_name = _deployment.get("litellm_params", {}).get("model")
item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {})
deployment_view: object = _deployment
map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view)
# check if user provided input_cost_per_token and output_cost_per_token in litellm_params
item_input_cost = None
@ -247,10 +272,10 @@ class LowestCostLoggingHandler(CustomLogger):
item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token")
if item_input_cost is None:
item_input_cost = item_litellm_model_cost_map.get("input_cost_per_token", 5.0)
item_input_cost = map_input_cost if map_input_cost is not None else 5.0
if item_output_cost is None:
item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0)
item_output_cost = map_output_cost if map_output_cost is not None else 5.0
# if litellm["model"] is not in model_cost map -> use item_cost = $10

37
litellm/router_utils/deployment_pricing.py Normal file
View file

@ -0,0 +1,37 @@
from collections.abc import Iterator, Mapping
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter, ValidationError
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_STR_KEYED: Final = TypeAdapter(dict[str, object])
def pricing_models(base_model: object, model: object) -> tuple[str, ...]:
"""Cost-map keys to price a deployment under, ``model_info.base_model`` first, then the
deployment's own model, blanks and duplicates dropped. Every spend and routing surface that
prices a deployment walks this order, so a ``base_model`` the cost map does not know falls
through to the deployment model instead of billing or ranking at the unmapped default."""
return tuple(dict.fromkeys(_named_models(base_model, model)))
def deployment_pricing_models(deployment: object) -> tuple[str, ...]:
"""``pricing_models`` read off a raw deployment dict (``model_info.base_model``, then
``litellm_params.base_model``, then ``litellm_params.model``); malformed blocks count as unset."""
view: Final = str_keyed(deployment)
params: Final = str_keyed(view.get("litellm_params"))
info: Final = str_keyed(view.get("model_info"))
return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model"))
def _named_models(*candidates: object) -> Iterator[str]:
return (candidate for candidate in candidates if isinstance(candidate, str) and candidate)
def str_keyed(value: object) -> Mapping[str, object]:
"""``value`` as a read-only str-keyed mapping, empty when it is not a dict."""
try:
return MappingProxyType(_STR_KEYED.validate_python(value))
except ValidationError:
return _EMPTY

2
litellm/types/integrations/anthropic_cache_control_hook.py
View file

@ -22,7 +22,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
class CacheControlToolConfigInjectionPoint(TypedDict):
"""Type for tool_config-level injection points (Bedrock)."""
"""Type for tool_config-level injection points (Anthropic and Bedrock)."""
location: Literal["tool_config"]
control: ChatCompletionCachedContent | None

7
litellm/types/router.py
View file

@ -327,6 +327,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
otpm: int | None = None
timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/
stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/
# per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key
timeout_by_service_tier: Mapping[str, float] | None = None
stream_timeout_by_service_tier: Mapping[str, float] | None = None
max_retries: int | None = None
drop_params: bool | str | None = None
organization: str | None = None # for openai orgs
@ -503,6 +506,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
api_version: str | None
timeout: float | str | httpx.Timeout | None
stream_timeout: float | str | None
timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
max_retries: int | None
organization: list | str | None # for openai orgs
configurable_clientside_auth_params: (
@ -923,6 +928,8 @@ class RouterModelGroupAliasItem(TypedDict):
class RetryAttemptRecord(TypedDict):
model_group: ReadOnly[str | None]
deployment_id: ReadOnly[str | None]
model: ReadOnly[str | None]
base_model: ReadOnly[str | None]
exception_type: ReadOnly[str]
exception_string: ReadOnly[str]
attempted_retries: ReadOnly[int | None]

7
litellm/types/utils.py
View file

@ -185,6 +185,12 @@ class SearchContextCostPerQuery(TypedDict, total=False):
search_context_size_high: float
class SearchFreeRequests(TypedDict):
count: ReadOnly[int]
period: ReadOnly[Literal["day", "month"]]
shared_group: ReadOnly[str]
class AgenticLoopParams(TypedDict, total=False):
"""
Parameters passed to agentic loop hooks (e.g., WebSearch interception).
@ -347,6 +353,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
web_search_billing_unit: (
Literal["per_query", "per_prompt"] | None
) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x)
search_free_requests: ReadOnly[SearchFreeRequests | None]
google_maps_grounding_cost_per_query: ReadOnly[float | None]
citation_cost_per_token: float | None # Cost per citation token for Perplexity
tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope

8
litellm/utils.py
View file

@ -1279,6 +1279,7 @@ def _schedule_async_success_logging(
return
def _enqueue_async_logging() -> None:
logging_obj.mark_async_success_log_enqueued()
asyncio.create_task(
_client_async_logging_helper(
logging_obj=logging_obj,
@ -5021,6 +5022,9 @@ def get_optional_params(
return optional_params
LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",))
def add_provider_specific_params_to_optional_params(
optional_params: dict,
passed_params: dict,
@ -5036,8 +5040,9 @@ def add_provider_specific_params_to_optional_params(
# for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46
if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False:
extra_body: Final = dict(passed_params.pop("extra_body", None) or {})
withheld_params: Final = frozenset[str]() if custom_llm_provider == "litellm_proxy" else LITELLM_ONLY_PARAMS
for k in passed_params:
if k not in openai_params and passed_params[k] is not None:
if k not in openai_params and k not in withheld_params and passed_params[k] is not None:
extra_body[k] = passed_params[k]
if not isinstance(optional_params.get("extra_body"), dict):
optional_params["extra_body"] = {}
@ -6256,6 +6261,7 @@ def _get_model_info_helper(
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
search_free_requests=_model_info.get("search_free_requests", None),
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),

1
tests/code_coverage_tests/router_code_coverage.py
View file

@ -88,6 +88,7 @@ ignored_function_names = [
"_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"_deployment_params_base_model", # Tested through log_retry in test_router_helper_utils.py
]

70
tests/e2e/llm_translation/test_messages_count_tokens_bedrock_e2e.py Normal file
View file

@ -0,0 +1,70 @@
"""Live e2e: `/v1/messages/count_tokens` on a Bedrock Claude model that bedrock-runtime
cannot count.
Claude Opus 4.8 is offered only through cross-region inference, and bedrock-runtime's
CountTokens answers 400 for it. The proxy then has to count through bedrock-mantle's
Anthropic count_tokens, and the answer must sit within a few percent of what `/v1/messages`
bills as `usage.input_tokens`. The local tokenizer fallback undercounts these models by
about 40%, so this is the line that proves the real count is served. Both calls go through
the real Anthropic SDK, the client customers count with
"""
from __future__ import annotations
from typing import Final
import pytest
from anthropic.types import MessageParam
from e2e_config import unique_marker
from e2e_metadata import Domain, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
CROSS_REGION_ONLY_CLAUDE_BACKEND: Final = "bedrock/global.anthropic.claude-opus-4-8"
COUNT_TOLERANCE: Final = 0.05
def _register(proxy: ProxyClient, resources: ResourceManager, backend: str) -> tuple[str, str]:
model: Final = f"e2e-count-tokens-bedrock-{unique_marker()}"
model_id: Final = proxy.create_model(
model,
LiteLLMParamsBody(
model=backend,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
),
)
resources.defer(lambda: proxy.delete_model(model_id))
return model, resources.key()
class TestBedrockMessagesCountTokens:
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.COUNT_TOKENS,
providers=(Provider.BEDROCK,),
models=(CROSS_REGION_ONLY_CLAUDE_BACKEND,),
)
)
def test_count_matches_billed_input_tokens_for_a_model_bedrock_runtime_cannot_count(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model, key = _register(proxy, resources, CROSS_REGION_ONLY_CLAUDE_BACKEND)
client: Final = sdk.anthropic(key)
prompt: Final = f"{unique_marker()} " + "The quick brown fox jumps over the lazy dog. " * 40
message: Final[MessageParam] = {"role": "user", "content": prompt}
counted: Final = client.messages.count_tokens(model=model, messages=[message])
answered: Final = client.messages.create(
model=model, max_tokens=1, messages=[message], extra_body=NO_PROXY_CACHE
)
billed: Final = answered.usage.input_tokens
assert billed, answered.usage
assert abs(counted.input_tokens - billed) <= billed * COUNT_TOLERANCE, (counted, answered.usage)

1206
tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py Normal file
View file

File diff suppressed because it is too large Load diff

57
tests/local_testing/test_router_timeout.py
View file

@ -7,6 +7,7 @@ import time
import traceback
import pytest
from typing import Final
from unittest.mock import patch, MagicMock, AsyncMock
@ -337,3 +338,59 @@ def test_unit_test_streaming_timeout(stream):
assert stream_timeout_val == stream_timeout
else:
assert stream_timeout_val == normal_timeout
@pytest.mark.parametrize(
"request_kwargs, expected_timeout",
[
({"service_tier": "flex"}, 900.0),
({"service_tier": "default"}, 610.0),
({}, 610.0),
({"service_tier": "flex", "timeout": 20}, 20),
({"service_tier": "flex", "stream": True}, 950.0),
({"service_tier": "default", "stream": True}, 610.0),
({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30),
],
)
def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout(
request_kwargs: dict[str, object], expected_timeout: float
) -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 610,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout
def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0
assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0

92
tests/proxy_unit_tests/test_proxy_token_counter.py
View file

@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from dotenv import load_dotenv
from pydantic import JsonValue
load_dotenv()
@ -25,6 +26,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import ProxyException, TokenCountRequest
from litellm.proxy.anthropic_endpoints.endpoints import (
count_tokens as anthropic_count_tokens,
@ -871,41 +873,45 @@ def test_vertex_ai_partner_models_token_counting_endpoint(vertex_location):
)
class _FailingRuntimeHandler(BedrockCountTokensHandler):
def __init__(self, error: Exception) -> None:
super().__init__()
self._error = error
async def handle_count_tokens_request(
self,
request_data: dict[str, object],
litellm_params: dict[str, object],
resolved_model: str,
client: AsyncHTTPHandler | None = None,
) -> dict[str, JsonValue]:
raise self._error
@pytest.mark.asyncio
async def test_bedrock_token_counter_error_propagation_bedrock_error():
"""
Test that BedrockTokenCounter properly returns error response when BedrockError is raised.
Verifies that the status code and error message are preserved.
"""
counter = BedrockTokenCounter()
counter = BedrockTokenCounter(
runtime_handler=_FailingRuntimeHandler(BedrockError(status_code=429, message="Rate limit exceeded"))
)
# Mock the handler to raise BedrockError with specific status code
with patch.object(
counter, "count_tokens", wraps=counter.count_tokens
) as mock_count:
# We need to patch at the handler level
with patch(
"litellm.llms.bedrock.count_tokens.bedrock_token_counter.BedrockCountTokensHandler"
) as MockHandler:
mock_handler_instance = MockHandler.return_value
mock_handler_instance.handle_count_tokens_request = AsyncMock(
side_effect=BedrockError(status_code=429, message="Rate limit exceeded")
)
result = await counter.count_tokens(
model_to_use="anthropic.claude-3-sonnet",
messages=[{"role": "user", "content": "hello"}],
contents=None,
deployment={"litellm_params": {}},
request_model="bedrock/anthropic.claude-3-sonnet",
)
result = await counter.count_tokens(
model_to_use="anthropic.claude-3-sonnet",
messages=[{"role": "user", "content": "hello"}],
contents=None,
deployment={"litellm_params": {}},
request_model="bedrock/anthropic.claude-3-sonnet",
)
assert result is not None
assert result.error is True
assert result.status_code == 429
assert "Rate limit exceeded" in result.error_message
assert result.tokenizer_type == "bedrock_api"
assert result.total_tokens == 0
assert result is not None
assert result.error is True
assert result.status_code == 429
assert "Rate limit exceeded" in result.error_message
assert result.tokenizer_type == "bedrock_api"
assert result.total_tokens == 0
@pytest.mark.asyncio
@ -913,28 +919,20 @@ async def test_bedrock_token_counter_error_propagation_generic_exception():
"""
Test that BedrockTokenCounter returns error response with 500 status for generic exceptions.
"""
counter = BedrockTokenCounter()
counter = BedrockTokenCounter(runtime_handler=_FailingRuntimeHandler(Exception("Unexpected error")))
with patch(
"litellm.llms.bedrock.count_tokens.bedrock_token_counter.BedrockCountTokensHandler"
) as MockHandler:
mock_handler_instance = MockHandler.return_value
mock_handler_instance.handle_count_tokens_request = AsyncMock(
side_effect=Exception("Unexpected error")
)
result = await counter.count_tokens(
model_to_use="anthropic.claude-3-sonnet",
messages=[{"role": "user", "content": "hello"}],
contents=None,
deployment={"litellm_params": {}},
request_model="bedrock/anthropic.claude-3-sonnet",
)
result = await counter.count_tokens(
model_to_use="anthropic.claude-3-sonnet",
messages=[{"role": "user", "content": "hello"}],
contents=None,
deployment={"litellm_params": {}},
request_model="bedrock/anthropic.claude-3-sonnet",
)
assert result is not None
assert result.error is True
assert result.status_code == 500
assert "Unexpected error" in result.error_message
assert result is not None
assert result.error is True
assert result.status_code == 500
assert "Unexpected error" in result.error_message
@pytest.mark.asyncio

28
tests/proxy_unit_tests/test_update_spend.py
View file

@ -320,3 +320,31 @@ async def test_update_spend_logs_multiple_batches_with_failure():
# Verify all logs were cleared from transactions
assert len(prisma_client.spend_log_transactions) == 0
@pytest.mark.asyncio
async def test_update_spend_logs_drops_row_prisma_cannot_encode_and_keeps_the_rest():
"""A row the Prisma engine cannot utf-8 encode (a lone surrogate in a string
column) is permanent, so it must be dropped like any other poison row instead
of being requeued at the head of the queue where it would block every tenant."""
prisma_client = MockPrismaClient()
proxy_logging_obj = create_mock_proxy_logging()
prisma_client.spend_log_transactions = [
{"request_id": "good-1", "request_tags": '["a"]'},
{"request_id": "poison", "request_tags": "\ud800"},
{"request_id": "good-2", "request_tags": '["b"]'},
]
written = []
async def create_many(data, skip_duplicates):
if any("\ud800" in row["request_tags"] for row in data):
raise UnicodeEncodeError("utf-8", "\ud800", 0, 1, "surrogates not allowed")
written.extend(row["request_id"] for row in data)
prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=create_many)
await update_spend(prisma_client, None, proxy_logging_obj)
assert sorted(written) == ["good-1", "good-2"]
assert prisma_client.spend_log_transactions == []
proxy_logging_obj.failure_handler.assert_not_called()

41
tests/router_unit_tests/test_router_helper_utils.py
View file

@ -647,7 +647,12 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) ->
"model": "gpt-3.5-turbo",
"api_key": "sk-must-not-be-recorded",
"messages": [{"role": "user", "content": "hi"}],
metadata_key: {"model_info": {"id": "deployment-1"}, "attempted_retries": 2, "user_api_key": "sk-proxy"},
metadata_key: {
"model_info": {"id": "deployment-1", "base_model": "azure/gpt-3.5-turbo"},
"deployment": "azure/my-35-deployment",
"attempted_retries": 2,
"user_api_key": "sk-proxy",
},
},
e=rate_limit_error,
)
@ -655,6 +660,8 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) ->
{
"model_group": "gpt-3.5-turbo",
"deployment_id": "deployment-1",
"model": "azure/my-35-deployment",
"base_model": "azure/gpt-3.5-turbo",
"exception_type": "RateLimitError",
"exception_string": "litellm.RateLimitError: slow down",
"attempted_retries": 2,
@ -666,6 +673,38 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) ->
assert router.log_retry(kwargs=planted_kwargs, e=rate_limit_error)[metadata_key]["request_retry_count"] == 1
def test_log_retry_reads_base_model_from_the_deployments_litellm_params() -> None:
"""An Azure named deployment declares ``base_model`` under ``litellm_params``, not ``model_info``, and
that is the model the failure row is priced at, so the retry record has to carry it too"""
router = Router(
model_list=[
{
"model_name": "azure-group",
"litellm_params": {
"model": "azure/my-named-deployment",
"api_key": "sk-fake",
"api_base": "https://example.openai.azure.com",
"base_model": "azure/gpt-4o-mini",
},
"model_info": {"id": "dep-azure-named"},
}
]
)
new_kwargs = router.log_retry(
kwargs={
"model": "azure-group",
"metadata": {
"model_info": {"id": "dep-azure-named"},
"deployment": "azure/my-named-deployment",
"attempted_retries": 0,
},
},
e=litellm.Timeout(message="slow", llm_provider="azure", model="azure/my-named-deployment"),
)
record = new_kwargs["metadata"]["previous_models"][0]
assert (record["model"], record["base_model"]) == ("azure/my-named-deployment", "azure/gpt-4o-mini")
def test_update_usage(model_list):
"""Test if the '_update_usage' function is working correctly"""
router = Router(model_list=model_list)

85
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -99,6 +99,31 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re
assert completion_cost == pytest.approx(380 * 9.7e-7)
@pytest.mark.parametrize(
("model_info", "expected_prompt_cost"),
[
({"input_cost_per_token": 1e-7, "input_cost_per_character": 2.5e-8, "output_cost_per_token": 0.0}, 35 * 2.5e-8),
({"input_cost_per_token": 1.5e-7, "output_cost_per_token": 0.0}, 11 * 1.5e-7),
],
ids=["character_priced_model_bills_characters_instead_of_tokens", "token_priced_model_ignores_the_character_count"],
)
def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit(
model_info: ModelInfo, expected_prompt_cost: float
) -> None:
usage = Usage(
prompt_tokens=11,
completion_tokens=0,
total_tokens=11,
prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35),
)
prompt_cost, completion_cost = generic_cost_per_token(
model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info
)
assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0)
def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None:
openai_info: ModelInfo = {"input_cost_per_token": 2e-6}
fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6}
@ -486,6 +511,66 @@ def test_video_tokens_fallback_to_base_cost():
assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12)
def test_prompt_audio_and_video_tokens_without_modality_rates_bill_at_tiered_input_rate():
"""Prompt audio/video tokens with no modality rate are prompt tokens, so a >200k prompt bills them above-200k."""
from unittest.mock import patch
mock_model_info = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_above_200k_tokens": 4e-6,
"output_cost_per_token": 1.2e-5,
"output_cost_per_token_above_200k_tokens": 1.8e-5,
}
usage = Usage(
prompt_tokens=816032,
completion_tokens=4559,
total_tokens=820591,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=9033, audio_tokens=14999, video_tokens=792000),
completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=4559),
)
with patch(
"litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info",
return_value=mock_model_info,
):
prompt_cost, completion_cost = generic_cost_per_token(
model="test-model", usage=usage, custom_llm_provider="vertex_ai"
)
assert round(prompt_cost, 9) == round(816032 * 4e-6, 9)
assert round(completion_cost, 9) == round(4559 * 1.8e-5, 9)
def test_prompt_modality_rates_still_win_over_tiered_input_rate():
"""An explicit input_cost_per_audio_token / input_cost_per_video_token beats the fallback, even above 200k."""
from unittest.mock import patch
mock_model_info = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_above_200k_tokens": 4e-6,
"input_cost_per_audio_token": 3e-6,
"input_cost_per_video_token": 1e-6,
"output_cost_per_token": 1.2e-5,
}
usage = Usage(
prompt_tokens=300000,
completion_tokens=10,
total_tokens=300010,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100000, audio_tokens=100000, video_tokens=100000),
completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=10),
)
with patch(
"litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info",
return_value=mock_model_info,
):
prompt_cost, _ = generic_cost_per_token(model="test-model", usage=usage, custom_llm_provider="vertex_ai")
assert round(prompt_cost, 9) == round(100000 * 4e-6 + 100000 * 3e-6 + 100000 * 1e-6, 9)
def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map):
# gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing
model = "gemini-2.5-pro"

38
tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py
View file

@ -13,6 +13,7 @@ import pytest
import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod
import litellm.proxy.common_request_processing as common_request_processing_mod
from litellm.cost_calculator import _select_model_name_for_cost_calc as select_model_name_for_cost_calc
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
ResponseMetadata,
@ -22,7 +23,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.types.utils import ModelResponse
from litellm.types.utils import ModelResponse, Usage
class TestCallbackDurationMs:
@ -583,3 +584,38 @@ class TestLoggingInitCallbackDuration:
# Should still be set (deep copy of None is essentially a no-op)
assert hasattr(obj, "callback_duration_ms")
assert obj.callback_duration_ms >= 0
def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model():
start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0)
logging_obj: Final = Logging(
model="azure/ptu-deployment",
messages=[{"role": "user", "content": "Hello"}],
stream=False,
call_type="completion",
start_time=start_time,
litellm_call_id="provider-model-response-metadata",
function_id="f",
)
logging_obj.update_environment_variables(
model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure"
)
result: Final = ModelResponse(
model="gpt-5.4-nano-2026-03-17",
usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18),
)
update_response_metadata(
result=result,
logging_obj=logging_obj,
model="azure/ptu-deployment",
kwargs={},
start_time=start_time,
end_time=start_time + datetime.timedelta(seconds=2),
)
result.model = "model-group-alias"
assert (
select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure")
== "azure/gpt-5.4-nano-2026-03-17"
)

101
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -16,7 +16,7 @@ from openai._legacy_response import HttpxBinaryResponseContent
import litellm
from litellm._logging import session_id_var, trace_id_var
from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.cost_calculator import ocr_batch_cost
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
@ -74,6 +74,20 @@ def test_get_masked_api_base(logging_obj):
assert type(masked_api_base) == str
@pytest.mark.parametrize("no_upstream_call", [False, True])
def test_pre_call_stamps_the_provider_handoff_only_when_a_provider_call_is_made(logging_obj, no_upstream_call):
"""The proxy fires pre_call for a failure it raised itself (auth, rate limit, client disconnect)
so callbacks see it; that call never reached a provider, so neither handoff instant may be set."""
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
if no_upstream_call:
logging_obj.model_call_details[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True
logging_obj.pre_call(input="", api_key="")
assert ("api_call_start_time" in logging_obj.model_call_details) is not no_upstream_call
assert (logging_obj.model_call_details.get("first_api_call_start_time") is not None) is not no_upstream_call
def test_pre_call_tolerates_missing_api_base(logging_obj):
"""Presigned batch retrieves (Mistral, Bedrock) build their own URL and pass api_base=None
to pre_call; masking must not raise or the request's pre-call logging is silently lost."""
@ -7287,6 +7301,7 @@ class TestAzurePTUSpilloverCost:
ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id"
SERVED_MODEL: Final = "azure/spill-served-model-ptu"
PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment"
PTU_MODEL_INFO: Final = {
"id": ROUTER_MODEL_ID,
"team_id": "team-1",
@ -7322,7 +7337,9 @@ class TestAzurePTUSpilloverCost:
litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None)
litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None)
def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging:
def _logging_obj(
self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None
) -> LitellmLogging:
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag)
obj = LitellmLogging(
model=self.SERVED_MODEL,
@ -7339,7 +7356,7 @@ class TestAzurePTUSpilloverCost:
optional_params={},
litellm_params={
"api_base": "",
"metadata": {"model_info": model_info},
"metadata": {"model_info": model_info, "deployment": deployment},
"input_cost_per_token": litellm_rate,
"output_cost_per_token": litellm_rate,
},
@ -7409,6 +7426,84 @@ class TestAzurePTUSpilloverCost:
finally:
self._unregister_models()
def test_spillover_payload_reports_the_pay_as_you_go_entry_billing_used(self, monkeypatch) -> None:
self._register_models()
try:
obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
response_cost: Final = obj._response_cost_calculator(result=response)
payload: Final = get_standard_logging_object_payload(
kwargs={
"model": self.SERVED_MODEL,
"custom_llm_provider": "azure",
"litellm_params": obj.litellm_params,
"response_cost": response_cost,
},
init_response_obj=response,
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
logging_obj=obj,
status="success",
)
assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST)
assert payload is not None
model_map_value: Final = payload["model_map_information"]["model_map_value"]
assert model_map_value is not None
assert model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"]
assert model_map_value["output_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"]
assert response_cost == pytest.approx(
100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"]
)
finally:
self._unregister_models()
def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None:
self._register_models()
litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"}
try:
obj = self._logging_obj(
dict(self.PTU_MODEL_INFO),
flag="True",
litellm_rate=0.0,
monkeypatch=monkeypatch,
deployment=self.PTU_DEPLOYMENT,
)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
response._hidden_params["provider_response_model"] = response.model
response_cost: Final = obj._response_cost_calculator(result=response)
response.model = "spill-model-group-alias"
payload: Final = get_standard_logging_object_payload(
kwargs={
"model": self.SERVED_MODEL,
"custom_llm_provider": "azure",
"litellm_params": obj.litellm_params,
"response_cost": response_cost,
},
init_response_obj=response,
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
logging_obj=obj,
status="success",
)
assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST)
assert payload is not None
model_map_information: Final = payload["model_map_information"]
assert model_map_information["model_map_key"] != self.PTU_DEPLOYMENT
model_map_value: Final = model_map_information["model_map_value"]
assert model_map_value is not None
assert response_cost == pytest.approx(
100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"]
)
finally:
litellm.model_cost.pop(self.PTU_DEPLOYMENT, None)
self._unregister_models()
def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None:
self._register_models()
custom_model_id: Final = "non-ptu-custom-router-model-id"

91
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -1,3 +1,4 @@
import copy
import pytest
@ -17,7 +18,7 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
from litellm.types.utils import ServerToolUse, Usage
@ -6451,3 +6452,91 @@ def test_eager_input_streaming_reaches_anthropic_request_tools():
assert result["tools"][0]["eager_input_streaming"] is True
assert result["tools"][0]["name"] == "write_file"
def _make_transform_request(optional_params: dict, litellm_params: dict) -> dict:
return AnthropicConfig().transform_request(
model="claude-3-5-sonnet-20241022",
messages=[{"role": "user", "content": "hi"}],
optional_params=optional_params,
litellm_params=litellm_params,
headers={},
)
def _two_function_tools() -> list[dict]:
return [
{
"type": "function",
"function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}},
}
for i in range(2)
]
@pytest.mark.parametrize(
"control, expected_control",
[
(None, {"type": "ephemeral"}),
({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}),
],
)
def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control):
"""LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which
Anthropic rejected with 400 Extra inputs are not permitted."""
tools = _two_function_tools()
tools_before = copy.deepcopy(tools)
point = {"location": "tool_config", **({"control": control} if control is not None else {})}
litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}}
data = _make_transform_request(
optional_params={"tools": tools, "cache_control_injection_points": [point]},
litellm_params=litellm_params,
)
assert "cache_control_injection_points" not in data
assert data["tools"][-1]["cache_control"] == expected_control
assert "cache_control" not in data["tools"][0]
assert tools == tools_before
assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1"
def test_transform_request_tool_config_point_keeps_the_clients_tool_marker():
tools = _two_function_tools()
tools[0]["cache_control"] = {"type": "ephemeral"}
data = _make_transform_request(
optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]},
litellm_params={},
)
assert "cache_control_injection_points" not in data
assert data["tools"][0]["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in data["tools"][1]
def test_transform_request_tool_config_point_skips_tool_search_tools():
tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES))
tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}]
data = _make_transform_request(
optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]},
litellm_params={},
)
assert data["tools"][1]["cache_control"] == {"type": "ephemeral"}
assert "cache_control" not in data["tools"][2]
assert "cache_control" not in data["tools"][0]
def test_transform_request_tool_config_point_without_tools_drops_the_key():
litellm_params = {"metadata": {}}
data = _make_transform_request(
optional_params={"cache_control_injection_points": [{"location": "tool_config"}]},
litellm_params=litellm_params,
)
assert "cache_control_injection_points" not in data
assert "tools" not in data
assert litellm_params["metadata"] == {}

48
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -1,3 +1,4 @@
import copy
import asyncio
import json
import os
@ -1554,7 +1555,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict
return safeguards, safeguard_results
def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler:
def _upstream_answering_with(
safeguard_results: list[dict[str, object]], captured: dict[str, object]
) -> AsyncHTTPHandler:
def upstream_records_the_request(request: httpx.Request) -> httpx.Response:
captured["body"] = json.loads(request.content)
captured["anthropic-beta"] = request.headers.get("anthropic-beta")
@ -1580,7 +1583,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture
_CLIENT_BETA_HEADERS: Final = (
pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"),
pytest.param(
{"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"
),
pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"),
pytest.param({}, id="client_sends_no_beta_header"),
)
@ -1698,3 +1703,42 @@ async def test_anthropic_messages_adds_no_dangerous_tool_use_beta_to_azure_ai_fo
assert "safeguards" not in captured["body"]
assert "dangerous-tool-use-2026-09-03" not in (captured["anthropic-beta"] or "").split(",")
def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch):
"""LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the
native path dropped it silently, so the tools went out without any cache breakpoint."""
from litellm.llms.anthropic.experimental_pass_through.messages import handler
captured = {}
def fake_native(**kwargs):
captured["params"] = kwargs["anthropic_messages_optional_request_params"]
captured["kwargs"] = kwargs["kwargs"]
captured["messages"] = kwargs["messages"]
return "native-passthrough"
monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native)
tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)]
tools_before = copy.deepcopy(tools)
result = handler.anthropic_messages_handler(
max_tokens=100,
messages=[{"role": "user", "content": "Hello"}],
model="anthropic/claude-opus-4-6",
api_key="sk-test",
tools=tools,
litellm_metadata={},
cache_control_injection_points=[
{"location": "message", "role": "user"},
{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}},
],
)
assert result == "native-passthrough"
assert "cache_control_injection_points" not in captured["params"]
assert "cache_control_injection_points" not in captured["kwargs"]
assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
assert "cache_control" not in captured["params"]["tools"][0]
assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"}
assert tools == tools_before

109
tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py Normal file
View file

@ -0,0 +1,109 @@
import json
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
MANTLE_COUNT_URL: Final = "https://bedrock-mantle.us-east-1.api.aws/anthropic/v1/messages/count_tokens"
LITELLM_PARAMS: Final = {
"aws_access_key_id": "AKIATESTACCESSKEY",
"aws_secret_access_key": "test-secret",
"aws_region_name": "us-east-1",
}
REQUEST: Final[dict[str, object]] = {
"model": "global.anthropic.claude-opus-4-8",
"messages": [{"role": "user", "content": "The quick brown fox"}],
"system": "You are a terse assistant.",
"tools": [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}],
}
class _MantleEndpoint:
def __init__(self, status_code: int, body: Mapping[str, object]) -> None:
self.status_code: Final = status_code
self.body: Final = body
self.requests: tuple[httpx.Request, ...] = ()
def __call__(self, request: httpx.Request) -> httpx.Response:
self.requests = (*self.requests, request)
return httpx.Response(self.status_code, json=dict(self.body), request=request)
def only_request(self) -> httpx.Request:
assert len(self.requests) == 1, self.requests
return self.requests[0]
def _client(endpoint: _MantleEndpoint) -> AsyncHTTPHandler:
return AsyncHTTPHandler(transport=httpx.MockTransport(endpoint))
@pytest.fixture(autouse=True)
def _sigv4_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
@pytest.mark.asyncio
async def test_counts_on_mantle_with_the_base_model_and_the_deployment_credentials() -> None:
mantle: Final = _MantleEndpoint(200, {"input_tokens": 2177})
result: Final = await BedrockMantleCountTokensHandler().handle_count_tokens_request(
request_data=dict(REQUEST),
litellm_params=dict(LITELLM_PARAMS),
resolved_model="anthropic.claude-opus-4-8",
client=_client(mantle),
)
assert result == {"input_tokens": 2177}
posted: Final = mantle.only_request()
assert str(posted.url) == MANTLE_COUNT_URL
assert posted.headers["Authorization"].startswith("AWS4-HMAC-SHA256")
assert "us-east-1/bedrock/aws4_request" in posted.headers["Authorization"]
assert posted.headers["anthropic-version"] == "2023-06-01"
assert json.loads(posted.content) == {
"model": "anthropic.claude-opus-4-8",
"messages": REQUEST["messages"],
"system": REQUEST["system"],
"tools": REQUEST["tools"],
}
@pytest.mark.asyncio
async def test_bedrock_mantle_api_base_env_names_the_host(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", "https://vpce-abc.bedrock-mantle.us-east-1.vpce.api.aws")
mantle: Final = _MantleEndpoint(200, {"input_tokens": 3})
await BedrockMantleCountTokensHandler().handle_count_tokens_request(
request_data=dict(REQUEST),
litellm_params=dict(LITELLM_PARAMS),
resolved_model="anthropic.claude-opus-4-8",
client=_client(mantle),
)
assert (
str(mantle.only_request().url)
== "https://vpce-abc.bedrock-mantle.us-east-1.vpce.api.aws/anthropic/v1/messages/count_tokens"
)
@pytest.mark.asyncio
async def test_non_200_answers_raise_bedrock_error_with_mantle_status() -> None:
mantle: Final = _MantleEndpoint(
404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}
)
with pytest.raises(BedrockError) as raised:
await BedrockMantleCountTokensHandler().handle_count_tokens_request(
request_data=dict(REQUEST),
litellm_params=dict(LITELLM_PARAMS),
resolved_model="anthropic.claude-sonnet-5-5",
client=_client(mantle),
)
assert raised.value.status_code == 404
assert "does not exist" in raised.value.message

176
tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_token_counter.py Normal file
View file

@ -0,0 +1,176 @@
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
from pydantic import JsonValue, TypeAdapter
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
UNSUPPORTED: Final = {"message": "The provided model doesn't support counting tokens."}
DEPLOYMENT: Final = {
"litellm_params": {
"aws_access_key_id": "AKIATESTACCESSKEY",
"aws_secret_access_key": "test-secret",
"aws_region_name": "us-east-1",
}
}
MESSAGES: Final = [{"role": "user", "content": "The quick brown fox jumps over the lazy dog."}]
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
class _Bedrock:
def __init__(self, runtime: tuple[int, Mapping[str, object]], mantle: tuple[int, Mapping[str, object]]) -> None:
self.runtime: Final = runtime
self.mantle: Final = mantle
self.requests: tuple[httpx.Request, ...] = ()
def __call__(self, request: httpx.Request) -> httpx.Response:
self.requests = (*self.requests, request)
status_code, body = self.runtime if request.url.host == RUNTIME_HOST else self.mantle
return httpx.Response(status_code, json=dict(body), request=request)
def posted_hosts(self) -> tuple[str, ...]:
return tuple(request.url.host for request in self.requests)
def _counter(bedrock: _Bedrock) -> BedrockTokenCounter:
return BedrockTokenCounter(client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)))
@pytest.fixture(autouse=True)
def _sigv4_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
@pytest.mark.asyncio
async def test_claude_model_bedrock_runtime_cannot_count_is_counted_on_mantle() -> None:
bedrock: Final = _Bedrock(runtime=(400, UNSUPPORTED), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="global.anthropic.claude-opus-4-8",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-opus-4-8",
system="You are a terse assistant.",
)
assert result is not None
assert result.error is False
assert result.total_tokens == 2177
assert result.tokenizer_type == "bedrock_mantle_api"
assert result.original_response == {"input_tokens": 2177}
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
mantle_body: Final = _JSON_BODY.validate_json(bedrock.requests[1].content)
assert mantle_body["model"] == "anthropic.claude-opus-4-8"
assert mantle_body["system"] == "You are a terse assistant."
@pytest.mark.asyncio
async def test_bedrock_runtime_count_is_kept_when_it_answers() -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 1353}), mantle=(200, {"input_tokens": 1336}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="global.anthropic.claude-sonnet-4-6",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-sonnet-4-6",
)
assert result is not None
assert result.total_tokens == 1353
assert result.tokenizer_type == "bedrock_api"
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model_to_use", "runtime"),
(
("global.anthropic.claude-opus-4-8", (401, {"Message": "not authorized to perform: bedrock:CountTokens"})),
("global.anthropic.claude-opus-4-8", (403, {"Message": "not authorized to perform: bedrock:CountTokens"})),
("amazon.nova-pro-v1:0", (400, UNSUPPORTED)),
("amazon.nova-pro-v1:0", (404, UNSUPPORTED)),
),
)
async def test_other_bedrock_runtime_failures_are_not_retried_on_mantle(
model_to_use: str, runtime: tuple[int, Mapping[str, object]]
) -> None:
bedrock: Final = _Bedrock(runtime=runtime, mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use=model_to_use,
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model=model_to_use,
)
assert result is not None
assert result.error is True
assert result.status_code == runtime[0]
assert result.tokenizer_type == "bedrock_api"
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
@pytest.mark.asyncio
async def test_mantle_failure_is_reported_with_its_status() -> None:
bedrock: Final = _Bedrock(
runtime=(404, UNSUPPORTED),
mantle=(404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}),
)
result: Final = await _counter(bedrock).count_tokens(
model_to_use="global.anthropic.claude-sonnet-5-5",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-sonnet-5-5",
)
assert result is not None
assert result.error is True
assert result.status_code == 404
assert result.tokenizer_type == "bedrock_mantle_api"
assert "does not exist" in (result.error_message or "")
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
@pytest.mark.asyncio
async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> None:
bedrock: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177}))
tools: Final = [
{
"name": "lookup",
"description": "Find a record",
"input_schema": {"type": "object", "properties": {"query": {"type": "string"}}},
}
]
result: Final = await _counter(bedrock).count_tokens(
model_to_use="global.anthropic.claude-opus-4-8",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-opus-4-8",
system="You are a terse assistant.",
tools=tools,
)
assert result is not None
assert result.error is False
assert result.total_tokens == 2177
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
mantle_body: Final = _JSON_BODY.validate_json(bedrock.requests[1].content)
assert mantle_body == {
"model": "anthropic.claude-opus-4-8",
"messages": MESSAGES,
"system": "You are a terse assistant.",
"tools": tools,
}

36
tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py
View file

@ -1210,3 +1210,39 @@ class TestCoerceNumericFormFields:
numeric_fields=self.numeric_fields,
)
assert result == {"n": 3, "temperature": None, "image": buffer}
@pytest.mark.asyncio
async def test_form_data_metadata_with_lone_surrogate_escape_is_rejected_like_a_json_body():
"""``json.loads`` accepts ``"\\ud800"`` and yields a lone surrogate that no
provider or database can utf-8 encode. The JSON body path already answers
400, so the multipart ``metadata`` field must too, instead of letting the
value through to the spend log writers."""
mock_request = MagicMock()
mock_request.form = AsyncMock(
return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["poison-\\ud800-tag"]}'})
)
mock_request.headers = {"content-type": "multipart/form-data"}
mock_request.scope = {}
mock_request.state._cached_headers = None
with pytest.raises(ProxyException) as exc_info:
await _read_request_body(mock_request)
assert exc_info.value.code == "400"
assert "metadata" in exc_info.value.message
@pytest.mark.asyncio
async def test_form_data_metadata_keeps_non_ascii_and_valid_surrogate_pairs():
mock_request = MagicMock()
mock_request.form = AsyncMock(
return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["caf\\u00e9", "\\ud83d\\ude00"]}'})
)
mock_request.headers = {"content-type": "multipart/form-data"}
mock_request.scope = {}
mock_request.state._cached_headers = None
result = await _read_request_body(mock_request)
assert result["metadata"]["tags"] == ["café", "😀"]

51
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -3562,26 +3562,33 @@ async def test_commit_spend_updates_retries_deadlock_on_every_entity_path(monkey
@pytest.mark.asyncio
@pytest.mark.parametrize(
"call_type, expects_flush",
[("aresponses", True), ("responses", True), ("acompletion", False)],
"call_type, model_id, expects_flush",
[
("aresponses", "dep-bridged", True),
("responses", "dep-bridged", True),
("aresponses", "dep-forced-bridge", True),
("aresponses", None, True),
("aresponses", "dep-native", False),
("responses", "dep-native", False),
("acompletion", "dep-bridged", False),
],
)
async def test_insert_spend_log_asks_for_an_immediate_flush_on_rows_other_workers_read_back(
call_type: str, expects_flush: bool
async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bridge_reads_back(
call_type: str, model_id: str | None, expects_flush: bool
):
"""
A `previous_response_id` chained straight off the previous turn reads the DB, so a
Responses row cannot sit in this worker's queue until the monitor's next poll.
A `previous_response_id` chained straight off a bridged turn reads that turn's row from the
DB, so it cannot sit in this worker's queue until the monitor's next poll. A native Responses
provider holds the conversation itself, so its row waits for the batch write like a chat row.
"""
db_writer = DBSpendUpdateWriter()
prisma = _tool_usage_prisma()
prisma.spend_log_flush_requested = asyncio.Event()
payload = {"request_id": "req-1", "call_type": call_type, "model_id": model_id}
await db_writer._insert_spend_log_to_db(
payload={"request_id": "req-1", "call_type": call_type},
prisma_client=prisma,
)
await db_writer._insert_spend_log_to_db(payload=payload, prisma_client=prisma, llm_router=_responses_router())
assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}]
assert prisma.spend_log_transactions == [payload]
assert prisma.spend_log_flush_requested.is_set() is expects_flush
@ -3980,3 +3987,25 @@ async def test_daily_transaction_attributes_caching_savings_only_with_an_injecti
assert transaction["cache_creation_input_tokens"] == 1111
assert transaction["prompt_caching_savings_spend"] != 0.0
assert transaction["gateway_injected_caching_savings_spend"] == 0.0
def _responses_router() -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "native",
"litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test"},
"model_info": {"id": "dep-native"},
},
{
"model_name": "bridged",
"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"},
"model_info": {"id": "dep-bridged"},
},
{
"model_name": "forced-bridge",
"litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test", "use_chat_completions_api": True},
"model_info": {"id": "dep-forced-bridge"},
},
]
)

28
tests/test_litellm/proxy/google_endpoints/test_google_api_endpoints.py
View file

@ -3,6 +3,7 @@
Test to verify the Google GenAI proxy API endpoints
"""
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -179,3 +180,30 @@ def test_google_count_tokens_unchanged():
assert response.status_code == 200
body = response.json()
assert body["totalTokens"] == 7
def test_google_count_tokens_uses_normalized_total_when_provider_shape_differs():
"""A provider that counts in Anthropic shape (Bedrock, Anthropic) carries no totalTokens key, so the normalized total must win over the raw response's missing field."""
try:
client: Final = _build_test_client()
except ImportError as e:
pytest.skip(f"Skipping test due to missing dependency: {e}")
fake_response: Final = MagicMock()
fake_response.original_response = {"input_tokens": 2167}
fake_response.total_tokens = 2167
with patch(
"litellm.proxy.proxy_server.token_counter",
new_callable=AsyncMock,
return_value=fake_response,
):
response: Final = client.post(
"/v1beta/models/claude-opus-4-8:countTokens",
json={"contents": [{"role": "user", "parts": [{"text": "Hello"}]}]},
)
assert response.status_code == 200
body: Final = response.json()
assert body["totalTokens"] == 2167
assert body["promptTokensDetails"] == []

151
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -28,7 +28,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
@pytest.mark.asyncio
@ -94,6 +94,65 @@ async def test_async_post_call_failure_hook():
assert metadata["original_key"] == "original_value"
@pytest.mark.asyncio
async def test_each_earlier_billed_attempt_gets_its_own_spend_row():
"""A timed-out request that the router retried writes one row per billed attempt: the final
attempt under the request id, every earlier Timeout attempt under ``<id>-attempt-<n>`` with its
own cost, attempt number and deployment."""
from litellm.proxy.utils import BilledFailedAttempt
logger = _ProxyDBLogger()
user_api_key_dict = UserAPIKeyAuth(api_key="test_api_key", user_id="u1", team_id="t1")
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
request_data = {
"model": "gpt-4.1",
"litellm_call_id": "call-1",
"standard_logging_object": {"id": "call-1", "model_id": "dep-final"},
"metadata": {
"attempted_retries": 2,
"model_info": {"id": "dep-final", "input_cost_per_token": 0.5},
"previous_models": ({"exception_type": "Timeout"}, {"exception_type": "Timeout"}),
},
"combined_usage_object": usage,
"response_cost": 0.5,
"billed_failed_attempts": (
BilledFailedAttempt(attempted_retries=0, deployment_id="dep-a", usage=usage, response_cost=0.25),
BilledFailedAttempt(attempted_retries=0, deployment_id="dep-final", usage=usage, response_cost=0.5),
),
}
with patch(
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
new_callable=AsyncMock,
) as mock_update_database:
await logger.async_post_call_failure_hook(
request_data=request_data,
original_exception=litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai"),
user_api_key_dict=user_api_key_dict,
)
rows = [call.kwargs for call in mock_update_database.call_args_list]
assert [row["response_cost"] for row in rows] == [0.5, 0.25, 0.5]
final_row, attempt_0, attempt_1 = (row["kwargs"] for row in rows)
assert final_row["litellm_call_id"] == "call-1"
assert final_row["litellm_params"]["metadata"]["attempted_retries"] == 2
assert attempt_0["litellm_call_id"] == "call-1-attempt-0"
assert attempt_0["standard_logging_object"]["id"] == "call-1-attempt-0"
assert attempt_0["litellm_params"]["metadata"]["attempted_retries"] == 0
assert attempt_0["litellm_params"]["metadata"]["model_info"]["id"] == "dep-a"
assert "previous_models" not in attempt_0["litellm_params"]["metadata"]
assert "billed_failed_attempts" not in attempt_0
assert attempt_0["combined_usage_object"] is usage
assert attempt_1["litellm_call_id"] == "call-1-attempt-1"
assert attempt_1["litellm_params"]["metadata"]["attempted_retries"] == 0
assert attempt_1["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final"
assert final_row["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final"
for row in rows:
assert row["token"] == "test_api_key"
assert row["user_id"] == "u1"
assert row["team_id"] == "t1"
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata():
"""
@ -2693,3 +2752,93 @@ async def test_batch_accounting_does_not_retry_after_settling_a_budget_reservati
)
assert reservation["finalized"] is True
assert outcome.retryable_failure is False
class _SharedCounter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
return self.totals[key]
def _grounded_gemini_kwargs(response_cost: float) -> dict:
return {
"model": "vertex_ai/gemini-test",
"call_type": "acompletion",
"response_cost": response_cost,
"litellm_params": {"metadata": {"user_api_key": "sk-test", "user_api_key_user_id": "user-1"}},
"standard_logging_object": {
"response_cost": response_cost,
"request_tags": None,
"cache_hit": None,
"custom_llm_provider": "vertex_ai",
"cost_breakdown": {
"input_cost": 0.0,
"output_cost": 0.0,
"tool_usage_cost": 0.5,
"total_cost": response_cost,
},
"model_map_information": {
"model_map_key": "vertex_ai/gemini-test",
"model_map_value": {
"litellm_provider": "vertex_ai",
"mode": "chat",
"web_search_billing_unit": "per_query",
"search_context_cost_per_query": {"search_context_size_medium": 0.1},
"search_free_requests": {"count": 100, "period": "month", "shared_group": "test"},
},
},
},
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.parametrize("opted_in", [True, False])
async def test_track_cost_callback_frees_grounding_under_the_allowance_only_when_opted_in(monkeypatch, opted_in):
monkeypatch.setattr(litellm, "apply_search_free_tier", opted_in, raising=False)
counter = _SharedCounter()
kwargs = _grounded_gemini_kwargs(response_cost=1.0)
response = ModelResponse(
usage=Usage(
prompt_tokens=10,
completion_tokens=5,
total_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=5),
)
)
with (
patch( # test-quality-ok: increment_spend_counters is a proxy_server global the callback reads lazily, no seam
"litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock
),
patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam
"litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock
),
patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam
"litellm.proxy.proxy_server.proxy_logging_obj"
) as mock_proxy_logging,
):
mock_proxy_logging.failed_tracking_alert = AsyncMock()
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True)
mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
mock_proxy_logging.internal_usage_cache.dual_cache.redis_cache = counter
await _ProxyDBLogger()._PROXY_track_cost_callback(
kwargs=kwargs, completion_response=response, start_time=datetime.now(), end_time=datetime.now()
)
await asyncio.sleep(0)
mock_proxy_logging.failed_tracking_alert.assert_not_called()
mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once()
breakdown = kwargs["standard_logging_object"]["cost_breakdown"]
if opted_in:
assert kwargs["response_cost"] == pytest.approx(1.0 - 5 * 0.1)
assert breakdown["original_cost"] == 1.0 and breakdown["discount_amount"] == pytest.approx(5 * 0.1)
assert sum(counter.totals.values()) == 5
else:
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in breakdown
assert counter.totals == {}

210
tests/test_litellm/proxy/hooks/test_search_free_tier.py Normal file
View file

@ -0,0 +1,210 @@
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from litellm.proxy.hooks.search_free_tier import (
apply_search_free_tier,
freed_units,
search_free_tier_key,
search_free_tier_ttl,
)
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc)
RATE: Final = 0.01
class _Counter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
self.ttls: dict[str, int | None] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
self.ttls[key] = ttl
return self.totals[key]
class _BrokenCounter:
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
raise ConnectionError("redis is down")
def _model_info(billing_unit: str, count: int, period: str = "month", provider: str = "vertex_ai") -> dict:
return {
"litellm_provider": provider,
"mode": "chat",
"web_search_billing_unit": billing_unit,
"search_context_cost_per_query": {"search_context_size_medium": RATE},
"search_free_requests": {"count": count, "period": period, "shared_group": "test-group"},
}
def _response(queries: int | None) -> ModelResponse:
details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None
return ModelResponse(
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details)
)
def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]:
sl_object = {
"response_cost": response_cost,
"cache_hit": cache_hit,
"custom_llm_provider": model_info["litellm_provider"],
"cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost},
"model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info},
}
return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object
@pytest.mark.parametrize(
("allowance", "units", "total_after", "expected"),
[
(5000, 5, 5, 5),
(5000, 5, 5000, 5),
(5000, 10, 5005, 5),
(5000, 5, 5010, 0),
(2, 5, 5, 2),
(0, 3, 3, 0),
],
)
def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected):
assert freed_units(allowance, units, total_after) == expected
def test_counter_key_is_per_provider_group_and_utc_period():
assert search_free_tier_key("gemini", {"count": 1, "period": "month", "shared_group": "g3"}, NOW) == (
"litellm:search_free_tier:gemini:g3:2026-10"
)
assert search_free_tier_key("vertex_ai", {"count": 1, "period": "day", "shared_group": "g2"}, NOW) == (
"litellm:search_free_tier:vertex_ai:g2:2026-10-20"
)
@pytest.mark.parametrize(
("period", "boundary"),
[("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))],
)
def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary):
expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW))
assert boundary < expiry < boundary + timedelta(hours=2)
@pytest.mark.asyncio
async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call():
counter = _Counter()
model_info = _model_info("per_query", count=7)
reduced = []
for _ in range(3):
kwargs, sl_object = _call(model_info, response_cost=1.0)
cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"]))
(c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced
assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE)
assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE)
assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5
assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE)
assert b2["discount_amount"] == pytest.approx(2 * RATE)
assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 15}
assert counter.ttls["litellm:search_free_tier:vertex_ai:test-group:2026-10"] > 0
@pytest.mark.asyncio
async def test_gemini_api_and_vertex_ai_grounding_draw_on_separate_allowances():
counter = _Counter()
gemini_api = _model_info("per_query", count=3, provider="gemini")
vertex_ai = _model_info("per_query", count=3, provider="vertex_ai")
kwargs, sl_object = _call(gemini_api, response_cost=1.0)
await apply_search_free_tier(kwargs, sl_object, 1.0, _response(2), counter, NOW)
kwargs, sl_object = _call(vertex_ai, response_cost=1.0)
vertex_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(4), counter, NOW)
kwargs, sl_object = _call(gemini_api, response_cost=1.0)
gemini_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(1), counter, NOW)
assert vertex_cost == pytest.approx(1.0 - 3 * RATE), "vertex ai had its whole allowance left"
assert gemini_cost == pytest.approx(1.0 - RATE), "gemini api had one free query left"
@pytest.mark.asyncio
async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count():
counter = _Counter()
model_info = _model_info("per_prompt", count=1, period="day")
kwargs, sl_object = _call(model_info, response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
kwargs, sl_object = _call(model_info, response_cost=1.0)
second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
assert first == pytest.approx(1.0 - RATE)
assert second == 1.0
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10-20": 2}
@pytest.mark.asyncio
async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
response = {
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
"prompt_tokens_details": {"web_search_requests": 3},
}
}
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", [None, _BrokenCounter()])
async def test_full_price_when_no_counter_is_available(counter):
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()])
async def test_requests_without_grounding_never_touch_the_counter(response):
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_models_without_an_allowance_never_touch_the_counter():
counter = _Counter()
model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"}
kwargs, sl_object = _call(model_info, response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_a_second_callback_run_on_the_same_request_counts_it_once():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW)
assert first == second == pytest.approx(1.0 - 5 * RATE)
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5}
@pytest.mark.asyncio
async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True)
assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0
assert counter.totals == {}
assert "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
async def test_freed_cost_never_exceeds_the_billed_cost():
counter = _Counter()
billed = 3 * RATE
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed)
assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0
assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed)
assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5}

26
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -1243,6 +1243,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value(
assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected
def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier():
litellm_params = {
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
}
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params)
== 20.0
)
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params)
== 950.0
)
assert (
resolve_llm_passthrough_timeout(
kwargs={"service_tier": "flex", "stream": True},
litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}},
)
== 30.0
)
def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner():
with pytest.raises(ValidationError):
resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)})

112
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -7416,6 +7416,118 @@ def _apply_stream_usage_tracking(
)
async def _start_partial_responses_stream(self, recorder: _RecordingSuccessLogger, *, relay_text: bool = True):
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
mocked: Final = await litellm.aresponses(
model="gpt-4o-mini",
input="tell me a story",
mock_response="The codename is AZURE-FALCON-42 and the story is long.",
stream=True,
api_key="test-key",
)
logging_obj: Final = mocked.logging_obj
logging_obj.dynamic_async_success_callbacks = [recorder]
stream: Final = ResponsesAPIStreamingIterator(
response=httpx.Response(200),
model="gpt-4o-mini",
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=logging_obj,
request_data={"model": "gpt-4o-mini", "input": "tell me a story"},
custom_llm_provider="openai",
)
created: Final = {
"type": "response.created",
"sequence_number": 0,
"response": {
"id": "resp_partial",
"object": "response",
"created_at": 1700000000.0,
"model": "gpt-4o-mini",
"status": "in_progress",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"service_tier": "priority",
},
}
delta: Final = {
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_partial",
"output_index": 0,
"content_index": 0,
"delta": "The codename is AZURE-FALCON-42",
}
for event in (created, delta) if relay_text else (created,):
stream._process_chunk(json.dumps(event))
return stream
async def _bill_and_wait(self, recorder: _RecordingSuccessLogger, streamed: object, logging_obj: object) -> bool:
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed)
for _ in range(50):
if recorder.success_events:
break
await asyncio.sleep(0.1)
await asyncio.sleep(0.5)
return billed
@pytest.mark.asyncio
async def test_disconnect_bills_partial_native_responses_stream_with_counted_usage(self):
from litellm.types.llms.openai import ResponsesAPIResponse
recorder: Final = _RecordingSuccessLogger()
stream: Final = await self._start_partial_responses_stream(recorder)
billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert isinstance(partial_response, ResponsesAPIResponse)
assert partial_response.status == "incomplete"
assert getattr(partial_response, "service_tier") == "priority"
assert partial_response.usage.output_tokens == litellm.token_counter(
model="gpt-4o-mini", text="The codename is AZURE-FALCON-42"
)
assert partial_response.usage.input_tokens > 0
@pytest.mark.asyncio
async def test_disconnect_bills_partial_default_route_messages_stream_through_the_responses_adapter(self):
from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import (
AnthropicResponsesStreamWrapper,
)
from litellm.router import FallbackAwareAnthropicMessagesStream
from litellm.types.llms.openai import ResponsesAPIResponse
async def _sse_frames() -> AsyncGenerator[bytes, None]:
yield b"event: message_start\n\n"
recorder: Final = _RecordingSuccessLogger()
stream: Final = await self._start_partial_responses_stream(recorder)
wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=stream, model="gpt-4o-mini")
streamed: Final = FallbackAwareAnthropicMessagesStream(_sse_frames(), wrapper.async_anthropic_sse_wrapper())
billed: Final = await self._bill_and_wait(recorder, streamed, stream.logging_obj)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert isinstance(partial_response, ResponsesAPIResponse)
assert partial_response.usage.output_tokens == litellm.token_counter(
model="gpt-4o-mini", text="The codename is AZURE-FALCON-42"
)
@pytest.mark.asyncio
async def test_disconnect_before_any_responses_output_bills_nothing(self):
recorder: Final = _RecordingSuccessLogger()
stream: Final = await self._start_partial_responses_stream(recorder, relay_text=False)
billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj)
assert billed is False
assert recorder.success_events == []
class TestApplyStreamUsageTracking:
def test_default_injects_usage_and_marks_strip_for_chat_completions(self):
data = {"stream": True, "model": "gpt-5.4-nano"}

2
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -7546,6 +7546,7 @@ _PLANTED_STAMPS = {
"attempted_fallbacks": 99,
"original_model_group": "spoofed-group",
"request_retry_count": -100,
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
"_client_output_ceiling": {"api_base": "https://attacker.example"},
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9,
"client_key": "client_value",
@ -7580,6 +7581,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
assert "original_model_group" not in updated["metadata"]
assert "_client_output_ceiling" not in updated["metadata"]
assert "request_retry_count" not in updated["metadata"]
assert "previous_models" not in updated["metadata"]
assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"]
assert updated["metadata"]["client_key"] == "client_value"

20
tests/test_litellm/proxy/test_pricing_field_strip.py
View file

@ -58,7 +58,7 @@ class TestStripClientPricingOverrides:
# The strip set is built from the model so additions are picked up
# automatically — this test guards against the model and the strip
# set drifting apart if someone replaces the auto-derivation later.
assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset(
assert _CLIENT_PRICING_CONTROL_FIELDS >= frozenset(
CustomPricingLiteLLMParams.model_fields.keys()
)
# Sanity: the obvious top-level pricing fields are in the set.
@ -84,6 +84,22 @@ class TestStripClientPricingOverrides:
"messages": [{"role": "user", "content": "hi"}],
}
def test_root_base_model_dropped(self):
# ``base_model`` picks the cost-map entry a spend row is priced with
# (success rows via the standard logging payload, billed failure rows
# via the failure lift), so a caller naming a cheaper model lowers
# their own recorded spend.
data = {
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hi"}],
"base_model": "gpt-4o-mini",
}
_strip_client_pricing_overrides(data)
assert data == {
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hi"}],
}
def test_metadata_model_info_dropped(self):
data = {
"model": "gpt-4",
@ -264,6 +280,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():
"model": "gpt-4",
"messages": [{"role": "user", "content": "hi"}],
"input_cost_per_token": 0.0001,
"base_model": "gpt-4o-mini",
"metadata": {"model_info": {"output_cost_per_token": 0.0002}},
}
@ -279,6 +296,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():
assert updated["input_cost_per_token"] == 0.0001
assert updated["metadata"]["model_info"] == {"output_cost_per_token": 0.0002}
assert updated["base_model"] == "gpt-4o-mini"
@pytest.mark.asyncio

673
tests/test_litellm/proxy/test_proxy_utils.py
View file

@ -2,6 +2,7 @@ import datetime as real_datetime
import smtplib
from typing import Final
import httpx
import pytest
from fastapi import HTTPException
@ -441,6 +442,47 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime:
assert "litellm_logging_obj" not in request_data
class TestPostCallFailureHookDoesNotInventAProviderHandoff:
"""A failure the proxy raised before any provider call (here an auth error) is logged through
_handle_logging_proxy_only_error, which fires pre_call for the callbacks. That must not stamp a
handoff instant onto the request, or every downstream reader (preprocessing latency, the
dispatched check behind the failure-row estimate) sees a provider call that never happened."""
@pytest.mark.asyncio
async def test_proxy_only_failure_lifts_no_first_api_call_start_time(self):
from datetime import datetime
from unittest.mock import AsyncMock, patch
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth
logging_obj = Logging(
model="gpt-4.1",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="acompletion",
start_time=datetime.now(),
litellm_call_id="pre-handoff-auth-failure",
function_id="pre-handoff",
)
request_data: Final[dict[str, object]] = {"litellm_logging_obj": logging_obj, "model": "gpt-4.1", "metadata": {}}
proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
with (
patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()),
patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()),
):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=Exception("Authentication Error, invalid key"),
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"),
error_type=ProxyErrorTypes.auth_error,
)
assert request_data.get("first_api_call_start_time") is None
assert "api_call_start_time" not in logging_obj.model_call_details
class TestPostCallFailureHookLiftsRecoveredPartialSpend:
"""A stream that broke mid-flight still billed the provider for the chunks
already delivered. The streaming handler stashes that recovered usage and
@ -685,11 +727,13 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
"""A non-stream request that failed after dispatch (timeout, provider
error) consumed provider-billed input tokens but recovered no usage.
post_call_failure_hook must estimate the input side onto request_data so
the spend log's failure row records what was sent instead of zero, while
never charging spend for the failure (LIT-5690).
the spend log's failure row records what was sent instead of zero
(LIT-5690). On a timeout that estimate is priced at the model's input
rate since the provider bills the prompt it accepted (LIT-145); other
post-dispatch failures keep zero spend.
"""
async def _run(self, request_data):
async def _run(self, request_data, original_exception: Exception | None = None):
from unittest.mock import AsyncMock, patch
from litellm.proxy._types import UserAPIKeyAuth
@ -699,17 +743,277 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=Exception("boom"),
original_exception=original_exception or Exception("boom"),
user_api_key_dict=UserAPIKeyAuth(),
)
def _timeout(self, model: str, cause: Exception) -> Exception:
try:
try:
raise cause
except Exception as inner:
raise litellm.Timeout("Request timed out.", model=model, llm_provider="openai") from inner
except litellm.Timeout as timed_out:
return timed_out
def _logging_obj(self, model_call_details):
logging_obj = MagicMock()
logging_obj.model_call_details = model_call_details
return logging_obj
@pytest.mark.asyncio
async def test_dispatched_failure_estimates_input_tokens_with_zero_cost(self):
async def test_timed_out_request_estimates_input_tokens_priced_at_input_rate(self):
from datetime import datetime
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime.now(),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
),
"metadata": {},
"response_cost": 123.0,
}
await self._run(
request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ReadTimeout("no bytes in time"))
)
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
assert estimated.prompt_tokens > 0
assert estimated.completion_tokens == 0
assert estimated.total_tokens == estimated.prompt_tokens
expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens)
assert expected_input_cost > 0
assert request_data["response_cost"] == pytest.approx(expected_input_cost)
@pytest.mark.asyncio
async def test_client_disconnect_after_dispatch_is_priced_at_input_rate(self):
from datetime import datetime
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime.now(),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True,
}
),
"metadata": {},
}
await self._run(
request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
)
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens)
assert expected_input_cost > 0
assert request_data["response_cost"] == pytest.approx(expected_input_cost)
@pytest.mark.asyncio
async def test_upstream_499_error_is_a_plain_failure_not_a_priced_disconnect(self):
"""A provider answering HTTP 499 surfaces as litellm.APIError with status_code 499. Only the
proxy's own disconnect HTTPException means the caller hung up mid-generation, so the upstream
error keeps the estimated usage at zero cost like any other post-dispatch provider error."""
from datetime import datetime
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 0),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
),
"metadata": {},
}
await self._run(
request_data,
original_exception=litellm.APIError(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
message="client closed request",
llm_provider="openai",
model="gpt-3.5-turbo",
),
)
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
assert estimated.prompt_tokens > 0
assert request_data["response_cost"] == 0.0
@pytest.mark.asyncio
async def test_client_disconnect_after_the_call_already_completed_is_not_priced_again(self):
"""The LLM task finished and claimed its success log (which carries the real usage and cost)
while a during_call guardrail task was still pending, then the caller hung up. The 499 raised
for that disconnect must not price the prompt a second time."""
from datetime import datetime
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "count these input tokens please"}],
stream=False,
call_type="acompletion",
start_time=datetime(2026, 1, 1, 12, 0, 0),
litellm_call_id="completed-before-disconnect",
function_id="fn",
)
logging_obj.model_call_details.update(
{
"first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
)
assert logging_obj.claim_async_success_log()
logging_obj.mark_async_success_log_enqueued()
request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}
await self._run(
request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
)
assert request_data.get("response_cost", 0.0) == 0.0
@pytest.mark.asyncio
async def test_client_disconnect_with_a_claimed_but_deferred_success_log_is_still_priced(self):
"""With a post_call guardrail registered the success log is claimed but parked in the deferred slot,
and a 499 raised before the flush means it never runs. Nothing else bills that request, so the
failure row keeps the estimated input cost."""
from datetime import datetime
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "count these input tokens please"}],
stream=False,
call_type="acompletion",
start_time=datetime(2026, 1, 1, 12, 0, 0),
litellm_call_id="deferred-success-never-flushed",
function_id="fn",
)
logging_obj.model_call_details.update(
{
"first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
)
logging_obj.defer_async_logging = True
assert logging_obj.claim_async_success_log()
logging_obj.enqueue_deferred_logging = lambda: None
request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}
await self._run(
request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
)
assert request_data["response_cost"] > 0.0
@pytest.mark.asyncio
async def test_client_disconnect_before_provider_handoff_is_not_priced(self):
"""A caller that hung up while the request was still in pre-call hooks never reached the
provider, so there is no handoff instant on the logging object and the 499 row is not priced."""
from datetime import datetime
from unittest.mock import AsyncMock, patch
from fastapi import HTTPException
from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy._types import UserAPIKeyAuth
logging_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "count these input tokens please"}],
stream=False,
call_type="acompletion",
start_time=datetime.now(),
litellm_call_id="lit145-pre-handoff",
function_id="lit145",
)
assert logging_obj.model_call_details.get("first_api_call_start_time") is None
request_data = {"litellm_logging_obj": logging_obj, "model": "gpt-3.5-turbo", "metadata": {}}
proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
with (
patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()),
patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()),
):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=HTTPException(
status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
),
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"),
)
assert request_data.get("first_api_call_start_time") is None
assert "combined_usage_object" not in request_data
assert request_data.get("response_cost", 0.0) == 0.0
@pytest.mark.asyncio
async def test_connect_timeout_never_reached_the_provider_stays_zero_cost(self):
from datetime import datetime
from litellm.types.utils import Usage
request_data = {
"litellm_logging_obj": self._logging_obj(
{
"first_api_call_start_time": datetime.now(),
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "count these input tokens please"}],
"call_type": "acompletion",
}
),
"metadata": {},
}
await self._run(
request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ConnectTimeout("no route"))
)
assert isinstance(request_data["combined_usage_object"], Usage)
assert request_data["response_cost"] == 0.0
@pytest.mark.asyncio
async def test_dispatched_non_timeout_failure_estimates_input_tokens_with_zero_cost(self):
from datetime import datetime
from litellm.types.utils import Usage
@ -731,8 +1035,6 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
estimated = request_data["combined_usage_object"]
assert isinstance(estimated, Usage)
assert estimated.prompt_tokens > 0
assert estimated.completion_tokens == 0
assert estimated.total_tokens == estimated.prompt_tokens
assert request_data["response_cost"] == 0.0
@pytest.mark.asyncio
@ -1899,7 +2201,357 @@ def test_usage_the_stream_already_recovered_beats_an_estimate():
assert lifted == (recovered, 0.25)
def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost():
def test_a_timed_out_request_on_an_unmapped_model_keeps_its_row_at_zero_cost():
from litellm.proxy.utils import _failure_usage_to_lift
lifted = _failure_usage_to_lift(
model_call_details={
"call_type": "acompletion",
"model": "my-unmapped-model",
"custom_llm_provider": "openai",
"messages": ONE_USER_MESSAGE,
},
request_body={},
dispatched=True,
provider_billed=True,
)
assert lifted is not None
usage, response_cost = lifted
assert usage.prompt_tokens > 0
assert response_cost == 0.0
def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the_deployment(monkeypatch):
"""Router._create_deployment registers a deployment's own ``input_cost_per_token`` under its
``model_info.id``; the failure row prices with that entry through the same
``deployment_pricing_model_info`` the success path uses."""
from litellm.proxy.utils import _failure_usage_to_lift
monkeypatch.setitem(
litellm.model_cost,
"dep-lit145",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0},
)
lifted = _failure_usage_to_lift(
model_call_details={
"call_type": "acompletion",
"model": FAILURE_USAGE_MODEL,
"messages": ONE_USER_MESSAGE,
},
request_body={},
dispatched=True,
provider_billed=True,
deployment_id="dep-lit145",
)
assert lifted is not None
usage, response_cost = lifted
assert usage.prompt_tokens > 0
assert response_cost == pytest.approx(usage.prompt_tokens * 0.25)
@pytest.mark.parametrize(("prompt_tokens", "expected_rate"), [(1_000, 1e-7), (250_000, 5e-7)])
def test_a_timed_out_request_is_priced_at_the_deployments_long_context_tier(monkeypatch, prompt_tokens, expected_rate):
"""A deployment that declares ``input_cost_per_token_above_200k_tokens`` next to its standard rate
bills a 250k-token prompt at the tier on the success path; the failure row must not flatten it to
the standard rate."""
from litellm.proxy.utils import _estimated_input_cost
monkeypatch.setitem(
litellm.model_cost,
"dep-lit-f9-tiered",
{
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-7,
"input_cost_per_token_above_200k_tokens": 5e-7,
"output_cost_per_token": 4e-7,
},
)
response_cost = _estimated_input_cost(
model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"},
prompt_tokens=prompt_tokens,
deployment_id="dep-lit-f9-tiered",
base_model=None,
)
assert response_cost == pytest.approx(prompt_tokens * expected_rate)
def test_every_earlier_timed_out_attempt_is_billed_at_its_own_deployment_rate(monkeypatch):
"""The router leaves one RetryAttemptRecord per failed attempt in metadata.previous_models, the
final attempt included; the earlier Timeout ones are billed (each at the deployment that served
it), a 429 attempt is not, and the final attempt is left to the failure row itself."""
from litellm.proxy.utils import _billed_earlier_attempts
monkeypatch.setitem(
litellm.model_cost,
"dep-lit145-b",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0},
)
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
public_rate, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=24)
attempts = _billed_earlier_attempts(
model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"},
litellm_params={
"metadata": {
"attempted_retries": 3,
"previous_models": (
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0},
{"deployment_id": "dep-lit145-b", "exception_type": "RateLimitError", "attempted_retries": 1},
{"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 2},
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 3},
),
}
},
estimated_usage=usage,
base_model=None,
deployment_id="dep-lit145-a",
)
assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [(0, "dep-lit145-a"), (2, "dep-lit145-b")]
assert attempts[0].response_cost == pytest.approx(public_rate)
assert attempts[0].response_cost > 0
assert attempts[1].response_cost == pytest.approx(24 * 0.25)
assert all(a.usage is usage for a in attempts)
def test_an_earlier_attempt_on_another_model_is_priced_at_that_models_rate_not_the_final_models(monkeypatch):
"""After a fallback hop the final attempt runs on another model group, so the records of the group
that timed out first name their own deployment model (and its ``base_model`` when the deployment
name is not in the cost map). Each is billed at that model's input rate, not at the final model's."""
from litellm.proxy.utils import _billed_earlier_attempts
for pricing_model, provider, input_rate in (
("lit145-final-model", "openai", 0.5),
("lit145-first-model", "openai", 0.02),
("azure/lit145-first-model", "azure", 0.02),
):
monkeypatch.setitem(
litellm.model_cost,
pricing_model,
{"litellm_provider": provider, "mode": "chat", "input_cost_per_token": input_rate, "output_cost_per_token": 1.0},
)
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
attempts = _billed_earlier_attempts(
model_call_details={"model": "openai/lit145-final-model", "custom_llm_provider": "openai"},
litellm_params={
"metadata": {
"attempted_retries": 1,
"model_info": {"id": "dep-lit145-final"},
"previous_models": (
{
"deployment_id": "dep-lit145-first",
"model": "openai/lit145-first-model",
"base_model": None,
"exception_type": "Timeout",
"attempted_retries": 0,
},
{
"deployment_id": "dep-lit145-first-azure",
"model": "azure/lit145-first-deployment",
"base_model": "azure/lit145-first-model",
"exception_type": "Timeout",
"attempted_retries": 1,
},
{
"deployment_id": "dep-lit145-final",
"model": "openai/lit145-final-model",
"base_model": None,
"exception_type": "Timeout",
"attempted_retries": 0,
},
{
"deployment_id": "dep-lit145-final",
"model": "openai/lit145-final-model",
"base_model": None,
"exception_type": "Timeout",
"attempted_retries": 1,
},
),
}
},
estimated_usage=usage,
base_model=None,
deployment_id="dep-lit145-final",
)
assert [(a.attempted_retries, a.deployment_id, a.response_cost) for a in attempts] == [
(0, "dep-lit145-first", pytest.approx(24 * 0.02)),
(1, "dep-lit145-first-azure", pytest.approx(24 * 0.02)),
(0, "dep-lit145-final", pytest.approx(24 * 0.5)),
]
def test_earlier_attempts_are_priced_from_the_prompt_estimate_not_the_final_streams_recovered_usage():
"""A final attempt whose stream broke mid-flight lifts its recovered usage (prompt plus the chunks
delivered); the earlier timed-out attempts never produced output, so they are priced from the
input-only estimate and not from that recovered usage."""
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.proxy.utils import _estimated_prompt_usage, _failure_fields_to_lift
logging_obj = _dispatched_failure_logging_obj(
FAILURE_USAGE_MODEL,
get_litellm_params(
metadata={
"attempted_retries": 1,
"model_info": {"id": "dep-lit145-final"},
"previous_models": (
{"deployment_id": "dep-lit145-final", "exception_type": "Timeout", "attempted_retries": 0},
),
}
),
)
recovered = litellm.Usage(prompt_tokens=500, completion_tokens=40, total_tokens=540)
logging_obj.model_call_details["combined_usage_object"] = recovered
logging_obj.model_call_details["response_cost"] = 0.01
lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
estimate = _estimated_prompt_usage(logging_obj.model_call_details, {})
assert lifted["combined_usage_object"] is recovered
assert [(a.usage.prompt_tokens, a.usage.completion_tokens) for a in lifted["billed_failed_attempts"]] == [
(estimate.prompt_tokens, 0)
]
assert estimate.prompt_tokens < recovered.prompt_tokens
def test_a_timed_out_attempt_from_the_group_a_fallback_left_is_still_billed():
"""After a fallback hop the router restarts ``attempted_retries`` at 0 for the new model group and
re-logs the attempt that exhausted the previous group, so the final attempt (dep-b, retry 1) must be
told from the previous group's attempt with the same count (dep-a, retry 1), and the re-logged
record must bill once."""
from litellm.proxy.utils import _billed_earlier_attempts
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
attempts = _billed_earlier_attempts(
model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"},
litellm_params={
"metadata": {
"attempted_retries": 1,
"model_info": {"id": "dep-lit145-b"},
"previous_models": (
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0},
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1},
{"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1},
{"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 0},
{"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 1},
),
}
},
estimated_usage=usage,
base_model=None,
deployment_id="dep-lit145-b",
)
assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [
(0, "dep-lit145-a"),
(1, "dep-lit145-a"),
(0, "dep-lit145-b"),
]
assert all(a.response_cost > 0 for a in attempts)
def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object:
"""A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params()
shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff
instant, so the lift treats the request as dispatched."""
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model=model,
messages=ONE_USER_MESSAGE,
stream=False,
call_type="acompletion",
start_time=real_datetime.datetime.now(),
litellm_call_id="call-lit145",
function_id="fn-lit145",
)
logging_obj.update_environment_variables(model=model, user="", optional_params={}, litellm_params=litellm_params)
logging_obj.model_call_details["first_api_call_start_time"] = real_datetime.datetime.now()
logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE
logging_obj.model_call_details["call_type"] = "acompletion"
return logging_obj
def test_the_deployment_rate_is_found_on_the_logging_objects_real_litellm_params(monkeypatch):
"""``get_litellm_params()`` always carries a ``litellm_metadata`` key (None on chat
completions) next to the router-stamped ``metadata.model_info.id``; the failure row must
still price at the deployment's own rate, like ``Logging.get_router_model_id`` on the
success path."""
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.proxy.utils import _failure_fields_to_lift
monkeypatch.setitem(
litellm.model_cost,
"dep-lit145-real-shape",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 0.25},
)
litellm_params = get_litellm_params(metadata={"model_info": {"id": "dep-lit145-real-shape"}})
assert "litellm_metadata" in litellm_params and litellm_params["litellm_metadata"] is None
logging_obj = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, litellm_params)
lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
assert lifted["response_cost"] == pytest.approx(lifted["combined_usage_object"].prompt_tokens * 0.25)
def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path():
"""An Azure deployment name is not in the cost map; the success path prices it through
``model_info.base_model`` (``get_base_model_from_metadata``) and so must the failure row."""
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.proxy.utils import _failure_fields_to_lift
logging_obj = _dispatched_failure_logging_obj(
"azure/lit145-not-in-cost-map",
get_litellm_params(metadata={"model_info": {"base_model": "azure/gpt-4.1"}}),
)
logging_obj.model_call_details["custom_llm_provider"] = "azure"
lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True)
expected_cost, _ = litellm.cost_per_token(
model="azure/gpt-4.1", prompt_tokens=lifted["combined_usage_object"].prompt_tokens
)
assert expected_cost > 0
assert lifted["response_cost"] == pytest.approx(expected_cost)
def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price():
"""``model_call_details`` is flattened with the request's optional_params, so a body key
named ``litellm_params`` can land there; the deployment rate must come from the logging
object's own ``litellm_params`` instead."""
from litellm.proxy.utils import _failure_usage_to_lift
lifted = _failure_usage_to_lift(
model_call_details={
"call_type": "acompletion",
"model": FAILURE_USAGE_MODEL,
"messages": ONE_USER_MESSAGE,
"litellm_params": {"input_cost_per_token": 0},
},
request_body={},
dispatched=True,
provider_billed=True,
deployment_id=None,
)
assert lifted is not None
usage, response_cost = lifted
expected_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens)
assert response_cost == pytest.approx(expected_cost)
assert response_cost > 0
def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rate():
from litellm.proxy.utils import _failure_usage_to_lift
lifted = _failure_usage_to_lift(
@ -1910,13 +2562,16 @@ def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost():
},
request_body={},
dispatched=True,
provider_billed=True,
)
assert lifted is not None
usage, response_cost = lifted
assert usage.prompt_tokens > 0
assert usage.completion_tokens == 0
assert response_cost == 0.0
expected_input_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens)
assert expected_input_cost > 0
assert response_cost == pytest.approx(expected_input_cost)
@pytest.mark.parametrize(

50
tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py
View file

@ -883,3 +883,53 @@ def test_disable_spend_updates_error_when_general_settings_unavailable(
monkeypatch.delattr(proxy_server_mod, "general_settings", raising=False)
with pytest.raises(ImportError):
ProxyUpdateSpend.disable_spend_updates()
async def _create_many_while_postgres_is_restarting(**_: object) -> int:
from prisma.engine.utils import handle_response_errors
handle_response_errors(
MagicMock(),
[
{
"error": "Error in connector: Can't reach database server at `127.0.0.1:5432`",
"user_facing_error": {"error_code": "P1001", "message": "Can't reach database server", "meta": None},
}
],
)
return 0
@pytest.mark.asyncio
async def test_update_spend_logs_retries_and_requeues_batch_when_postgres_restart_tears_down_prisma_engine(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
"""While Postgres restarts the prisma engine answers the write with a P1001
payload whose ``meta`` is ``None``, which prisma's own response parser trips
over before it can build a typed error. That outage must be retried like
any other, and once retries exhaust the dequeued batch goes back to the head
of the queue so the rows land when Postgres is back.
"""
async def _fake_sleep(_: float) -> None:
return None
monkeypatch.setattr(utils_mod.asyncio, "sleep", _fake_sleep)
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
side_effect=_create_many_while_postgres_is_restarting
)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="c")]
with pytest.raises(AttributeError):
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=2,
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
logs_to_process=[make_spend_log_row(request_id="a"), make_spend_log_row(request_id="b")],
)
assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["a", "b", "c"]
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 3

37
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -5322,3 +5322,40 @@ def test_convert_response_function_tool_call_keeps_the_index_it_is_given(index:
)
assert tool_call["index"] == index
@pytest.mark.asyncio
async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chunks():
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
chat_stream: Final = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "tell me a story"}],
mock_response="The codename is AZURE-FALCON-42 and the story is long.",
stream=True,
api_key="test-key",
)
iterator: Final = LiteLLMCompletionStreamingIterator(
model="gpt-4o-mini",
litellm_custom_stream_wrapper=chat_stream,
request_input="tell me a story",
responses_api_request={},
custom_llm_provider="openai",
)
assert iterator.partial_response_on_disconnect() is None, "nothing relayed yet, nothing to bill"
events: Final = [await iterator.__anext__() for _ in range(6)]
relayed_text: Final = "".join(
getattr(event, "delta", "") for event in events if event.type == "response.output_text.delta"
)
assert relayed_text, "the bridge should have relayed some text by now"
incomplete_event: Final = iterator.partial_response_on_disconnect()
assert incomplete_event is not None
assert incomplete_event.type == "response.incomplete"
assert incomplete_event.response.status == "incomplete"
assert incomplete_event.response.usage is not None
assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text)
assert incomplete_event.response.usage.input_tokens > 0

100
tests/test_litellm/responses/test_responses_streaming_iterator.py
View file

@ -7,7 +7,10 @@ crashed with exit 139 whenever any CustomLogger was registered).
"""
import asyncio
import json
import time
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
@ -17,6 +20,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module
from litellm.responses import streaming_iterator as responses_streaming_iterator_module
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.types.llms.openai import ResponsesAPIResponse
@ -156,3 +160,99 @@ async def test_sync_callbacks_run_only_after_async_handler_completes(recording_e
submit_times = recording_executor.submit_times_for(logging_obj)
assert len(submit_times) == 1
assert submit_times[0] >= recorder.async_hook_finished
_CREATED_RESPONSE_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = {
"id": "resp_partial",
"object": "response",
"created_at": 1700000000.0,
"model": "gpt-5.4-nano",
"status": "in_progress",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"service_tier": "priority",
}
_CREATED_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = {
"type": "response.created",
"sequence_number": 0,
"response": _CREATED_RESPONSE_WITH_PRIORITY_TIER,
}
def _text_delta(text: str, sequence_number: int) -> Mapping[str, object]:
return {
"type": "response.output_text.delta",
"sequence_number": sequence_number,
"item_id": "msg_partial",
"output_index": 0,
"content_index": 0,
"delta": text,
}
def _completed_after(text: str) -> Mapping[str, object]:
return {
"type": "response.completed",
"sequence_number": 9,
"response": {
**_CREATED_RESPONSE_WITH_PRIORITY_TIER,
"status": "completed",
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
},
}
def _live_iterator(logging_obj: LitellmLogging) -> ResponsesAPIStreamingIterator:
return ResponsesAPIStreamingIterator(
response=httpx.Response(200),
model="gpt-5.4-nano",
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=logging_obj,
request_data={"model": "gpt-5.4-nano", "input": "tell me a story"},
custom_llm_provider="openai",
)
def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier():
iterator: Final = _live_iterator(_make_logging_obj())
for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time ", 1), _text_delta("a fox slept.", 2)):
iterator._process_chunk(json.dumps(event))
incomplete_event: Final = iterator.partial_response_on_disconnect()
assert incomplete_event is not None
assert incomplete_event.type == "response.incomplete"
partial: Final = incomplete_event.response
assert partial.status == "incomplete"
assert getattr(partial, "service_tier") == "priority"
assert partial.output[0].content[0].text == "Once upon a time a fox slept."
assert partial.usage.output_tokens == litellm.token_counter(
model="gpt-5.4-nano", text="Once upon a time a fox slept."
)
assert partial.usage.input_tokens == litellm.token_counter(
model="gpt-5.4-nano", messages=[{"role": "user", "content": "tell me a story"}]
)
assert partial.usage.total_tokens == partial.usage.input_tokens + partial.usage.output_tokens
@pytest.mark.parametrize(
"events",
[
(),
(_CREATED_WITH_PRIORITY_TIER,),
(_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1), _completed_after("Once upon a time")),
],
ids=["nothing_relayed", "created_only", "already_completed"],
)
@pytest.mark.asyncio
async def test_partial_response_on_disconnect_is_none_when_there_is_nothing_to_bill_or_the_stream_billed_itself(
events: tuple[Mapping[str, object], ...],
):
iterator: Final = _live_iterator(_make_logging_obj())
for event in events:
iterator._process_chunk(json.dumps(event))
assert iterator.partial_response_on_disconnect() is None
await asyncio.sleep(0)

30
tests/test_litellm/responses/test_streaming_iterator.py
View file

@ -436,27 +436,27 @@ def _responses_api_response_with_usage() -> ResponsesAPIResponse:
)
def test_stamp_responses_usage_cost_stamps_computed_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
def teststamp_responses_usage_cost_stamps_computed_cost():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.return_value = 0.000704
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
logging_obj._response_cost_calculator.assert_called_once_with(result=response)
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
def teststamp_responses_usage_cost_keeps_provider_reported_cost():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
setattr(response.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
logging_obj._response_cost_calculator.assert_not_called()
@ -475,8 +475,8 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
)
def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
def teststamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage(
{
"input_tokens": 29,
@ -488,7 +488,7 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
)
logging_obj = Mock(spec=LiteLLMLoggingObj)
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(3e-05)
@ -496,27 +496,27 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
logging_obj._response_cost_calculator.assert_not_called()
def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
def teststamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149})
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.return_value = 0.000704
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(0.000704)
logging_obj._response_cost_calculator.assert_called_once_with(result=response)
def test_stamp_responses_usage_cost_survives_calculator_failure():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
def teststamp_responses_usage_cost_survives_calculator_failure():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
_stamp_responses_usage_cost(response, logging_obj)
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) is None

39
tests/test_litellm/router_strategy/test_lowest_cost.py
View file

@ -101,3 +101,42 @@ async def test_async_log_success_event_counts_a_response_with_no_completion_toke
)
assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
_TWO_DOLLARS_PER_TOKEN = {"input_cost_per_token": 1.0, "output_cost_per_token": 1.0}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"azure_litellm_params, base_model",
[
({"model": "azure/my-gpt4-deploy"}, "azure/gpt-4.1"),
({"model": "azure/gpt-4.1"}, "azure/not-a-real-model"),
],
ids=["base_model_priced", "unmapped_base_model_falls_back_to_deployment_model"],
)
async def test_cost_based_routing_prices_an_azure_deployment_through_its_base_model(
azure_litellm_params: dict, base_model: str
):
handler = LowestCostLoggingHandler(router_cache=DualCache())
healthy_deployments = [
{
"model_name": "gpt-4.1",
"litellm_params": {"model": "openai/gpt-4.1", **_TWO_DOLLARS_PER_TOKEN},
"model_info": {"id": "openai-two-dollars"},
},
{
"model_name": "gpt-4.1",
"litellm_params": azure_litellm_params,
"model_info": {"id": "azure-cost-map-priced", "base_model": base_model},
},
]
picked = await handler.async_get_available_deployments(
model_group="gpt-4.1", healthy_deployments=healthy_deployments
)
assert picked is not None and picked["model_info"]["id"] == "azure-cost-map-priced", (
"a cost-map priced Azure deployment must beat a $2 per token deployment, "
"it loses only when it is scored at the $10 unmapped default"
)

53
tests/test_litellm/router_utils/test_deployment_pricing.py Normal file
View file

@ -0,0 +1,53 @@
import pytest
from litellm.router_utils.deployment_pricing import deployment_pricing_models, pricing_models
@pytest.mark.parametrize(
"base_model, model, expected",
[
("azure/gpt-4.1", "azure/my-gpt4-deploy", ("azure/gpt-4.1", "azure/my-gpt4-deploy")),
(None, "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)),
("", "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)),
("azure/gpt-4.1", "azure/gpt-4.1", ("azure/gpt-4.1",)),
(None, None, ()),
],
ids=["base_model_first", "no_base_model", "blank_base_model", "same_name_once", "nothing_named"],
)
def test_pricing_models_walks_base_model_then_deployment_model(base_model, model, expected):
assert pricing_models(base_model, model) == expected
@pytest.mark.parametrize(
"deployment, expected",
[
(
{"litellm_params": {"model": "azure/my-gpt4-deploy"}, "model_info": {"base_model": "azure/gpt-4.1"}},
("azure/gpt-4.1", "azure/my-gpt4-deploy"),
),
(
{"litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4.1"}},
("azure/gpt-4.1", "azure/my-gpt4-deploy"),
),
(
{
"litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4o"},
"model_info": {"base_model": "azure/gpt-4.1"},
},
("azure/gpt-4.1", "azure/my-gpt4-deploy"),
),
({"litellm_params": {"model": "openai/gpt-4.1"}, "model_info": {}}, ("openai/gpt-4.1",)),
({"litellm_params": "not-a-dict", "model_info": None}, ()),
("not-a-deployment", ()),
],
ids=[
"model_info_base_model",
"params_base_model",
"model_info_wins",
"no_base_model",
"malformed_blocks",
"not_a_dict",
],
)
def test_deployment_pricing_models_reads_the_deployment_dict(deployment, expected):
assert deployment_pricing_models(deployment) == expected

29
tests/test_litellm/test_model_prices_schema.py
View file

@ -10,11 +10,13 @@ from typing import Final
import jsonschema
import pytest
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
from litellm.types.utils import SearchFreeRequests
REPO_ROOT = Path(__file__).parents[2]
GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py"
@ -490,3 +492,30 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
"unknown_root-new_family_models",
"vertex_ai-new_family_models",
]
def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]:
prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
return MappingProxyType(
{key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row}
)
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None:
rows = _search_free_requests_rows(path)
assert rows
allowances = {
key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items()
}
for key, row in rows.items():
assert row.get("supports_web_search") is True, key
assert row.get("search_context_cost_per_query"), key
assert allowances[key]["count"] > 0, key
for group in {allowance["shared_group"] for allowance in allowances.values()}:
same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group}
assert len(same_group) == 1, (group, same_group)
def test_search_free_requests_backup_rows_match_the_main_catalog() -> None:
assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH)

85
tests/test_litellm/test_router.py
View file

@ -11263,7 +11263,7 @@ async def test_cyclic_fallback_graph_does_not_amplify_one_request():
_FLAT_ATTEMPT_RECORD_KEYS = frozenset(
{"model_group", "deployment_id", "exception_type", "exception_string", "attempted_retries"}
{"model_group", "deployment_id", "model", "base_model", "exception_type", "exception_string", "attempted_retries"}
)
_BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doNotShip"
@ -12755,6 +12755,24 @@ def _anthropic_messages_make_wrapper() -> FallbackAwareAnthropicMessagesStream:
return FallbackAwareAnthropicMessagesStream(_anthropic_messages_empty_generator(), object())
def test_anthropic_messages_fallback_wrapper_bills_the_stream_behind_the_hidden_params_wrapper():
"""set_response_headers wraps the bare byte stream, the disconnect bill must still reach the stream inside it."""
from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
from litellm.types.llms.openai import ResponseIncompleteEvent
event: Final = MagicMock(spec=ResponseIncompleteEvent)
class _BillableByteStream:
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent:
return event
wrapped: Final = FallbackAwareAnthropicMessagesStream(
_anthropic_messages_empty_generator(), HiddenParamsAsyncIteratorWrapper(_BillableByteStream())
)
assert wrapped.partial_response_on_disconnect() is event
def _anthropic_messages_make_router() -> Router:
return Router(
model_list=[
@ -16919,3 +16937,68 @@ class TestMemberAutoRouterInference:
monkeypatch.setitem(sys.modules, "fastapi", None)
monkeypatch.delitem(sys.modules, "litellm.proxy.auth.auto_router_checks", raising=False)
assert (await self._route(router, {"metadata": {"user_api_key_team_id": "router-team"}})).model == "restricted-model"
@pytest.mark.asyncio
async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens():
"""After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone."""
import json
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
def _relayed_iterator(text: str) -> ResponsesAPIStreamingIterator:
logging_obj: Final = MagicMock(spec=LiteLLMLogging)
logging_obj.model_call_details = {"litellm_params": {}}
logging_obj.completion_start_time = None
iterator: Final = ResponsesAPIStreamingIterator(
response=httpx.Response(200),
model="gpt-5.4",
responses_api_provider_config=OpenAIResponsesAPIConfig(),
logging_obj=logging_obj,
request_data={"model": "gpt-5.4", "input": "Hi"},
custom_llm_provider="openai",
)
created: Final = {
"id": "resp_1",
"object": "response",
"created_at": 1700000000.0,
"model": "gpt-5.4",
"status": "in_progress",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
}
iterator._process_chunk(json.dumps({"type": "response.created", "sequence_number": 0, "response": created}))
iterator._process_chunk(
json.dumps(
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_1",
"output_index": 0,
"content_index": 0,
"delta": text,
}
)
)
return iterator
router: Final = Router(
model_list=[{"model_name": "primary", "litellm_params": {"model": "openai/gpt-5.4", "api_key": "k"}}]
)
primary: Final = _relayed_iterator("Once upon a time there was a lighthouse keeper")
fallback: Final = _relayed_iterator("who lit the lamp")
wrapped: Final = await router._aresponses_streaming_iterator(
response=primary,
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": AsyncMock()},
)
primary_tokens: Final = primary.partial_response_on_disconnect().response.usage.output_tokens
fallback_tokens: Final = fallback.partial_response_on_disconnect().response.usage.output_tokens
assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens
wrapped.adopt_fallback_headers(fallback)
assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens

30
tests/test_litellm/test_utils.py
View file

@ -5961,3 +5961,33 @@ def test_runtime_model_cost_registrations_resolve_bedrock_routing(monkeypatch: p
)
assert registrations[key] == rates
assert model_utils.runtime_model_cost_registrations("bedrock/converse/openai.other-converse", "bedrock") == ()
def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body():
"""LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no
step that consumes it, so it must not ride along in extra_body and 400 at OpenAI."""
optional_params = litellm.utils.get_optional_params(
model="gpt-4.1",
custom_llm_provider="openai",
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}],
extra_body={"kept": True},
)
assert optional_params["extra_body"] == {"kept": True}
assert "cache_control_injection_points" not in optional_params
def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy():
"""A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy
applies cache_control_injection_points itself, so the point must ride along in extra_body."""
point: Final = {"location": "tool_config", "control": {"type": "ephemeral"}}
optional_params: Final[dict[str, object]] = litellm.utils.get_optional_params(
model="claude-sonnet-4-5",
custom_llm_provider="litellm_proxy",
tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
cache_control_injection_points=[point],
extra_body={"kept": True},
)
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}

16
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -31151,6 +31151,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -31163,6 +31167,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */
@ -41884,6 +41892,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -41896,6 +41908,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */

2
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-10-18T01:20:16.83601Z"
exclude-newer = "2026-10-18T00:46:14.647863266Z"
exclude-newer-span = "P3D"
[manifest]