diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 73d8d2649bf..96634fb74fe 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -62,17 +62,6 @@ OFF_PEAK_WINDOW: Final[JsonSchema] = { } OBJECT_KEYS: dict[str, JsonSchema] = { - "search_free_requests": { - "type": "object", - "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.", - "required": ["count", "period", "shared_group"], - "additionalProperties": False, - "properties": { - "count": {"type": "integer", "minimum": 0}, - "period": {"type": "string", "enum": ["day", "month"]}, - "shared_group": {"type": "string", "pattern": "\\S"}, - }, - }, "off_peak_pricing": { "type": "object", "description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.", @@ -102,6 +91,17 @@ OBJECT_KEYS: dict[str, JsonSchema] = { }, "additionalProperties": False, }, + "search_free_requests": { + "type": "object", + "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.", + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + "properties": { + "count": {"type": "integer", "minimum": 0}, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": {"type": "string", "pattern": "\\S"}, + }, + }, "guardrail_cost_per_unit": { "type": "object", "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).", diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index a58798835d7..3010879627c 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -767,7 +767,11 @@ class CheckBatchCost: from litellm.files.main import afile_content from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging - from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials + from litellm.litellm_core_utils.litellm_logging import ( + deployment_pricing_model_info, + mask_api_base_credentials, + resolve_vertex_location_for_cost, + ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, ) @@ -904,6 +908,12 @@ class CheckBatchCost: custom_llm_provider=batch_file_provider, model_name=model_name, model_info=deployment_model_info, + vertex_location=resolve_vertex_location_for_cost( + custom_llm_provider=llm_provider, + litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True), + optional_params=None, + model=model_name, + ), ) error_file_failed_requests: Final = await count_error_file_failed_requests( response, diff --git a/litellm/__init__.py b/litellm/__init__.py index 21f2f570157..5f483655caa 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -556,6 +556,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma model_cost = get_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +apply_search_free_tier: bool = False cost_margin_config: Dict[ str, Union[float, Dict[str, float]] ] = {} # Provider-specific or global cost margins. Examples: diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 6e16f5e81f2..b99901bceaf 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -73,6 +73,7 @@ async def calculate_batch_cost_and_usage( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """ Calculate the cost and usage of a batch. @@ -82,16 +83,21 @@ async def calculate_batch_cost_and_usage( pricing. Threaded through to batch_cost_calculator so that deployment-specific pricing (e.g. input_cost_per_token_batches) is used instead of the global cost map. + vertex_location: The Vertex AI location the batch ran on; non-global + locations apply the model's regional-endpoint uplift multiplier. """ first_row: Final = file_content_dictionary[0] if file_content_dictionary else None if _uses_native_vertex_output(custom_llm_provider, model_name, first_row): - return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name, model_info=model_info) + return calculate_vertex_ai_batch_cost_and_usage( + file_content_dictionary, model_name, model_info=model_info, vertex_location=vertex_location + ) return _aggregate_batch_cost_usage_models( entries=file_content_dictionary, custom_llm_provider=custom_llm_provider, model_name=model_name, model_info=model_info, + vertex_location=vertex_location, ) @@ -101,6 +107,7 @@ async def _handle_completed_batch( model_name: str | None = None, litellm_params: dict | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """Fetch a completed batch's output file and aggregate its cost, usage, and models in a single pass over the JSONL lines, so the parsed file content is @@ -114,6 +121,8 @@ async def _handle_completed_batch( model_info: Optional deployment-level model info with custom pricing, threaded through so a deployment's configured rates win over the global cost map. + vertex_location: The Vertex AI location the batch ran on; non-global + locations apply the model's regional-endpoint uplift multiplier. """ # A completed batch whose request lines all failed has no output file - the # results are written to a separate error_file_id and output_file_id is None. @@ -141,7 +150,10 @@ async def _handle_completed_batch( output_file_result: Final = ( calculate_vertex_ai_batch_cost_and_usage( - _iter_batch_output_entries(file_content), model_name, model_info=model_info + _iter_batch_output_entries(file_content), + model_name, + model_info=model_info, + vertex_location=vertex_location, ) if _uses_native_vertex_output( custom_llm_provider, model_name, next(_iter_batch_output_entries(file_content), None) @@ -151,6 +163,7 @@ async def _handle_completed_batch( custom_llm_provider=custom_llm_provider, model_name=model_name, model_info=model_info, + vertex_location=vertex_location, ) ) @@ -186,6 +199,7 @@ def _classify_output_line_stats( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> Iterator[_BatchOutputLineStats | _LineOutcome]: """Classify every output line in a single pass, so counting failures never needs a second read of a potentially huge output file. A line the provider reported as @@ -196,7 +210,7 @@ def _classify_output_line_stats( if not _batch_response_was_successful(entry, custom_llm_provider): yield _LineOutcome.PROVIDER_FAILED continue - stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info) + stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location) yield stats if stats is not None else _LineOutcome.UNCOSTABLE @@ -205,13 +219,14 @@ def _safe_output_line_stats( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> _BatchOutputLineStats | None: """Return the stats for one provider-successful batch output line, or None when it cannot be costed, so a single bad line never aborts the whole batch's cost accounting.""" custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None try: - return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info) + return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location) except Exception as e: # noqa: BLE001 # any single line's costing failure must not abort the whole batch verbose_logger.warning( "batch output line could not be costed, so it is billed at $0 and the rest of the batch " @@ -227,6 +242,7 @@ def _compute_output_line_stats( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> _BatchOutputLineStats: response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider) usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider) @@ -241,6 +257,7 @@ def _compute_output_line_stats( model_name=model_name, response_model=response_model, model_info=model_info, + vertex_location=vertex_location, ) return _BatchOutputLineStats( prompt_cost=line_prompt_cost, @@ -270,6 +287,7 @@ def _output_line_cost( model_name: str | None, response_model: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> tuple[float, float]: """(prompt_cost, completion_cost) for one output line, priced at batch rates.""" from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost @@ -290,6 +308,7 @@ def _output_line_cost( model=cost_model, custom_llm_provider=custom_llm_provider, model_info=model_info, + vertex_location=vertex_location, ) @@ -298,11 +317,14 @@ def _aggregate_batch_cost_usage_models( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """Aggregate cost, usage, models, and pass/fail counts from batch output entries in a single pass, holding one small stats record per line instead of the parsed file.""" - all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info)) + all_results: Final = tuple( + _classify_output_line_stats(entries, custom_llm_provider, model_name, model_info, vertex_location) + ) line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats)) failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED) successful_requests: Final = len(all_results) - failed_requests @@ -349,6 +371,7 @@ def calculate_vertex_ai_batch_cost_and_usage( vertex_ai_batch_responses: Iterable[dict], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """ Cost and usage of a native Vertex predictions.jsonl, one @@ -367,6 +390,7 @@ def calculate_vertex_ai_batch_cost_and_usage( model_info=model_info, calculate_usage=VertexGeminiConfig._calculate_usage, cost_calculator=batch_cost_calculator, + vertex_location=vertex_location, ) for row in vertex_ai_batch_responses ) diff --git a/litellm/constants.py b/litellm/constants.py index 3b14d99bfc3..4420fd73e77 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1871,6 +1871,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "public_model_groups", "public_model_groups_links", "cost_discount_config", + "apply_search_free_tier", "cost_margin_config", "block_requests_for_models_without_pricing", "budget_exceeded_throttle_percentage", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 55aeaaa10ff..bbc41974daf 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( get_batch_cost_rates, get_billable_input_tokens, get_token_type_cost_breakdown, + get_vertex_regional_endpoint_uplift, parse_prompt_tokens_details, select_cost_metric_for_model, ) @@ -631,6 +632,7 @@ def cost_per_token( model=model, custom_llm_provider=custom_llm_provider, data_residency=data_residency, + vertex_location=vertex_location, ) elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): @@ -2664,6 +2666,8 @@ def batch_cost_calculator( custom_llm_provider: str | None = None, model_info: ModelInfo | None = None, data_residency: str | None = None, + *, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculate the cost of a batch job. @@ -2749,7 +2753,8 @@ def batch_cost_calculator( total_prompt_cost *= uplift total_completion_cost *= uplift - return total_prompt_cost, total_completion_cost + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift def _batch_prompt_cost( diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 1ad2f006736..4299736bcff 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,9 +1,11 @@ +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from typing import Final, TypeAlias import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError +from pydantic_core import to_jsonable_python from typing_extensions import assert_never import litellm @@ -30,6 +32,7 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, UnsupportedDecisionsRequest, ) +from litellm.types.llms.openai import ChatCompletionUserMessage from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -148,6 +151,30 @@ def _provider_ir_request( return _UnsupportedSafetyIdentifier() +_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _jsonable_fallback(value: object) -> object: + return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value) + + +def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage: + caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python( + to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64") + ) + sent_fields: Final = json.dumps( + dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {})) + ) + return ChatCompletionUserMessage(role="user", content=sent_fields) + + +def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None: + """Record every request field the caller sent as the logged `messages`, before any validation.""" + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_messages([_logged_request_message(caller_request)]) + + def prepare_call( *, model: str, diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 8b13d04770a..a31e385dac4 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -13,6 +13,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, send, ) @@ -59,6 +60,14 @@ def _prepare_decisions_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + log_caller_request( + kwargs, + { + "input": decision_input, + "questions": questions, + **({"state": kwargs["state"]} if "state" in kwargs else {}), + }, + ) return prepare_call( model=model, fields=_request_fields(decision_input, questions, kwargs), diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e03bc59b838..821abe56580 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -42,6 +42,7 @@ from litellm.constants import ( DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, EMPTY_MAPPING, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROVIDER_REQUEST_ID_HEADERS, SENTRY_DENYLIST, SENTRY_PII_DENYLIST, @@ -484,7 +485,7 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None: return None -def _resolve_vertex_location_for_cost( +def resolve_vertex_location_for_cost( custom_llm_provider: str | None, litellm_params: Mapping[str, object] | None, optional_params: Mapping[str, object] | None, @@ -710,6 +711,7 @@ class Logging(LiteLLMLoggingBaseClass): self._defer_async_logging: bool = False self._enqueue_deferred_logging: Callable[[], None] | None = None self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None + self._async_success_enqueued: bool = False self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None: @@ -1401,15 +1403,20 @@ class Logging(LiteLLMLoggingBaseClass): "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) - self.model_call_details["api_call_start_time"] = datetime.datetime.now() - # Set-once first provider-handoff instant. api_call_start_time - # is overwritten on every retry, so it can't measure one-time - # preprocessing; pinning the first attempt excludes retry loops - # + backoff. Logging object only — must NOT go into - # litellm_params["metadata"] (caller request metadata, typed - # Dict[str, str], echoed downstream; a datetime breaks it). - if self.model_call_details.get("first_api_call_start_time") is None: - self.model_call_details["first_api_call_start_time"] = self.model_call_details["api_call_start_time"] + # A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks + # without any provider handoff, so it must not stamp the handoff instants. + if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details: + self.model_call_details["api_call_start_time"] = datetime.datetime.now() + # Set-once first provider-handoff instant. api_call_start_time + # is overwritten on every retry, so it can't measure one-time + # preprocessing; pinning the first attempt excludes retry loops + # + backoff. Logging object only — must NOT go into + # litellm_params["metadata"] (caller request metadata, typed + # Dict[str, str], echoed downstream; a datetime breaks it). + if self.model_call_details.get("first_api_call_start_time") is None: + self.model_call_details["first_api_call_start_time"] = self.model_call_details[ + "api_call_start_time" + ] # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or []) for callback in callbacks: @@ -1876,7 +1883,7 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - custom_pricing: Final = self._custom_pricing_for(priced_result) + custom_pricing: Final = self.custom_pricing_for(priced_result) prompt = self._prompt_for_cost_calculation() @@ -1903,7 +1910,7 @@ class Logging(LiteLLMLoggingBaseClass): if hasattr(self, "litellm_params") and self.litellm_params else None ), - "vertex_location": _resolve_vertex_location_for_cost( + "vertex_location": resolve_vertex_location_for_cost( custom_llm_provider=self.model_call_details.get("custom_llm_provider", None), litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None), optional_params=self.optional_params, @@ -2015,7 +2022,7 @@ class Logging(LiteLLMLoggingBaseClass): model=model, completion_response=result, custom_llm_provider=custom_llm_provider, - custom_pricing=self._custom_pricing_for(result), + custom_pricing=self.custom_pricing_for(result), base_model=_get_base_model_from_metadata(model_call_details=self.model_call_details), router_model_id=router_model_id or self.get_router_model_id(), region_name=_resolve_mantle_region_for_cost( @@ -2040,20 +2047,23 @@ class Logging(LiteLLMLoggingBaseClass): usage=usage, ) - def _custom_pricing_for(self, result: object) -> bool: - litellm_params: Final = getattr(self, "litellm_params", None) + def spilled_over_ptu_for(self, result: object) -> bool: result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) additional_headers: Final = ( result_hidden_params.get("additional_headers") if isinstance(result_hidden_params, dict) else getattr(result_hidden_params, "additional_headers", None) ) - spilled_over: Final = is_spilled_over_ptu_request( - model_info=_deployment_model_info(litellm_params), + return is_spilled_over_ptu_request( + model_info=_deployment_model_info(getattr(self, "litellm_params", None)), response_headers=self.model_call_details.get("response_headers"), additional_headers=additional_headers, ) - return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params) + + def custom_pricing_for(self, result: object) -> bool: + if self.spilled_over_ptu_for(result): + return False + return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None)) def _prompt_for_cost_calculation(self) -> str: """ @@ -2260,6 +2270,16 @@ class Logging(LiteLLMLoggingBaseClass): except Exception: return True + @property + def async_success_log_enqueued(self) -> bool: + """True once the success log task was actually created, not merely claimed, so a later failure raised + around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second + time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False.""" + return self._async_success_enqueued + + def mark_async_success_log_enqueued(self) -> None: + self._async_success_enqueued = True + def has_run_logging( self, event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"], @@ -3261,6 +3281,12 @@ class Logging(LiteLLMLoggingBaseClass): model_name=self.get_deployment_model_for_cost(), litellm_params=self.litellm_params, model_info=self.get_router_deployment_model_info(), + vertex_location=resolve_vertex_location_for_cost( + custom_llm_provider=self.custom_llm_provider, + litellm_params=self.litellm_params if hasattr(self, "litellm_params") else None, + optional_params=None, + model=self.get_deployment_model_for_cost() or self.model, + ), ) result._hidden_params["response_cost"] = batch_result.cost @@ -6519,13 +6545,16 @@ def get_standard_logging_object_payload( ) ## Get model cost information ## - base_model = _get_base_model_from_metadata(model_call_details=kwargs) - # The router overrides completion_response.model to the model-group alias before - # this payload is built, so cost-map lookup via that alias always misses. - # Fall back to the actual deployment model set by the router in metadata. - if base_model is None: - base_model = metadata.get("deployment") - custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params) + configured_base_model: Final = _get_base_model_from_metadata(model_call_details=kwargs) + spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj) + # The router overrides completion_response.model to the model-group alias before this payload is + # built, so cost-map lookup via that alias misses and the deployment model stands in for base_model. + # A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates, + # which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there. + base_model: Final = ( + metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model + ) + custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") llm_response_cost: Final[float] = raw_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 8ca79874564..d935823fe5c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1071,6 +1071,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu ) +def _modality_token_cost( + model_info: ModelInfo, + modality_cost_key: str, + tokens: int, + prompt_base_cost: float, + service_tier: str | None, +) -> float: + """ + Audio, image and video prompt tokens bill at their modality rate when the model has one. + Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which + `_get_token_base_cost` has already resolved for the service tier and long-context tier. + """ + tiered_key: Final = _get_service_tier_cost_key(modality_cost_key, service_tier) + cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key + if model_info.get(cost_key) is None: + return float(tokens) * prompt_base_cost + return calculate_cost_component(model_info, cost_key, tokens) + + def _calculate_input_cost( prompt_tokens_details: PromptTokensDetailsResult, model_info: ModelInfo, @@ -1082,8 +1101,14 @@ def _calculate_input_cost( ) -> float: """ Calculates the input cost for a given model, prompt tokens, and completion tokens. + + A model priced per character (`input_cost_per_character`) bills the reported character count + in place of its text tokens, so the same prompt is never charged under both units. """ - prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost + bills_characters: Final = ( + prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None + ) + prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] @@ -1101,28 +1126,37 @@ def _calculate_input_cost( if prompt_tokens_details["audio_tokens"] and not ( prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None ): - audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) - prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) + prompt_cost += _modality_token_cost( + model_info, + "input_cost_per_audio_token", + prompt_tokens_details["audio_tokens"], + prompt_base_cost, + service_tier, + ) ### IMAGE TOKEN COST if prompt_tokens_details["image_tokens"] and not ( prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None ): - # For image token costs: - # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. - image_token_cost_key = "input_cost_per_image_token" - if model_info.get(image_token_cost_key) is None: - image_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]) + prompt_cost += _modality_token_cost( + model_info, + "input_cost_per_image_token", + prompt_tokens_details["image_tokens"], + prompt_base_cost, + service_tier, + ) ### VIDEO TOKEN COST if prompt_tokens_details["video_tokens"] and not ( prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None ): - video_token_cost_key = "input_cost_per_video_token" - if model_info.get(video_token_cost_key) is None: - video_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"]) + prompt_cost += _modality_token_cost( + model_info, + "input_cost_per_video_token", + prompt_tokens_details["video_tokens"], + prompt_base_cost, + service_tier, + ) ### CACHE WRITING COST - Now uses tiered pricing if ( diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 503814cc143..6c38a6ba2d1 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -4,6 +4,7 @@ from functools import reduce from typing import Final import httpx +from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers @@ -101,6 +102,32 @@ def response_timing_metrics( return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms} +_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object]) + + +def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]: + try: + return _HIDDEN_PARAMS_DICT.validate_python(hidden_params) + except ValidationError: + return {} + + +def _existing_provider_response_model(result: object) -> str | None: + """The pricing model a response already carries, like the decisions path sets, wins over the response model.""" + hidden_params: Final[object] = getattr(result, "_hidden_params", None) + carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr( + hidden_params, "provider_response_model", None + ) + return carried if isinstance(carried, str) and carried else None + + +def _provider_response_model(result: object) -> str | None: + """The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the + client asked for and cost-map lookups for the standard logging payload would otherwise miss.""" + model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None + return model if isinstance(model, str) and model else None + + class ResponseMetadata: """ Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses @@ -133,6 +160,8 @@ class ResponseMetadata: preserve_litellm_internal_headers=True, ), "litellm_model_name": model, + "provider_response_model": _existing_provider_response_model(self.result) + or _provider_response_model(self.result), } self._update_hidden_params(new_params) diff --git a/litellm/litellm_core_utils/request_timeout_resolver.py b/litellm/litellm_core_utils/request_timeout_resolver.py index a32651fedee..0e011dc06a0 100644 --- a/litellm/litellm_core_utils/request_timeout_resolver.py +++ b/litellm/litellm_core_utils/request_timeout_resolver.py @@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that: from __future__ import annotations -from typing import Final +from collections.abc import Mapping +from typing import Final, Literal, Protocol + +from pydantic import TypeAdapter from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None) + def get_configured_request_timeout() -> float | None: """Return the explicitly-configured ``litellm.request_timeout``, else ``None``.""" @@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None: if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS): return timeout return None + + +class _DeploymentParams(Protocol): + def get(self, key: str, /) -> object: ... + + +def get_service_tier_timeout( + service_tier: object, + litellm_params: _DeploymentParams, + key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"], +) -> float | None: + """Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``.""" + if not isinstance(service_tier, str): + return None + timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key)) + return None if timeouts is None else timeouts.get(service_tier) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 3bffee48d6a..c641ebecdb4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -103,6 +103,7 @@ from ..common_utils import ( process_anthropic_headers, requires_native_compaction_beta, strip_advisor_blocks_from_messages, + tools_marked_for_a_tool_config_point, ) if TYPE_CHECKING: @@ -2056,6 +2057,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Remove internal LiteLLM parameters that should not be sent to Anthropic API optional_params.pop("is_vertex_request", None) optional_params.pop("client_metadata", None) + request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature + dict[str, object], optional_params + ) + marked_tools: Final = tools_marked_for_a_tool_config_point( + request_params.get("tools"), request_params.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature + Mapping[str, object], litellm_params + ) + AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1) # ``top_k`` is a provider-specific kwarg that bypasses # ``map_openai_params``; gate it here, the single boundary shared by @@ -2075,6 +2089,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "model": model, "messages": anthropic_messages, **optional_params, + **({} if marked_tools is None else {"tools": list(marked_tools)}), } self._apply_output_config(data=data, model=model, optional_params=optional_params) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bf2d588dd3a..68439e5a12c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -33,6 +33,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, + ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, @@ -73,7 +74,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) -_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -115,6 +116,40 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None ) +def _tool_config_point_control(points: object) -> Mapping[object, object] | None: + point_fields: Final = tuple(_validated_claude_code_mapping(point) for point in validated_object_list(points) or ()) + tool_config_point: Final = next( + (fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None + ) + if tool_config_point is None: + return None + control: Final = tool_config_point.get("control") + return {"type": "ephemeral"} if control is None else _validated_claude_code_mapping(control) + + +def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None: + """Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches. + + Anthropic caches tools as one prefix ending at the marked tool, so the last one carries + the mark, skipping tool-search tools whose transform drops cache markers. A client that + already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked. + """ + control: Final = _tool_config_point_control(points) + tool_list: Final = validated_object_list(tools) or () + tool_fields: Final = tuple(_validated_claude_code_mapping(tool) for tool in tool_list) + if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields): + return None + cacheable: Final = tuple( + (position, fields) + for position, fields in enumerate(tool_fields) + if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + ) + if not cacheable: + return None + position, fields = cacheable[-1] + return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :]) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code sends its API calls through the Anthropic SDK as `claude-cli/` and its own fetches, such as gateway model discovery, as `claude-code/`""" @@ -128,9 +163,9 @@ def _validated_claude_code_mapping(value: object) -> dict[object, object] | None return None -def _validated_claude_code_list(value: object) -> list[object] | None: +def validated_object_list(value: object) -> Sequence[object] | None: try: - return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value) + return OBJECT_LIST_ADAPTER.validate_python(value) except ValidationError: return None @@ -157,7 +192,7 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None: if isinstance(system, str): return (system,) - blocks: Final = _validated_claude_code_list(system) + blocks: Final = validated_object_list(system) if blocks is None: return None block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index ac4240690c1..cd0e1dcd8a5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -12,6 +12,7 @@ from functools import partial from typing import Any, Final, cast import litellm +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( @@ -19,6 +20,7 @@ from litellm.llms.anthropic.common_utils import ( sanitize_tool_use_ids_in_anthropic_messages, strip_empty_content_blocks_from_anthropic_messages, strip_provider_specific_fields_from_anthropic_messages, + tools_marked_for_a_tool_config_point, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -636,7 +638,13 @@ def anthropic_messages_handler( f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}" ) - local_vars.update(kwargs) + request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped + marked_tools: Final = tools_marked_for_a_tool_config_point( + local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) + local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( params=local_vars, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index f753e87fee3..3d9c3ea32b3 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -4,11 +4,12 @@ import asyncio import json import traceback from collections import deque -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncGenerator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) @@ -25,6 +26,7 @@ from .transformation import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject + from litellm.types.llms.openai import ResponseIncompleteEvent class AnthropicResponsesStreamWrapper: @@ -391,8 +393,11 @@ class AnthropicResponsesStreamWrapper: raise StopAsyncIteration - async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: - """Yield SSE-encoded bytes for each Anthropic event chunk.""" + def async_anthropic_sse_wrapper(self) -> "AnthropicResponsesSSEByteStream": + """SSE-encoded bytes for each Anthropic event chunk.""" + return AnthropicResponsesSSEByteStream(self) + + async def _sse_frames(self) -> AsyncGenerator[bytes, None]: async for chunk in self: if isinstance(chunk, dict): event_type: str = str(chunk.get("type", "message")) @@ -400,3 +405,30 @@ class AnthropicResponsesStreamWrapper: yield payload.encode() else: yield chunk + + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + return partial_responses_incomplete_event_on_disconnect(self.responses_stream) + + +class AnthropicResponsesSSEByteStream: + """ + The byte stream the proxy relays for a default-route /v1/messages request. + Keeps the Responses API iterator reachable so a client disconnect can bill + the partial output the way native /v1/responses streams do. + """ + + def __init__(self, wrapper: AnthropicResponsesStreamWrapper) -> None: + self._wrapper: Final = wrapper + self._frames: Final = wrapper._sse_frames() # pyright: ignore[reportPrivateUsage] # the wrapper's own byte stream + + def __aiter__(self) -> "AnthropicResponsesSSEByteStream": + return self + + async def __anext__(self) -> bytes: + return await self._frames.__anext__() + + async def aclose(self) -> None: + await self._frames.aclose() + + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + return self._wrapper.partial_response_on_disconnect() diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index 11c99d130be..be66b395b1c 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -54,6 +54,8 @@ class NativeVertexBatchCostCalculator(Protocol): model: str, custom_llm_provider: str | None = None, model_info: ModelInfo | None = None, + *, + vertex_location: str | None = None, ) -> tuple[float, float]: ... @@ -90,6 +92,7 @@ def native_vertex_batch_row_stats( model_info: ModelInfo | None, calculate_usage: Callable[[GenerateContentResponseBody], Usage], cost_calculator: NativeVertexBatchCostCalculator, + vertex_location: str | None = None, ) -> NativeVertexBatchRowStats | None: """ Usage and cost of one native Vertex predictions.jsonl row, a @@ -121,7 +124,11 @@ def native_vertex_batch_row_stats( ) try: prompt_cost, completion_cost = cost_calculator( - usage=usage, model=model, custom_llm_provider="vertex_ai", model_info=model_info + usage=usage, + model=model, + custom_llm_provider="vertex_ai", + model_info=model_info, + vertex_location=vertex_location, ) except Exception as e: # noqa: BLE001 # one unpriceable row must not abort the batch's cost accounting verbose_logger.warning( diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index e4bbdd1bd0d..e4f7c513eb9 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,14 +1,34 @@ import types from typing import Final, Literal -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter +from typing_extensions import ReadOnly, TypedDict from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels -from litellm.types.utils import EmbeddingResponse, Usage +from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage from .types import * +class _PredictMetadata(TypedDict, total=False): + billableCharacterCount: ReadOnly[int] + + +class _PredictResponseMetadata(TypedDict, total=False): + metadata: ReadOnly[_PredictMetadata] + + +_PREDICT_RESPONSE_METADATA: Final = TypeAdapter(_PredictResponseMetadata) + + +def billable_character_count(response: _PredictResponseMetadata) -> int | None: + """ + `metadata.billableCharacterCount` of a Vertex text embedding predict response, the count Google + bills per-character embedding models (text-embedding-005, text-multilingual-embedding-002) on. + """ + return response.get("metadata", {}).get("billableCharacterCount") + + class VertexAITextEmbeddingConfig(BaseModel): """ Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#TextEmbeddingInput @@ -248,7 +268,15 @@ class VertexAITextEmbeddingConfig(BaseModel): model_response.object = "list" model_response.data = embedding_response model_response.model = model - usage: Final = Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens) + billable_characters: Final = billable_character_count(_PREDICT_RESPONSE_METADATA.validate_python(response)) + usage: Final = Usage( + prompt_tokens=input_tokens, + completion_tokens=0, + total_tokens=input_tokens, + prompt_tokens_details=( + None if billable_characters is None else PromptTokensDetailsWrapper(character_count=billable_characters) + ), + ) setattr(model_response, "usage", usage) return model_response diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index fc67aa8c553..a54689a8f4d 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -5,6 +5,8 @@ from typing import Final from pydantic import TypeAdapter +from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout + DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 _SECONDS: Final = TypeAdapter(float) @@ -48,21 +50,27 @@ def resolve_llm_passthrough_timeout( Anthropic /v1/messages). Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params - timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout - -> 600s default. + timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout + -> general_settings.pass_through_request_timeout -> 600s default. Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before any generic timeout, matching ``Router._get_stream_timeout`` on the completion route: - kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the - non-streaming chain above. + kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] -> + litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above. Only the first set value is validated as seconds, so a value in a lower-precedence field never fails the call. """ request: Final = kwargs if kwargs is not None else _NO_PARAMS deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS + service_tier: Final = request.get("service_tier") stream_candidates: Final = ( - (request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout) + ( + request.get("stream_timeout"), + get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"), + deployment.get("stream_timeout"), + router_stream_timeout, + ) if request.get("stream") else () ) @@ -70,6 +78,7 @@ def resolve_llm_passthrough_timeout( *stream_candidates, request.get("timeout"), request.get("request_timeout"), + get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"), deployment.get("timeout"), deployment.get("request_timeout"), router_timeout, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1d642e72de5..ef165e58134 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -111,6 +111,7 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails +from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.router_utils.common_utils import resolve_model_group_alias @@ -418,6 +419,13 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons # A natural end-of-stream success event already fired and released the # slot; do not bill again, and let the caller skip the slot release. return True + partial_responses_event: Final = partial_responses_incomplete_event_on_disconnect(response) + if partial_responses_event is not None: + verbose_proxy_logger.debug( + "Billing partial streamed Responses API spend after client disconnect, litellm_call_id=%s", + logging_obj.litellm_call_id, + ) + return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event) chunks: Final[object] = getattr(response, "chunks", None) if not isinstance(chunks, list) or not chunks: return False @@ -447,6 +455,10 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons partial_usage: Final = getattr(partial_response, "usage", None) if isinstance(partial_usage, Usage): backfill_missing_cache_usage_fields(partial_usage) + return await _dispatch_disconnect_billing_event(logging_obj, partial_response) + + +async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool: try: await logging_obj.dispatch_success_handlers( partial_response, diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c2bd7ea217..e5aea016839 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -119,6 +119,23 @@ def coerce_numeric_form_fields( } +def _parse_form_metadata(raw_metadata: str) -> object: + """The JSON body path rejects a lone surrogate escape (``"\\ud800"``) with a + 400 because no provider or database can encode it. ``json.loads`` accepts + the escape, so the multipart ``metadata`` field gets the same check.""" + parsed: Final[object] = json.loads(raw_metadata) + try: + json.dumps(parsed, ensure_ascii=False).encode("utf-8") + except UnicodeEncodeError as e: + raise ProxyException( + message=f"Invalid form payload: metadata {e}", + type="invalid_request_error", + param="metadata", + code=status.HTTP_400_BAD_REQUEST, + ) from e + return parsed + + async def _read_request_body(request: Request | None) -> dict: """ Safely read the request body and parse it as JSON. @@ -158,9 +175,13 @@ async def _read_request_body(request: Request | None) -> dict: param="request_body", code=status.HTTP_400_BAD_REQUEST, ) - parsed_body = dict(form_data) - if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str): - parsed_body["metadata"] = json.loads(parsed_body["metadata"]) + form_body: Final = dict(form_data) + raw_metadata: Final = form_body.get("metadata") + parsed_body = ( + {**form_body, "metadata": _parse_form_metadata(raw_metadata)} + if isinstance(raw_metadata, str) + else form_body + ) else: # Read the request body body: Final = await request.body() diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 17e6152bef6..cfc2563988f 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -88,6 +88,7 @@ if TYPE_CHECKING: from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution from litellm.proxy.utils import PrismaClient, ProxyLogging + from litellm.router import Router else: PrismaClient = Any ProxyLogging = Any @@ -97,6 +98,34 @@ RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, Call _SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +def _turn_chains_through_the_db(payload: Mapping[str, object], llm_router: "Router | None") -> bool: + """A Responses turn the chat-completions bridge replays from its spend log on the next `previous_response_id`. + + Native Responses providers hold the conversation themselves, so their rows can wait for the batch write. + A row with no router deployment behind it is treated as bridged, which keeps today's immediate flush. + """ + if payload.get("call_type") not in RESPONSES_SESSION_CALL_TYPES: + return False + model_id: Final = payload.get("model_id") + deployment: Final = ( + llm_router.get_deployment(model_id=model_id) if llm_router is not None and isinstance(model_id, str) else None + ) + if deployment is None: + return True + from litellm.responses.main import provider_stores_the_conversation + + litellm_params: Final = deployment.litellm_params + return not provider_stores_the_conversation( + litellm_params.model, + litellm_params.custom_llm_provider, + { + "use_chat_completions_api": litellm_params.use_chat_completions_api, + "model_info": deployment.model_info.model_dump(), + "api_base": litellm_params.api_base, + }, + ) + + def _org_member_transaction_key(org_id: str, user_id: str) -> str: return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}" @@ -527,12 +556,13 @@ class DBSpendUpdateWriter: get_request_model_access_groups, ) + llm_router: Final = get_llm_router() payload: Final = get_logging_payload( kwargs=kwargs, response_obj=completion_response, start_time=start_time, end_time=end_time, - llm_router=get_llm_router(), + llm_router=llm_router, ) payload["spend"] = response_cost or 0.0 if isinstance(payload["startTime"], datetime): @@ -547,7 +577,10 @@ class DBSpendUpdateWriter: payload["team_id"] = team_id if not await self._record_spend_log( - payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs + payload=payload, + prisma_client=prisma_client, + disable_spend_logs=disable_spend_logs, + llm_router=llm_router, ): return False @@ -608,14 +641,18 @@ class DBSpendUpdateWriter: return True async def _record_spend_log( - self, payload: SpendLogsPayload, prisma_client: "PrismaClient | None", disable_spend_logs: bool + self, + payload: SpendLogsPayload, + prisma_client: "PrismaClient | None", + disable_spend_logs: bool, + llm_router: "Router | None" = None, ) -> bool: if prisma_client is not None and _is_batch_cost_row(payload): return await self._claim_batch_cost_spend_log( payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs ) if disable_spend_logs is False: - await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client) + await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router) return True async def _claim_batch_cost_spend_log( @@ -1462,6 +1499,7 @@ class DBSpendUpdateWriter: payload: dict | SpendLogsPayload, prisma_client: PrismaClient | None = None, spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"), + llm_router: "Router | None" = None, ) -> PrismaClient | None: verbose_proxy_logger.debug( "Writing spend log to db - request_id: {}, spend: {}".format( @@ -1472,7 +1510,7 @@ class DBSpendUpdateWriter: from litellm.proxy.utils import enqueue_spend_logs, request_spend_log_flush await enqueue_spend_logs(prisma_client, (payload,)) - if payload.get("call_type") in RESPONSES_SESSION_CALL_TYPES: + if _turn_chains_through_the_db(_SPEND_METADATA_ADAPTER.validate_python(payload), llm_router): request_spend_log_flush(prisma_client) else: verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.") diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 92cda0d61e2..a1cf81e4540 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -3,10 +3,11 @@ import traceback from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar -from datetime import datetime +from datetime import datetime, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -33,6 +34,7 @@ from litellm.proxy.db.db_spend_update_writer import ( debitable_model_access_groups, get_llm_router, ) +from litellm.proxy.hooks.search_free_tier import apply_search_free_tier from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.spend_tracking.spend_event import ( ObjectMapping, @@ -53,7 +55,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_request_model_access_groups, should_store_prompts_and_responses_in_spend_logs, ) -from litellm.proxy.utils import ProxyUpdateSpend +from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend from litellm.types.utils import ( CallTypes, LiteLLMBatch, @@ -126,6 +128,79 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter: return proxy_logging_obj.db_spend_update_writer +def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]: + """The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any.""" + attempts: Final = request_data.get("billed_failed_attempts") + if not isinstance(attempts, tuple): + return () + lifted: Final = cast("tuple[object, ...]", attempts) # cast-ok: lifted by _failure_fields_to_lift, checked below + return tuple(attempt for attempt in lifted if isinstance(attempt, BilledFailedAttempt)) + + +def _failure_spend_rows( + request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object +) -> tuple[tuple[float, Mapping[str, object]], ...]: + """One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the + request id, then every earlier attempt the provider also billed under its own row.""" + final_row: Final[Mapping[str, object]] = ( + request_data + if late_guardrails_snapshot is None + else {**request_data, "standard_logging_object": late_guardrails_snapshot} + ) + return ( + (final_cost, final_row), + *( + (attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position)) + for position, attempt in enumerate(_billed_failed_attempts(request_data)) + ), + ) + + +def _attempt_spend_row_kwargs( + request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int +) -> Mapping[str, object]: + """The final attempt's row kwargs re-keyed for one earlier billed attempt: its own + ``request_id`` (``-attempt-``, since ``request_id`` is the spend-log primary key + and the retry count restarts after a fallback hop), its estimated usage and cost, its retry + count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``.""" + suffix: Final = f"-attempt-{position}" + raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict + litellm_params: Final = dict(raw_params) + metadata: Final = { + **cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict + "attempted_retries": attempt.attempted_retries, + } + metadata.pop("previous_models", None) + if attempt.deployment_id is not None: + metadata["model_info"] = { + **cast("Mapping[str, object]", metadata.get("model_info") or {}), # cast-ok: router dict + "id": attempt.deployment_id, + } + litellm_params["metadata"] = metadata + row: Final = { + **request_data, + "litellm_params": litellm_params, + "combined_usage_object": attempt.usage, + "response_cost": attempt.response_cost, + } + row.pop("billed_failed_attempts", None) + call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id") + if isinstance(call_id, str): + row["litellm_call_id"] = f"{call_id}{suffix}" + standard_logging_object: Final = request_data.get("standard_logging_object") + if isinstance(standard_logging_object, dict): + payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict + if isinstance(payload.get("id"), str): + row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"} + return row + + +def _str_keyed_fields(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return {} + return cast("Mapping[str, object]", value) # cast-ok: payload dicts are str-keyed + + class _ProxyDBLogger(CustomLogger): def __init__( self, @@ -293,20 +368,40 @@ class _ProxyDBLogger(CustomLogger): existing_metadata.get("standard_logging_guardrail_information") ) - await self._spend_writer().update_database( - token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), - response_cost=recovered_response_cost, - user_id=user_api_key_dict.user_id, - end_user_id=user_api_key_dict.end_user_id, - team_id=user_api_key_dict.team_id, - kwargs=request_data, - completion_response=original_exception, + await self._write_failure_spend_rows( + user_api_key_dict=user_api_key_dict, + original_exception=original_exception, start_time=actual_start_time, - end_time=datetime.now(), - org_id=user_api_key_dict.org_id, - project_id=user_api_key_dict.project_id, + spend_rows=_failure_spend_rows( + cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below + recovered_response_cost, + None, + ), ) + async def _write_failure_spend_rows( + self, + user_api_key_dict: UserAPIKeyAuth, + original_exception: Exception, + start_time: datetime, + spend_rows: Sequence[tuple[float, Mapping[str, object]]], + ) -> None: + """One spend row (and one key/team/user spend deduction) per ``(response_cost, kwargs)`` pair.""" + for row_cost, row_kwargs in spend_rows: + await self._spend_writer().update_database( + token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), + response_cost=row_cost, + user_id=user_api_key_dict.user_id, + end_user_id=user_api_key_dict.end_user_id, + team_id=user_api_key_dict.team_id, + kwargs=dict(row_kwargs), + completion_response=original_exception, + start_time=start_time, + end_time=datetime.now(), + org_id=user_api_key_dict.org_id, + project_id=user_api_key_dict.project_id, + ) + @log_db_metrics async def _PROXY_track_cost_callback( self, @@ -371,8 +466,21 @@ class _ProxyDBLogger(CustomLogger): key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None)) end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None) sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None) + raw_cost: Final = _str_keyed_fields( + sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload + ).get("response_cost") + priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None response_cost: Final = ( - sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None) + await apply_search_free_tier( + kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload + sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists + response_cost=priced_cost, + completion_response=completion_response, + counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache, + now=datetime.now(timezone.utc), + ) + if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None + else priced_cost ) tags: Final = _get_request_tags_for_cost_tracking( sl_object=sl_object, diff --git a/litellm/proxy/hooks/search_free_tier.py b/litellm/proxy/hooks/search_free_tier.py new file mode 100644 index 00000000000..254fe567bf2 --- /dev/null +++ b/litellm/proxy/hooks/search_free_tier.py @@ -0,0 +1,123 @@ +from collections.abc import MutableMapping +from datetime import datetime +from types import MappingProxyType +from typing import Final, Protocol + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time +from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage +from litellm.llms.gemini.cost_calculator import cost_per_web_search_request +from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage + +_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"}) +_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"}) +_TTL_GRACE_SECONDS: Final = 3600 +_COUNTED_KWARG: Final = "search_free_tier_counted" +_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown) + + +class SharedCounter(Protocol): + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ... + + +class _UsageCarrier(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + usage: Usage | None = None + + +def freed_units(allowance: int, units: int, total_after: int) -> int: + """Units of this request that landed under the allowance, given the shared total after they were added.""" + return max(0, min(units, allowance - (total_after - units))) + + +def search_free_tier_key(provider: str, allowance: SearchFreeRequests, now: datetime) -> str: + period_stamp: Final = now.strftime(_PERIOD_STAMP[allowance["period"]]) + return f"litellm:search_free_tier:{provider}:{allowance['shared_group']}:{period_stamp}" + + +def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int: + reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC") + return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS + + +def _usage_of(completion_response: object) -> Usage | None: + try: + return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage + except ValidationError: + return None + + +def _grounding_queries(usage: Usage) -> int: + details: Final = usage.prompt_tokens_details + from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None + return from_details or get_web_search_requests_from_usage(usage) or 0 + + +def _billable_units(usage: Usage, model_info: ModelInfo) -> int: + queries: Final = _grounding_queries(usage) + if queries == 0: + return 0 + return queries if model_info.get("web_search_billing_unit") == "per_query" else 1 + + +def _record_freed_cost( + kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it + sl_object: StandardLoggingPayload, + original_cost: float, + freed_cost: float, +) -> float: + reduced_cost: Final = original_cost - freed_cost + breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost) + sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload + { + **breakdown, + "total_cost": reduced_cost, + "original_cost": breakdown.get("original_cost", original_cost), + "discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost, + } + ) + sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload + kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"] + return reduced_cost + + +async def apply_search_free_tier( + kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it + sl_object: StandardLoggingPayload, + response_cost: float, + completion_response: object, + counter: SharedCounter | None, + now: datetime, +) -> float: + """Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged.""" + model_info: Final = sl_object["model_map_information"]["model_map_value"] + allowance: Final = model_info.get("search_free_requests") if model_info is not None else None + usage: Final = _usage_of(completion_response) + if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]: + return response_cost + units: Final = _billable_units(usage, model_info) + if units == 0 or kwargs.get(_COUNTED_KWARG): + return response_cost + if counter is None: + verbose_proxy_logger.warning( + "apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price" + ) + return response_cost + provider: Final = sl_object["custom_llm_provider"] or model_info["litellm_provider"] + key: Final = search_free_tier_key(provider, allowance, now) + try: + total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now)) + except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row + verbose_proxy_logger.warning( + "search free tier counter %s is unavailable, grounding is billed at full price", key + ) + return response_cost + kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once + freed: Final = freed_units(allowance["count"], units, int(total_after)) + if freed == 0: + return response_cost + freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units) + return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 56f647d5acc..0acdbfaedd8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -358,8 +358,9 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess # ``litellm.model_cost`` map. Both effects belong to deployment configuration, # not to user-supplied request bodies, so the proxy strips them before they # reach the call path. Built from the Pydantic model so newly-added pricing -# fields are covered automatically. -_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) +# fields are covered automatically. ``base_model`` selects which cost-map entry +# prices the row, so a caller naming a cheaper model lowers their own spend. +_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"} # ``model_info`` carries the same pricing fields when read by # ``use_custom_pricing_for_model``; strip from metadata for the same reason. # ``standard_logging_guardrail_information`` is proxy-written telemetry summed @@ -373,6 +374,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( { "attempted_fallbacks", "original_model_group", + "previous_models", "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b8cc30ad8a7..6a03bc7ad83 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -46,11 +46,13 @@ from typing import ( overload, ) +import httpx from typing_extensions import ReadOnly, TypedDict from litellm import _custom_logger_compatible_callbacks_literal from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, + LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, PROXY_REJECTED_BEFORE_ROUTING_KEY, @@ -143,7 +145,8 @@ from litellm.litellm_core_utils.core_helpers import ( independent_snapshot, is_expected_client_error, ) -from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -236,6 +239,7 @@ from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.router_utils.deployment_pricing import pricing_models from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES from litellm.types.llms.openai import ResponsesAPIResponse @@ -248,6 +252,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionRe from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams from litellm.utils import ( _add_custom_logger_callback_to_specific_event, # pyright: ignore[reportPrivateUsage] # only string-to-logger helper + _get_base_model_from_metadata, # pyright: ignore[reportPrivateUsage] # main exposes it as get_base_model_from_metadata ) if TYPE_CHECKING: @@ -939,20 +944,48 @@ def _failure_usage_to_lift( model_call_details: Mapping[str, object], request_body: Mapping[str, object], dispatched: bool, + provider_billed: bool = False, + deployment_id: str | None = None, + base_model: str | None = None, ) -> tuple[object, object] | None: """A stream that broke mid-flight still billed the provider for the chunks already delivered; the streaming handler stashes that recovered usage and cost in model_call_details, so prefer it. Otherwise a request that was dispatched to a provider and failed without upstream usage gets an - estimated input-side Usage with zero cost. The raw request body backfills - the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a - chat-completions provider) without filling optional_params. Returns the + estimated input-side Usage. When the failure is one the provider bills + anyway (``provider_billed``: a read timeout or a client disconnect, where + the provider had the prompt and kept working after LiteLLM hung up) that + usage is priced at the model's input rate, the way the success path prices it: + at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``) + when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure + deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch + failure stays at zero cost. Returns the (combined_usage_object, response_cost) pair to lift, or None.""" recovered_usage: Final = model_call_details.get("combined_usage_object") if recovered_usage is not None: return recovered_usage, model_call_details.get("response_cost") - if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL): + if not dispatched: return None + if model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL) and not provider_billed: + # The proxy-only error handler stamps this flag on a client disconnect too, even though the + # request had already reached the provider, so a billed failure keeps its estimate. + return None + estimated_usage: Final = _estimated_prompt_usage(model_call_details, request_body) + if estimated_usage is None: + return None + if not provider_billed: + return estimated_usage, 0.0 + return estimated_usage, _estimated_input_cost( + model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model + ) + + +def _estimated_prompt_usage( + model_call_details: Mapping[str, object], request_body: Mapping[str, object] +) -> Usage | None: + """The input-only Usage estimate for the prompt this request sent, or None when the call type has no + estimable input. The raw request body backfills the system prompt when the SDK bridges an endpoint + (e.g. /v1/messages on a chat-completions provider) without filling optional_params.""" if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES: return None optional_params: Final = model_call_details.get("optional_params") @@ -962,14 +995,205 @@ def _failure_usage_to_lift( else None ) system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions") - estimated_usage: Final = _estimate_dispatched_failure_usage( + return _estimate_dispatched_failure_usage( model=str(model_call_details.get("model") or ""), request_input=model_call_details.get("messages"), system_input=system_input, ) - if estimated_usage is None: + + +@dataclass(frozen=True) +class BilledFailedAttempt: + """An earlier retry attempt the provider also billed, written as its own spend row.""" + + attempted_retries: int + deployment_id: str | None + usage: Usage + response_cost: float + + +_NO_RETRY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _retry_metadata_bucket(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]: + """The request metadata bucket the router writes retry state into, ``litellm_metadata`` when the + route carries one (/messages, /responses) and ``metadata`` otherwise, same order as the deployment id.""" + if litellm_params is None: + return _NO_RETRY_METADATA + for metadata_key in ("litellm_metadata", "metadata"): + metadata = litellm_params.get(metadata_key) + if isinstance(metadata, dict): + return cast("Mapping[str, object]", metadata) # cast-ok: untyped router dict, keys read with .get + return _NO_RETRY_METADATA + + +def _attempt_number(record: Mapping[str, object], index: int) -> int: + """The attempt a retry record describes: its own ``attempted_retries`` count, else its position.""" + recorded: Final = record.get("attempted_retries") + return recorded if isinstance(recorded, int) else index + + +def _retry_attempt_records(litellm_params: Mapping[str, object] | None) -> tuple[Mapping[str, object], ...]: + """The router's ``RetryAttemptRecord`` breadcrumbs, one per failed attempt, read from the same bucket.""" + if litellm_params is None: + return () + for metadata_key in ("litellm_metadata", "metadata"): + metadata = litellm_params.get(metadata_key) + if not isinstance(metadata, dict): + continue + records = cast("Mapping[str, object]", metadata).get("previous_models") # cast-ok: untyped router dict + if isinstance(records, (list, tuple)): + return tuple( + cast("Mapping[str, object]", record) # cast-ok: RetryAttemptRecord TypedDict, read with .get + for record in cast("Sequence[object]", records) # cast-ok: untyped router tuple + if isinstance(record, dict) + ) + return () + + +def _record_deployment_id(record: Mapping[str, object]) -> str | None: + deployment_id: Final = record.get("deployment_id") + return deployment_id if isinstance(deployment_id, str) else None + + +def _attempt_identity(record: Mapping[str, object]) -> tuple[str | None, int | None]: + """What tells one provider call from another across the request: the deployment that served it + and the retry count within its model group. Two records with the same identity describe the + same call (the fallback handler re-logs the attempt that exhausted the previous group).""" + attempted_retries: Final = record.get("attempted_retries") + return _record_deployment_id(record), attempted_retries if isinstance(attempted_retries, int) else None + + +def _timed_out_earlier_records( + litellm_params: Mapping[str, object] | None, final_identity: tuple[str | None, int | None] +) -> tuple[Mapping[str, object], ...]: + """The ``Timeout`` breadcrumbs for calls other than the final attempt, one per provider call.""" + by_identity: Final = { + _attempt_identity(record): record + for record in _retry_attempt_records(litellm_params) + if record.get("exception_type") == litellm.Timeout.__name__ + } + return tuple(record for identity, record in by_identity.items() if identity != final_identity) + + +def _record_pricing( + model_call_details: Mapping[str, object], record: Mapping[str, object], base_model: str | None +) -> tuple[Mapping[str, object], str | None]: + """The ``(model_call_details, base_model)`` to price one earlier attempt with: the record's own + deployment model and ``base_model`` when it served a different model than the final attempt + (the provider is then inferred from that model), else the final attempt's.""" + record_model: Final = record.get("model") + if not isinstance(record_model, str) or record_model == model_call_details.get("model"): + return model_call_details, base_model + record_base_model: Final = record.get("base_model") + return ( + {**model_call_details, "model": record_model, "custom_llm_provider": None}, + record_base_model if isinstance(record_base_model, str) else None, + ) + + +def _billed_earlier_attempt( + model_call_details: Mapping[str, object], + record: Mapping[str, object], + index: int, + estimated_usage: Usage, + base_model: str | None, +) -> BilledFailedAttempt: + pricing_details, pricing_base_model = _record_pricing(model_call_details, record, base_model) + return BilledFailedAttempt( + attempted_retries=_attempt_number(record, index), + deployment_id=_record_deployment_id(record), + usage=estimated_usage, + response_cost=_estimated_input_cost( + pricing_details, + estimated_usage.prompt_tokens, + deployment_id=_record_deployment_id(record), + base_model=pricing_base_model, + ), + ) + + +def _billed_earlier_attempts( + model_call_details: Mapping[str, object], + litellm_params: Mapping[str, object] | None, + estimated_usage: Usage, + base_model: str | None, + deployment_id: str | None, +) -> tuple[BilledFailedAttempt, ...]: + """Every earlier retry attempt the provider also billed. The router records one + ``RetryAttemptRecord`` per failed attempt in ``metadata.previous_models``, the final attempt + included, which the failure row itself bills. The final attempt is the record with the + request's own deployment and retry count (``attempted_retries`` restarts at 0 after a fallback + hop, so the count alone cannot tell the groups apart). An earlier attempt that ended in a + ``Timeout`` had the same prompt accepted by its deployment, so it is billed like the final + attempt, at that deployment's input rate (else the cost map's rate for the record's own + ``base_model`` or model, which a fallback hop can make a different model than the final one). + Only the exception class survives on the record, so an earlier connect timeout is billed too; + any other failure class is not.""" + final_attempt: Final = _retry_metadata_bucket(litellm_params).get("attempted_retries") + final_identity: Final = (deployment_id, final_attempt if isinstance(final_attempt, int) else None) + return tuple( + _billed_earlier_attempt(model_call_details, record, index, estimated_usage, base_model) + for index, record in enumerate(_timed_out_earlier_records(litellm_params, final_identity)) + ) + + +def _deployment_priced_input_cost( + model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo +) -> float: + """Input price for the estimated prompt at the deployment's registered rates, read through + ``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success + path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``) + is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0.""" + usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens) + try: + prompt_cost, _ = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""), + model_info=deployment_pricing, + ) + except Exception as exc: # noqa: BLE001 # a malformed registered entry must not drop the failure row + verbose_proxy_logger.debug("failure spend estimate: could not price deployment of %s: %s", model, exc) + return 0.0 + return prompt_cost + + +def _priced_input_cost(pricing_model: str, custom_llm_provider: str | None, prompt_tokens: int) -> float | None: + try: + estimated_input_cost, _ = litellm.cost_per_token( + model=pricing_model, + custom_llm_provider=custom_llm_provider, + prompt_tokens=prompt_tokens, + ) + except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written + verbose_proxy_logger.debug("failure spend estimate: could not price %s: %s", pricing_model, exc) return None - return estimated_usage, 0.0 + return estimated_input_cost + + +def _estimated_input_cost( + model_call_details: Mapping[str, object], + prompt_tokens: int, + deployment_id: str | None, + base_model: str | None, +) -> float: + """Input price for the estimated prompt: the deployment's registered rates when it has any (the id + is ``Logging.get_router_model_id()``, never the flattened ``model_call_details`` copy a request body + key can overwrite), else the cost map's rate for ``base_model``, else the deployment model, the same + candidate order ``completion_cost`` walks (an Azure deployment name is not in the cost map, its + ``base_model`` is; a ``base_model`` the cost map does not know falls through to the deployment + model). A model no candidate can price keeps the row at $0 rather than dropping the failure row.""" + model: Final = str(model_call_details.get("model") or "") + custom_llm_provider: Final = str(model_call_details.get("custom_llm_provider") or "") or None + deployment_pricing: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model) + if deployment_pricing is not None: + return _deployment_priced_input_cost(model, custom_llm_provider, prompt_tokens, deployment_pricing) + pricing_models_to_try: Final = pricing_models(base_model, model) + candidate_costs: Final = ( + _priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens) for pricing_model in pricing_models_to_try + ) + return next((cost for cost in candidate_costs if cost is not None), 0.0) _EMPTY_LIFT: Final = MappingProxyType({}) @@ -1085,7 +1309,38 @@ def _call_type_for_route(route: str | None) -> str | None: _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException) -def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]: +_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16 + + +def _exception_chain(exception: BaseException) -> tuple[BaseException, ...]: + chain: Final[list[BaseException]] = [exception] # mutable-ok: built once, returned frozen + for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH): + following = chain[-1].__cause__ or chain[-1].__context__ + if following is None or following in chain: + break + chain.append(following) + return tuple(chain) + + +def _timed_out_reading_from_provider(exception: Exception) -> bool: + """A read timeout means the request was sent and the provider was working on it; a connect + timeout (``httpx.ConnectTimeout``) never reached the provider, so it is not billed.""" + return isinstance(exception, litellm.Timeout) and any( + isinstance(cause, httpx.ReadTimeout) for cause in _exception_chain(exception) + ) + + +def _client_disconnected(exception: Exception) -> bool: + """The 499 the proxy itself raises when the caller hung up while the provider was still working on the + request. A provider's own 499 arrives as a litellm exception and is a plain upstream failure.""" + return isinstance(exception, HTTPException) and exception.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + + +def _provider_billed_the_failure(exception: Exception) -> bool: + return _timed_out_reading_from_provider(exception) or _client_disconnected(exception) + + +def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: bool = False) -> Mapping[str, object]: """Failure-path callbacks run after ``litellm_logging_obj`` is popped from request_data (it is not serialisable), so the caller merges these fields onto request_data first: the request start and first-handoff instants for @@ -1097,10 +1352,28 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, return _EMPTY_LIFT _model_call_details: Final = getattr(_logging_obj, "model_call_details", {}) _first_handoff: Final = _model_call_details.get("first_api_call_start_time") + _litellm_params: Final = getattr(_logging_obj, "litellm_params", None) + _base_model: Final = _get_base_model_from_metadata(model_call_details=_model_call_details) + _deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None + _success_already_logged: Final = isinstance(_logging_obj, Logging) and _logging_obj.async_success_log_enqueued + _billed: Final = provider_billed and not _success_already_logged _usage_to_lift: Final = _failure_usage_to_lift( model_call_details=_model_call_details, request_body=request_data, dispatched=_first_handoff is not None, + provider_billed=_billed, + deployment_id=_deployment_id, + base_model=_base_model, + ) + _prompt_estimate: Final = ( + _estimated_prompt_usage(_model_call_details, request_data) + if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage") + else None + ) + _earlier_attempts: Final = ( + _billed_earlier_attempts(_model_call_details, _litellm_params, _prompt_estimate, _base_model, _deployment_id) + if _prompt_estimate is not None + else () ) _entries: Final = ( ("start_time", _model_call_details.get("start_time")), @@ -1109,6 +1382,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, ("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]), ("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)), ("standard_logging_object", _model_call_details.get("standard_logging_object")), + ("billed_failed_attempts", _earlier_attempts or None), ) return MappingProxyType({key: value for key, value in _entries if value is not None}) @@ -3252,7 +3526,11 @@ class ProxyLogging: original_exception=original_exception, ) - request_data.update(await offload_token_count(_failure_fields_to_lift)(request_data)) + request_data.update( + await offload_token_count(_failure_fields_to_lift)( + request_data, provider_billed=_provider_billed_the_failure(original_exception) + ) + ) # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) @@ -7790,7 +8068,11 @@ MAX_SPEND_LOG_ISOLATION_FAILURES_PER_BATCH: Final = 256 def _is_transient_spend_log_write_error(e: Exception) -> bool: - return PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e) + return ( + PrismaDBExceptionHandler.is_database_transport_error(e) + or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) + or PrismaDBExceptionHandler.is_deadlock_error(e) + ) async def _create_spend_logs_with_poison_isolation( @@ -7824,7 +8106,7 @@ async def _create_spend_logs_with_poison_isolation( await repo.table.create_many(data=rows, skip_duplicates=True) return failure_budget except Exception as e: - if not PrismaDBExceptionHandler.is_prisma_data_error(e): + if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)): raise if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): raise diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 978a782c20c..9f89c811467 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -14,7 +14,7 @@ from litellm.responses.litellm_completion_transformation.custom_tools import ( from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) -from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator +from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PART_UNION_TYPES, @@ -1259,6 +1259,43 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) return reasoning_aligned + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: + """ + The response.incomplete event to bill when the client went away mid-stream. + Built from the chat completion chunks relayed so far, so the usage is what + stream_chunk_builder counts for them. + """ + if self.completed_response is not None or not self.collected_chat_completion_chunks: + return None + built: Final = stream_chunk_builder( + chunks=self.collected_chat_completion_chunks, + messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped + input=self.request_input, + responses_api_request=dict(self.responses_api_request), + ), + logging_obj=self.litellm_logging_obj, + ) + if not isinstance(built, ModelResponse): + return None + responses_api_response: Final = ( + LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped + request_input=self.request_input, + chat_completion_response=built, + responses_api_request=self.responses_api_request, + ) + ) + partial: Final = self._with_encoded_response_id( + responses_api_response.model_copy( + update={ + "id": self._cached_response_id or responses_api_response.id, + "status": "incomplete", + "output": list(self._output_with_streamed_item_ids(responses_api_response)), + } + ) + ) + stamp_responses_usage_cost(partial, self.litellm_logging_obj) + return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) + def _emit_terminal_response_event( self, litellm_model_response: ModelResponse ) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 93c72bc2d3b..0724d014f5d 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -547,6 +547,22 @@ def _will_bridge_to_chat_completions( ) +def provider_stores_the_conversation(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool: + """Whether the provider, not a LiteLLM spend log, holds this turn for a `previous_response_id` follow-up. + + A bridged turn is replayed from its spend log, so the bridge needs that row in the DB, and a bridged + first MCP round runs as an internal call that writes no spend log at all. + """ + api_base: Final = _api_base_kwarg(kwargs) + return not _will_bridge_to_chat_completions( + model, + custom_llm_provider or litellm.get_llm_provider(model=model, api_base=api_base)[1], + bool(kwargs.get("use_chat_completions_api")), + kwargs.get("model_info"), + api_base, + ) + + @contextmanager def _prompt_management_sees_a_provisional_message_list( kwargs: dict[str, object], # mutable-ok: the signal is read and popped out of the caller's own kwargs diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index c919c6c957c..0c167488318 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,10 +41,12 @@ from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.openai import ( PART_UNION_TYPES, ResponseAPIUsage, + ResponseIncompleteEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) +from litellm.types.responses.main import GenericResponseOutputItem, OutputText from litellm.types.utils import CallTypes from litellm.utils import async_post_call_success_deployment_hook @@ -289,6 +291,7 @@ class BaseResponsesAPIStreamingIterator: self._yielded_first_chunk = False self._generated_content = "" self._generated_tool_arguments = "" + self._created_response: ResponsesAPIResponse | None = None self._completed_response_cached = False self._completed_response_logged = False self._completed_response_cache_hit: bool | None = None @@ -450,6 +453,10 @@ class BaseResponsesAPIStreamingIterator: # Store the completed response (also for incomplete/failed so logging still fires) _chunk_type: Final = getattr(openai_responses_api_chunk, "type", None) openai_types = _get_openai_response_types() + if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED: + _created: Final[object] = getattr(openai_responses_api_chunk, "response", None) + if isinstance(_created, ResponsesAPIResponse): + self._created_response = _created if openai_responses_api_chunk and _chunk_type in ( openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, @@ -481,7 +488,7 @@ class BaseResponsesAPIStreamingIterator: else openai_responses_api_chunk.model_copy(update={"response": _billed_response}) ) self.completed_response = _terminal_chunk - _stamp_responses_usage_cost(_billed_response, self.logging_obj) + stamp_responses_usage_cost(_billed_response, self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() @@ -874,6 +881,34 @@ class BaseResponsesAPIStreamingIterator: except Exception: pass + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: + """ + The response.incomplete event to bill when the client went away mid-stream, + the same shape the success handler assembles for a natural end of stream. + Its response is the response.created payload (it carries the served + service_tier) plus the text and tool arguments relayed so far, with counted + usage since the provider never sent any. None when nothing was relayed yet + or the stream already reached its terminal event and billed itself. + """ + created: Final = self._created_response + generated: Final = self._generated_content + self._generated_tool_arguments + if self.completed_response is not None or created is None or not generated: + return None + usage: Final = _estimate_usage_safely( + self.model or "", self.request_data.get("input"), self.request_data, generated + ) + if usage is None: + return None + partial: Final = created.model_copy( + update={ + "status": "incomplete", + "usage": usage, + "output": _partial_output_items(self._generated_content), + } + ) + stamp_responses_usage_cost(partial, self.logging_obj) + return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) + async def call_post_streaming_hooks_for_testing( iterator: object, chunk: ResponsesAPIStreamingResponse @@ -1347,6 +1382,37 @@ def _add_text_like_part_events( ) +@runtime_checkable +class _BillsPartialStreamOnDisconnect(Protocol): + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: ... + + +def partial_responses_incomplete_event_on_disconnect(stream: object) -> ResponseIncompleteEvent | None: + """Ask the stream behind a proxy or router wrapper for the partial response to bill, if it can build one.""" + if not isinstance(stream, _BillsPartialStreamOnDisconnect): + return None + try: + partial: Final = stream.partial_response_on_disconnect() + except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown + verbose_logger.debug("Failed to build partial Responses API response for disconnect billing: %s", e) + return None + return partial if isinstance(partial, ResponseIncompleteEvent) else None + + +def _partial_output_items(generated_text: str) -> Sequence[GenericResponseOutputItem]: + if not generated_text: + return [] + return [ + GenericResponseOutputItem( + type="message", + id=f"msg_{uuid.uuid4()}", + status="incomplete", + role="assistant", + content=[OutputText(type="output_text", text=generated_text, annotations=[])], + ) + ] + + def _billed_terminal_response( response_obj: object, estimate: Callable[[], ResponseAPIUsage | None] | None ) -> ResponsesAPIResponse | None: @@ -1453,7 +1519,7 @@ def _estimate_usage_safely( return None -def _stamp_responses_usage_cost( +def stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: if response_obj is None or logging_obj is None: @@ -1479,7 +1545,7 @@ def build_synthetic_response_events( chunk_size: int, ) -> list[ResponsesAPIStreamingResponse]: openai_types: Final = _get_openai_response_types() - _stamp_responses_usage_cost(transformed, logging_obj) + stamp_responses_usage_cost(transformed, logging_obj) events: Final[list[ResponsesAPIStreamingResponse]] = [ _build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed), diff --git a/litellm/router.py b/litellm/router.py index 5e84f54c65d..d863d3fdb21 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -98,6 +98,7 @@ from litellm.litellm_core_utils.ptu_pricing import ( ) from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, + get_service_tier_timeout, ) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.sensitive_data_masker import ( @@ -199,6 +200,7 @@ from litellm.router_utils.cooldown_handlers import ( is_background_response_cost_poll_not_found, is_caller_timeout_408, ) +from litellm.router_utils.deployment_pricing import deployment_pricing_models from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, AttemptedFallbackTargets, @@ -368,6 +370,7 @@ if TYPE_CHECKING: from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from litellm.types.llms.openai import ( ResponseAPIUsage, + ResponseIncompleteEvent, ResponseInputParam, ResponsesAPIResponse, ) @@ -647,6 +650,11 @@ class FallbackAwareAnthropicMessagesStream: self._source_iterator = fallback_response self.fallback_headers_adopted = True + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect + + return partial_responses_incomplete_event_on_disconnect(self._source_iterator) + def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": return self @@ -3260,6 +3268,7 @@ class Router: from datetime import datetime self._async_generator = async_generator + self._billing_source: object = source_iterator # Mirror every attribute BaseResponsesAPIStreamingIterator.__init__ # would have set. The wrapper bypasses super().__init__ (it has no # httpx.Response of its own and no provider config to drive), so @@ -3308,10 +3317,29 @@ class Router: def adopt_fallback_headers(self, fallback_response: object) -> tuple[dict[str, object], dict[str, object]]: prepared: Final = Router._prepare_fallback_hidden_params(fallback_response) + self._billing_source = fallback_response self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} # mutable-ok: stream metadata self.fallback_headers_adopted = True return prepared + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + """ + The event to bill on a client disconnect. After a mid-stream hop the primary + stream's relayed tokens are folded into the fallback's event, the way + _combine_responses_fallback_usage does for a natural end of stream. + """ + from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect + + current: Final = partial_responses_incomplete_event_on_disconnect(self._billing_source) + if self._billing_source is source_iterator: + return current + primary: Final = partial_responses_incomplete_event_on_disconnect(source_iterator) + if current is None or primary is None: + return current or primary + if primary.response.usage is not None: + Router._combine_responses_fallback_usage(current, primary.response.usage) + return current + def __aiter__(self): return self @@ -4051,6 +4079,7 @@ class Router: """Helper to get stream timeout from kwargs or deployment params""" return ( kwargs.get("stream_timeout", None) # the params dynamically set by user + or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier") or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment or self.stream_timeout # timeout set on router or self.request_timeout # litellm_settings.request_timeout (per-attempt) @@ -4062,6 +4091,7 @@ class Router: timeout: Final = ( kwargs.get("timeout", None) # the params dynamically set by user or kwargs.get("request_timeout", None) # the params dynamically set by user + or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier") or data.get("timeout", None) # timeout set on litellm_params for this deployment or data.get("request_timeout", None) # timeout set on litellm_params for this deployment or self.request_timeout # litellm_settings.request_timeout (per-attempt) @@ -8313,17 +8343,24 @@ class Router: When a retry or fallback happens, record which model group, deployment and attempt just failed and why, and count it toward the request-wide num_retries_per_request cap """ + from litellm.litellm_core_utils.get_litellm_params import _get_base_model_from_litellm_call_metadata from litellm.types.router import RetryAttemptRecord _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" - request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var] + request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") model_info: Final = request_metadata.get("model_info") deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + base_model: Final = _get_base_model_from_litellm_call_metadata( + metadata=request_metadata + ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) + deployment_model: Final = request_metadata.get("deployment") attempted_retries: Final = request_metadata.get("attempted_retries") attempt_record: Final[RetryAttemptRecord] = { "model_group": model_group if isinstance(model_group, str) else None, "deployment_id": deployment_id if isinstance(deployment_id, str) else None, + "model": deployment_model if isinstance(deployment_model, str) else None, + "base_model": base_model if isinstance(base_model, str) else None, "exception_type": type(e).__name__, "exception_string": str(e), "attempted_retries": attempted_retries if type(attempted_retries) is int else None, @@ -8341,6 +8378,18 @@ class Router: kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap return kwargs + def _deployment_params_base_model(self, deployment_id: str | None) -> str | None: + """The ``base_model`` a deployment declares under ``litellm_params`` (the Azure named-deployment + shape), the same key the success path's ``get_base_model_from_metadata`` reads.""" + deployment: Final = self.get_deployment(model_id=deployment_id) if deployment_id is not None else None + if deployment is None: + return None + extra_params: Final = cast( # cast-ok: pydantic extras are untyped, base_model is isinstance-checked below + "Mapping[str, object]", deployment.litellm_params.model_extra or {} + ) + base_model: Final = extra_params.get("base_model") + return base_model if isinstance(base_model, str) else None + def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int: """ Update deployment rpm for that minute @@ -10335,19 +10384,9 @@ class Router: ) for deployment in deployments ) - params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments) - # base_model resolution mirrors get_router_model_info: unset or blank means the - # deployment's own model name is the cost-map key. - cost_map_keys: Final = tuple( - dict.fromkeys( # deduplicates while preserving config order - key - for key in ( - model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") - for model_info, litellm_params in zip(model_infos, params) - ) - if isinstance(key, str) and key - ) - ) + raw_deployments: Final[tuple[object, ...]] = deployments + pricing_keys: Final = (deployment_pricing_models(deployment) for deployment in raw_deployments) + cost_map_keys: Final = tuple(dict.fromkeys(keys[0] for keys in pricing_keys if keys)) return DeploymentModelListingInfo( cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 22c321c65fb..1659f959a14 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,5 +1,6 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) +from collections.abc import Mapping from datetime import datetime from typing import Final @@ -9,6 +10,30 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.router_utils.batch_utils import is_batch_retrieve_call_type +from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed + + +def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]: + """Input and output per-token prices from the first cost-map row among the deployment's + pricing models that carries one; (None, None) when no row does, so the caller falls back to + its unmapped default.""" + rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment)) + row: Final = next((row for row in rows if _declares_token_price(row)), None) + if row is None: + return None, None + return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token")) + + +def _cost_map() -> Mapping[str, object]: + return litellm.model_cost + + +def _declares_token_price(row: Mapping[str, object]) -> bool: + return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None + + +def _as_rate(value: object) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None class LowestCostLoggingHandler(CustomLogger): @@ -239,8 +264,8 @@ class LowestCostLoggingHandler(CustomLogger): or _deployment.get("model_info", {}).get("rpm", None) or float("inf") ) - item_litellm_model_name = _deployment.get("litellm_params", {}).get("model") - item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {}) + deployment_view: object = _deployment + map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view) # check if user provided input_cost_per_token and output_cost_per_token in litellm_params item_input_cost = None @@ -252,10 +277,10 @@ class LowestCostLoggingHandler(CustomLogger): item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token") if item_input_cost is None: - item_input_cost = item_litellm_model_cost_map.get("input_cost_per_token", 5.0) + item_input_cost = map_input_cost if map_input_cost is not None else 5.0 if item_output_cost is None: - item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) + item_output_cost = map_output_cost if map_output_cost is not None else 5.0 # if litellm["model"] is not in model_cost map -> use item_cost = $10 diff --git a/litellm/router_utils/deployment_pricing.py b/litellm/router_utils/deployment_pricing.py new file mode 100644 index 00000000000..622cc91ab5e --- /dev/null +++ b/litellm/router_utils/deployment_pricing.py @@ -0,0 +1,37 @@ +from collections.abc import Iterator, Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STR_KEYED: Final = TypeAdapter(dict[str, object]) + + +def pricing_models(base_model: object, model: object) -> tuple[str, ...]: + """Cost-map keys to price a deployment under, ``model_info.base_model`` first, then the + deployment's own model, blanks and duplicates dropped. Every spend and routing surface that + prices a deployment walks this order, so a ``base_model`` the cost map does not know falls + through to the deployment model instead of billing or ranking at the unmapped default.""" + return tuple(dict.fromkeys(_named_models(base_model, model))) + + +def deployment_pricing_models(deployment: object) -> tuple[str, ...]: + """``pricing_models`` read off a raw deployment dict (``model_info.base_model``, then + ``litellm_params.base_model``, then ``litellm_params.model``); malformed blocks count as unset.""" + view: Final = str_keyed(deployment) + params: Final = str_keyed(view.get("litellm_params")) + info: Final = str_keyed(view.get("model_info")) + return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model")) + + +def _named_models(*candidates: object) -> Iterator[str]: + return (candidate for candidate in candidates if isinstance(candidate, str) and candidate) + + +def str_keyed(value: object) -> Mapping[str, object]: + """``value`` as a read-only str-keyed mapping, empty when it is not a dict.""" + try: + return MappingProxyType(_STR_KEYED.validate_python(value)) + except ValidationError: + return _EMPTY diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 754b69142f9..b784082b00f 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -8,6 +8,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_other_format, send, @@ -29,10 +30,12 @@ def _prepare_systemone_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + fields: Final = SystemOneRequestFields(state=state, questions=questions) + log_caller_request(kwargs, fields) reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") return prepare_call( model=model, - fields=SystemOneRequestFields(state=state, questions=questions), + fields=fields, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 20e7885a2bf..7f55612f2c7 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -22,7 +22,7 @@ class CacheControlMessageInjectionPoint(TypedDict): class CacheControlToolConfigInjectionPoint(TypedDict): - """Type for tool_config-level injection points (Bedrock).""" + """Type for tool_config-level injection points (Anthropic and Bedrock).""" location: Literal["tool_config"] control: ChatCompletionCachedContent | None diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 83a42c235f9..3ebe7c7cd16 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -72,6 +72,8 @@ class ProviderConnection: request_timeout: float | None = None force_timeout: float | None = None stream_timeout: float | str | None = None + timeout_by_service_tier: Mapping[str, float] | None = None + stream_timeout_by_service_tier: Mapping[str, float] | None = None max_retries: int | None = None tenant_id: str | None = None client_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index b72809f625f..473f02058a3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -389,6 +389,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): otpm: int | None = None timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/ stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/ + # per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key + timeout_by_service_tier: Mapping[str, float] | None = None + stream_timeout_by_service_tier: Mapping[str, float] | None = None max_retries: int | None = None drop_params: bool | str | None = None organization: str | None = None # for openai orgs @@ -573,6 +576,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): api_version: str | None timeout: float | str | httpx.Timeout | None stream_timeout: float | str | None + timeout_by_service_tier: ReadOnly[Mapping[str, float] | None] + stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None] max_retries: int | None organization: list | str | None # for openai orgs provider_affinity_header: ReadOnly[str | None] @@ -996,6 +1001,8 @@ class RouterModelGroupAliasItem(TypedDict): class RetryAttemptRecord(TypedDict): model_group: ReadOnly[str | None] deployment_id: ReadOnly[str | None] + model: ReadOnly[str | None] + base_model: ReadOnly[str | None] exception_type: ReadOnly[str] exception_string: ReadOnly[str] attempted_retries: ReadOnly[int | None] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 97a51f42851..6671d0b65aa 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -223,6 +223,12 @@ class SearchContextCostPerQuery(TypedDict, total=False): search_context_size_high: float +class SearchFreeRequests(TypedDict): + count: ReadOnly[int] + period: ReadOnly[Literal["day", "month"]] + shared_group: ReadOnly[str] + + class AgenticLoopParams(TypedDict, total=False): """ Parameters passed to agentic loop hooks (e.g., WebSearch interception). @@ -399,6 +405,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): web_search_billing_unit: ( Literal["per_query", "per_prompt"] | None ) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x) + search_free_requests: ReadOnly[SearchFreeRequests | None] google_maps_grounding_cost_per_query: ReadOnly[float | None] citation_cost_per_token: float | None # Cost per citation token for Perplexity tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope diff --git a/litellm/utils.py b/litellm/utils.py index 5f32820df5e..59a93965993 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -954,10 +954,6 @@ def _ocr_document_summary(document: object) -> str: # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. -_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset( - (CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value) -) - def function_setup( original_function: str, @@ -1207,9 +1203,6 @@ def function_setup( if isinstance(search_query, list) else search_query ) - elif call_type in _DECISION_CALL_TYPES: - decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" - messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): @@ -1372,6 +1365,7 @@ def _schedule_async_success_logging( """ def _enqueue_async_logging() -> None: + logging_obj.mark_async_success_log_enqueued() asyncio.create_task( _client_async_logging_helper( logging_obj=logging_obj, @@ -5165,6 +5159,9 @@ def get_optional_params( EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"}) +LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",)) + + def add_provider_specific_params_to_optional_params( optional_params: dict, passed_params: dict, @@ -5180,8 +5177,9 @@ def add_provider_specific_params_to_optional_params( # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) + withheld_params: Final = frozenset[str]() if custom_llm_provider == "litellm_proxy" else LITELLM_ONLY_PARAMS for k in passed_params: - if k not in openai_params and passed_params[k] is not None: + if k not in openai_params and k not in withheld_params and passed_params[k] is not None: extra_body[k] = passed_params[k] if not isinstance(optional_params.get("extra_body"), dict): optional_params["extra_body"] = {} @@ -6446,6 +6444,7 @@ def _get_model_info_helper( supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None), web_search_billing_unit=_model_info.get("web_search_billing_unit", None), + search_free_requests=_model_info.get("search_free_requests", None), google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None), tpm=_model_info.get("tpm", None), rpm=_model_info.get("rpm", None), diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 06e5b020836..9c841087b9a 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -87,6 +87,7 @@ ignored_function_names = [ "_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py "_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py "_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name) + "_deployment_params_base_model", # Tested through log_retry in test_router_helper_utils.py ] diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index d3ad1d989c8..cabee3dcfd9 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -637,7 +637,12 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) -> "model": "gpt-3.5-turbo", "api_key": "sk-must-not-be-recorded", "messages": [{"role": "user", "content": "hi"}], - metadata_key: {"model_info": {"id": "deployment-1"}, "attempted_retries": 2, "user_api_key": "sk-proxy"}, + metadata_key: { + "model_info": {"id": "deployment-1", "base_model": "azure/gpt-3.5-turbo"}, + "deployment": "azure/my-35-deployment", + "attempted_retries": 2, + "user_api_key": "sk-proxy", + }, }, e=rate_limit_error, ) @@ -645,6 +650,8 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) -> { "model_group": "gpt-3.5-turbo", "deployment_id": "deployment-1", + "model": "azure/my-35-deployment", + "base_model": "azure/gpt-3.5-turbo", "exception_type": "RateLimitError", "exception_string": "litellm.RateLimitError: slow down", "attempted_retries": 2, @@ -656,6 +663,38 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) -> assert router.log_retry(kwargs=planted_kwargs, e=rate_limit_error)[metadata_key]["request_retry_count"] == 1 +def test_log_retry_reads_base_model_from_the_deployments_litellm_params() -> None: + """An Azure named deployment declares ``base_model`` under ``litellm_params``, not ``model_info``, and + that is the model the failure row is priced at, so the retry record has to carry it too""" + router = Router( + model_list=[ + { + "model_name": "azure-group", + "litellm_params": { + "model": "azure/my-named-deployment", + "api_key": "sk-fake", + "api_base": "https://example.openai.azure.com", + "base_model": "azure/gpt-4o-mini", + }, + "model_info": {"id": "dep-azure-named"}, + } + ] + ) + new_kwargs = router.log_retry( + kwargs={ + "model": "azure-group", + "metadata": { + "model_info": {"id": "dep-azure-named"}, + "deployment": "azure/my-named-deployment", + "attempted_retries": 0, + }, + }, + e=litellm.Timeout(message="slow", llm_provider="azure", model="azure/my-named-deployment"), + ) + record = new_kwargs["metadata"]["previous_models"][0] + assert (record["model"], record["base_model"]) == ("azure/my-named-deployment", "azure/gpt-4o-mini") + + def test_update_usage(model_list): """Test if the '_update_usage' function is working correctly""" router = Router(model_list=model_list) diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 7929a0b21af..6532855dd4e 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -1210,3 +1210,39 @@ class TestCoerceNumericFormFields: numeric_fields=self.numeric_fields, ) assert result == {"n": 3, "temperature": None, "image": buffer} + + +@pytest.mark.asyncio +async def test_form_data_metadata_with_lone_surrogate_escape_is_rejected_like_a_json_body(): + """``json.loads`` accepts ``"\\ud800"`` and yields a lone surrogate that no + provider or database can utf-8 encode. The JSON body path already answers + 400, so the multipart ``metadata`` field must too, instead of letting the + value through to the spend log writers.""" + mock_request = MagicMock() + mock_request.form = AsyncMock( + return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["poison-\\ud800-tag"]}'}) + ) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + mock_request.state._cached_headers = None + + with pytest.raises(ProxyException) as exc_info: + await _read_request_body(mock_request) + + assert exc_info.value.code == "400" + assert "metadata" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_form_data_metadata_keeps_non_ascii_and_valid_surrogate_pairs(): + mock_request = MagicMock() + mock_request.form = AsyncMock( + return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["caf\\u00e9", "\\ud83d\\ude00"]}'}) + ) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + mock_request.state._cached_headers = None + + result = await _read_request_body(mock_request) + + assert result["metadata"]["tags"] == ["café", "😀"] diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 7abb6e1ef92..a369d1c15da 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -3843,28 +3843,57 @@ async def test_commit_spend_updates_retries_deadlock_on_every_entity_path(monkey proxy_logging.failure_handler.assert_not_called() +def _responses_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "native", + "litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test"}, + "model_info": {"id": "dep-native"}, + }, + { + "model_name": "bridged", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"}, + "model_info": {"id": "dep-bridged"}, + }, + { + "model_name": "forced-bridge", + "litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test", "use_chat_completions_api": True}, + "model_info": {"id": "dep-forced-bridge"}, + }, + ] + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( - "call_type, expects_flush", - [("aresponses", True), ("responses", True), ("acompletion", False)], + "call_type, model_id, expects_flush", + [ + ("aresponses", "dep-bridged", True), + ("responses", "dep-bridged", True), + ("aresponses", "dep-forced-bridge", True), + ("aresponses", None, True), + ("aresponses", "dep-native", False), + ("responses", "dep-native", False), + ("acompletion", "dep-bridged", False), + ], ) -async def test_insert_spend_log_asks_for_an_immediate_flush_on_rows_other_workers_read_back( - call_type: str, expects_flush: bool +async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bridge_reads_back( + call_type: str, model_id: str | None, expects_flush: bool ): """ - A `previous_response_id` chained straight off the previous turn reads the DB, so a - Responses row cannot sit in this worker's queue until the monitor's next poll. + A `previous_response_id` chained straight off a bridged turn reads that turn's row from the + DB, so it cannot sit in this worker's queue until the monitor's next poll. A native Responses + provider holds the conversation itself, so its row waits for the batch write like a chat row. """ db_writer = DBSpendUpdateWriter() prisma = _tool_usage_prisma() prisma.spend_log_flush_requested = asyncio.Event() + payload = {"request_id": "req-1", "call_type": call_type, "model_id": model_id} - await db_writer._insert_spend_log_to_db( - payload={"request_id": "req-1", "call_type": call_type}, - prisma_client=prisma, - ) + await db_writer._insert_spend_log_to_db(payload=payload, prisma_client=prisma, llm_router=_responses_router()) - assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}] + assert prisma.spend_log_transactions == [payload] assert prisma.spend_log_flush_requested.is_set() is expects_flush diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 68f0b78172c..1c68f1adf92 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -9,6 +9,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import litellm + from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth @@ -28,7 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpend from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload -from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage +from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage @pytest.mark.asyncio @@ -94,6 +96,65 @@ async def test_async_post_call_failure_hook(): assert metadata["original_key"] == "original_value" +@pytest.mark.asyncio +async def test_each_earlier_billed_attempt_gets_its_own_spend_row(): + """A timed-out request that the router retried writes one row per billed attempt: the final + attempt under the request id, every earlier Timeout attempt under ``-attempt-`` with its + own cost, attempt number and deployment.""" + from litellm.proxy.utils import BilledFailedAttempt + + logger = _ProxyDBLogger() + user_api_key_dict = UserAPIKeyAuth(api_key="test_api_key", user_id="u1", team_id="t1") + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + request_data = { + "model": "gpt-4.1", + "litellm_call_id": "call-1", + "standard_logging_object": {"id": "call-1", "model_id": "dep-final"}, + "metadata": { + "attempted_retries": 2, + "model_info": {"id": "dep-final", "input_cost_per_token": 0.5}, + "previous_models": ({"exception_type": "Timeout"}, {"exception_type": "Timeout"}), + }, + "combined_usage_object": usage, + "response_cost": 0.5, + "billed_failed_attempts": ( + BilledFailedAttempt(attempted_retries=0, deployment_id="dep-a", usage=usage, response_cost=0.25), + BilledFailedAttempt(attempted_retries=0, deployment_id="dep-final", usage=usage, response_cost=0.5), + ), + } + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await logger.async_post_call_failure_hook( + request_data=request_data, + original_exception=litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai"), + user_api_key_dict=user_api_key_dict, + ) + + rows = [call.kwargs for call in mock_update_database.call_args_list] + assert [row["response_cost"] for row in rows] == [0.5, 0.25, 0.5] + final_row, attempt_0, attempt_1 = (row["kwargs"] for row in rows) + assert final_row["litellm_call_id"] == "call-1" + assert final_row["litellm_params"]["metadata"]["attempted_retries"] == 2 + assert attempt_0["litellm_call_id"] == "call-1-attempt-0" + assert attempt_0["standard_logging_object"]["id"] == "call-1-attempt-0" + assert attempt_0["litellm_params"]["metadata"]["attempted_retries"] == 0 + assert attempt_0["litellm_params"]["metadata"]["model_info"]["id"] == "dep-a" + assert "previous_models" not in attempt_0["litellm_params"]["metadata"] + assert "billed_failed_attempts" not in attempt_0 + assert attempt_0["combined_usage_object"] is usage + assert attempt_1["litellm_call_id"] == "call-1-attempt-1" + assert attempt_1["litellm_params"]["metadata"]["attempted_retries"] == 0 + assert attempt_1["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final" + assert final_row["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final" + for row in rows: + assert row["token"] == "test_api_key" + assert row["user_id"] == "u1" + assert row["team_id"] == "t1" + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata(): """ @@ -2785,3 +2846,93 @@ async def test_batch_accounting_does_not_retry_after_settling_a_budget_reservati ) assert reservation["finalized"] is True assert outcome.retryable_failure is False + + +class _SharedCounter: + def __init__(self) -> None: + self.totals: dict[str, float] = {} + + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: + self.totals[key] = self.totals.get(key, 0) + value + return self.totals[key] + + +def _grounded_gemini_kwargs(response_cost: float) -> dict: + return { + "model": "vertex_ai/gemini-test", + "call_type": "acompletion", + "response_cost": response_cost, + "litellm_params": {"metadata": {"user_api_key": "sk-test", "user_api_key_user_id": "user-1"}}, + "standard_logging_object": { + "response_cost": response_cost, + "request_tags": None, + "cache_hit": None, + "custom_llm_provider": "vertex_ai", + "cost_breakdown": { + "input_cost": 0.0, + "output_cost": 0.0, + "tool_usage_cost": 0.5, + "total_cost": response_cost, + }, + "model_map_information": { + "model_map_key": "vertex_ai/gemini-test", + "model_map_value": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "web_search_billing_unit": "per_query", + "search_context_cost_per_query": {"search_context_size_medium": 0.1}, + "search_free_requests": {"count": 100, "period": "month", "shared_group": "test"}, + }, + }, + }, + "stream": False, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("opted_in", [True, False]) +async def test_track_cost_callback_frees_grounding_under_the_allowance_only_when_opted_in(monkeypatch, opted_in): + monkeypatch.setattr(litellm, "apply_search_free_tier", opted_in, raising=False) + counter = _SharedCounter() + kwargs = _grounded_gemini_kwargs(response_cost=1.0) + response = ModelResponse( + usage=Usage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=5), + ) + ) + + with ( + patch( # test-quality-ok: increment_spend_counters is a proxy_server global the callback reads lazily, no seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ), + patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam + "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock + ), + patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + ): + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True) + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + mock_proxy_logging.internal_usage_cache.dual_cache.redis_cache = counter + + await _ProxyDBLogger()._PROXY_track_cost_callback( + kwargs=kwargs, completion_response=response, start_time=datetime.now(), end_time=datetime.now() + ) + await asyncio.sleep(0) + + mock_proxy_logging.failed_tracking_alert.assert_not_called() + mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once() + + breakdown = kwargs["standard_logging_object"]["cost_breakdown"] + if opted_in: + assert kwargs["response_cost"] == pytest.approx(1.0 - 5 * 0.1) + assert breakdown["original_cost"] == 1.0 and breakdown["discount_amount"] == pytest.approx(5 * 0.1) + assert sum(counter.totals.values()) == 5 + else: + assert kwargs["response_cost"] == 1.0 and "discount_amount" not in breakdown + assert counter.totals == {} diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index aa269a10867..14f32183d64 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1249,6 +1249,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value( assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected +def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier(): + litellm_params = { + "timeout": 610, + "stream_timeout": 30, + "timeout_by_service_tier": {"flex": 900}, + "stream_timeout_by_service_tier": {"flex": 950}, + } + assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0 + assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0 + assert ( + resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params) + == 20.0 + ) + assert ( + resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params) + == 950.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"service_tier": "flex", "stream": True}, + litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}}, + ) + == 30.0 + ) + + def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner(): with pytest.raises(ValidationError): resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)}) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 7b74e69685c..03c36932154 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -7637,6 +7637,118 @@ def _apply_stream_usage_tracking( ) + async def _start_partial_responses_stream(self, recorder: _RecordingSuccessLogger, *, relay_text: bool = True): + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mocked: Final = await litellm.aresponses( + model="gpt-4o-mini", + input="tell me a story", + mock_response="The codename is AZURE-FALCON-42 and the story is long.", + stream=True, + api_key="test-key", + ) + logging_obj: Final = mocked.logging_obj + logging_obj.dynamic_async_success_callbacks = [recorder] + stream: Final = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="gpt-4o-mini", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + logging_obj=logging_obj, + request_data={"model": "gpt-4o-mini", "input": "tell me a story"}, + custom_llm_provider="openai", + ) + created: Final = { + "type": "response.created", + "sequence_number": 0, + "response": { + "id": "resp_partial", + "object": "response", + "created_at": 1700000000.0, + "model": "gpt-4o-mini", + "status": "in_progress", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "service_tier": "priority", + }, + } + delta: Final = { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_partial", + "output_index": 0, + "content_index": 0, + "delta": "The codename is AZURE-FALCON-42", + } + for event in (created, delta) if relay_text else (created,): + stream._process_chunk(json.dumps(event)) + return stream + + async def _bill_and_wait(self, recorder: _RecordingSuccessLogger, streamed: object, logging_obj: object) -> bool: + billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed) + for _ in range(50): + if recorder.success_events: + break + await asyncio.sleep(0.1) + await asyncio.sleep(0.5) + return billed + + @pytest.mark.asyncio + async def test_disconnect_bills_partial_native_responses_stream_with_counted_usage(self): + from litellm.types.llms.openai import ResponsesAPIResponse + + recorder: Final = _RecordingSuccessLogger() + stream: Final = await self._start_partial_responses_stream(recorder) + billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert isinstance(partial_response, ResponsesAPIResponse) + assert partial_response.status == "incomplete" + assert getattr(partial_response, "service_tier") == "priority" + assert partial_response.usage.output_tokens == litellm.token_counter( + model="gpt-4o-mini", text="The codename is AZURE-FALCON-42" + ) + assert partial_response.usage.input_tokens > 0 + + @pytest.mark.asyncio + async def test_disconnect_bills_partial_default_route_messages_stream_through_the_responses_adapter(self): + from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( + AnthropicResponsesStreamWrapper, + ) + from litellm.router import FallbackAwareAnthropicMessagesStream + from litellm.types.llms.openai import ResponsesAPIResponse + + async def _sse_frames() -> AsyncGenerator[bytes, None]: + yield b"event: message_start\n\n" + + recorder: Final = _RecordingSuccessLogger() + stream: Final = await self._start_partial_responses_stream(recorder) + wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=stream, model="gpt-4o-mini") + streamed: Final = FallbackAwareAnthropicMessagesStream(_sse_frames(), wrapper.async_anthropic_sse_wrapper()) + billed: Final = await self._bill_and_wait(recorder, streamed, stream.logging_obj) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert isinstance(partial_response, ResponsesAPIResponse) + assert partial_response.usage.output_tokens == litellm.token_counter( + model="gpt-4o-mini", text="The codename is AZURE-FALCON-42" + ) + + @pytest.mark.asyncio + async def test_disconnect_before_any_responses_output_bills_nothing(self): + recorder: Final = _RecordingSuccessLogger() + stream: Final = await self._start_partial_responses_stream(recorder, relay_text=False) + billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj) + + assert billed is False + assert recorder.success_events == [] + + class TestApplyStreamUsageTracking: def test_default_injects_usage_and_marks_strip_for_chat_completions(self): data = {"stream": True, "model": "gpt-5.4-nano"} diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index ee42042bb8e..457409bdf6a 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -7764,6 +7764,7 @@ _PLANTED_STAMPS = { "attempted_fallbacks": 99, "original_model_group": "spoofed-group", "request_retry_count": -100, + "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, "client_key": "client_value", @@ -7798,6 +7799,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] + assert "previous_models" not in updated["metadata"] assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"] assert updated["metadata"]["client_key"] == "client_value" diff --git a/tests/test_litellm/proxy/test_pricing_field_strip.py b/tests/test_litellm/proxy/test_pricing_field_strip.py index a84c6ba2b8a..e2eaf714f7d 100644 --- a/tests/test_litellm/proxy/test_pricing_field_strip.py +++ b/tests/test_litellm/proxy/test_pricing_field_strip.py @@ -58,9 +58,7 @@ class TestStripClientPricingOverrides: # The strip set is built from the model so additions are picked up # automatically — this test guards against the model and the strip # set drifting apart if someone replaces the auto-derivation later. - assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset( - CustomPricingLiteLLMParams.model_fields.keys() - ) + assert _CLIENT_PRICING_CONTROL_FIELDS >= frozenset(CustomPricingLiteLLMParams.model_fields.keys()) # Sanity: the obvious top-level pricing fields are in the set. for field in ( "input_cost_per_token", @@ -84,6 +82,22 @@ class TestStripClientPricingOverrides: "messages": [{"role": "user", "content": "hi"}], } + def test_root_base_model_dropped(self): + # ``base_model`` picks the cost-map entry a spend row is priced with + # (success rows via the standard logging payload, billed failure rows + # via the failure lift), so a caller naming a cheaper model lowers + # their own recorded spend. + data = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "base_model": "gpt-4o-mini", + } + _strip_client_pricing_overrides(data) + assert data == { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + } + def test_metadata_model_info_dropped(self): data = { "model": "gpt-4", @@ -264,6 +278,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in(): "model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "input_cost_per_token": 0.0001, + "base_model": "gpt-4o-mini", "metadata": {"model_info": {"output_cost_per_token": 0.0002}}, } @@ -279,6 +294,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in(): assert updated["input_cost_per_token"] == 0.0001 assert updated["metadata"]["model_info"] == {"output_cost_per_token": 0.0002} + assert updated["base_model"] == "gpt-4o-mini" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index ea1870d3b73..5f9542cd3f8 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -2,6 +2,7 @@ import datetime as real_datetime import smtplib from typing import Final +import httpx import pytest from fastapi import HTTPException @@ -443,6 +444,47 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime: assert "litellm_logging_obj" not in request_data +class TestPostCallFailureHookDoesNotInventAProviderHandoff: + """A failure the proxy raised before any provider call (here an auth error) is logged through + _handle_logging_proxy_only_error, which fires pre_call for the callbacks. That must not stamp a + handoff instant onto the request, or every downstream reader (preprocessing latency, the + dispatched check behind the failure-row estimate) sees a provider call that never happened.""" + + @pytest.mark.asyncio + async def test_proxy_only_failure_lifts_no_first_api_call_start_time(self): + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth + + logging_obj = Logging( + model="gpt-4.1", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="pre-handoff-auth-failure", + function_id="pre-handoff", + ) + request_data: Final[dict[str, object]] = {"litellm_logging_obj": logging_obj, "model": "gpt-4.1", "metadata": {}} + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with ( + patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()), + patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()), + ): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Authentication Error, invalid key"), + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + error_type=ProxyErrorTypes.auth_error, + ) + + assert request_data.get("first_api_call_start_time") is None + assert "api_call_start_time" not in logging_obj.model_call_details + + class TestPostCallFailureHookLiftsRecoveredPartialSpend: """A stream that broke mid-flight still billed the provider for the chunks already delivered. The streaming handler stashes that recovered usage and @@ -687,11 +729,13 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: """A non-stream request that failed after dispatch (timeout, provider error) consumed provider-billed input tokens but recovered no usage. post_call_failure_hook must estimate the input side onto request_data so - the spend log's failure row records what was sent instead of zero, while - never charging spend for the failure (LIT-5690). + the spend log's failure row records what was sent instead of zero + (LIT-5690). On a timeout that estimate is priced at the model's input + rate since the provider bills the prompt it accepted (LIT-145); other + post-dispatch failures keep zero spend. """ - async def _run(self, request_data): + async def _run(self, request_data, original_exception: Exception | None = None): from unittest.mock import AsyncMock, patch from litellm.proxy._types import UserAPIKeyAuth @@ -701,17 +745,275 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data=request_data, - original_exception=Exception("boom"), + original_exception=original_exception or Exception("boom"), user_api_key_dict=UserAPIKeyAuth(), ) + def _timeout(self, model: str, cause: Exception) -> Exception: + try: + try: + raise cause + except Exception as inner: + raise litellm.Timeout("Request timed out.", model=model, llm_provider="openai") from inner + except litellm.Timeout as timed_out: + return timed_out + def _logging_obj(self, model_call_details): logging_obj = MagicMock() logging_obj.model_call_details = model_call_details return logging_obj @pytest.mark.asyncio - async def test_dispatched_failure_estimates_input_tokens_with_zero_cost(self): + async def test_timed_out_request_estimates_input_tokens_priced_at_input_rate(self): + from datetime import datetime + + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + "response_cost": 123.0, + } + await self._run( + request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ReadTimeout("no bytes in time")) + ) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + assert estimated.prompt_tokens > 0 + assert estimated.completion_tokens == 0 + assert estimated.total_tokens == estimated.prompt_tokens + expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens) + assert expected_input_cost > 0 + assert request_data["response_cost"] == pytest.approx(expected_input_cost) + + @pytest.mark.asyncio + async def test_client_disconnect_after_dispatch_is_priced_at_input_rate(self): + from datetime import datetime + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True, + } + ), + "metadata": {}, + } + await self._run( + request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + ) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens) + assert expected_input_cost > 0 + assert request_data["response_cost"] == pytest.approx(expected_input_cost) + + @pytest.mark.asyncio + async def test_upstream_499_error_is_a_plain_failure_not_a_priced_disconnect(self): + """A provider answering HTTP 499 surfaces as litellm.APIError with status_code 499. Only the + proxy's own disconnect HTTPException means the caller hung up mid-generation, so the upstream + error keeps the estimated usage at zero cost like any other post-dispatch provider error.""" + from datetime import datetime + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 0), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + } + await self._run( + request_data, + original_exception=litellm.APIError( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, + message="client closed request", + llm_provider="openai", + model="gpt-3.5-turbo", + ), + ) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + assert estimated.prompt_tokens > 0 + assert request_data["response_cost"] == 0.0 + + @pytest.mark.asyncio + async def test_client_disconnect_after_the_call_already_completed_is_not_priced_again(self): + """The LLM task finished and claimed its success log (which carries the real usage and cost) + while a during_call guardrail task was still pending, then the caller hung up. The 499 raised + for that disconnect must not price the prompt a second time.""" + from datetime import datetime + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "count these input tokens please"}], + stream=False, + call_type="acompletion", + start_time=datetime(2026, 1, 1, 12, 0, 0), + litellm_call_id="completed-before-disconnect", + function_id="fn", + ) + logging_obj.model_call_details.update( + { + "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ) + logging_obj.mark_async_success_log_enqueued() + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run( + request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + ) + + assert request_data.get("response_cost", 0.0) == 0.0 + + @pytest.mark.asyncio + async def test_client_disconnect_with_a_claimed_but_deferred_success_log_is_still_priced(self): + """With a post_call guardrail registered the success log is claimed but parked in the deferred slot, + and a 499 raised before the flush means it never runs. Nothing else bills that request, so the + failure row keeps the estimated input cost.""" + from datetime import datetime + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "count these input tokens please"}], + stream=False, + call_type="acompletion", + start_time=datetime(2026, 1, 1, 12, 0, 0), + litellm_call_id="deferred-success-never-flushed", + function_id="fn", + ) + logging_obj.model_call_details.update( + { + "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ) + logging_obj._defer_async_logging = True + logging_obj._enqueue_deferred_logging = lambda: None + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run( + request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + ) + + assert request_data["response_cost"] > 0.0 + + @pytest.mark.asyncio + async def test_client_disconnect_before_provider_handoff_is_not_priced(self): + """A caller that hung up while the request was still in pre-call hooks never reached the + provider, so there is no handoff instant on the logging object and the 499 row is not priced.""" + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import UserAPIKeyAuth + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "count these input tokens please"}], + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="lit145-pre-handoff", + function_id="lit145", + ) + assert logging_obj.model_call_details.get("first_api_call_start_time") is None + request_data = {"litellm_logging_obj": logging_obj, "model": "gpt-3.5-turbo", "metadata": {}} + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with ( + patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()), + patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()), + ): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + ) + + assert request_data.get("first_api_call_start_time") is None + assert "combined_usage_object" not in request_data + assert request_data.get("response_cost", 0.0) == 0.0 + + @pytest.mark.asyncio + async def test_connect_timeout_never_reached_the_provider_stays_zero_cost(self): + from datetime import datetime + + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + } + await self._run( + request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ConnectTimeout("no route")) + ) + + assert isinstance(request_data["combined_usage_object"], Usage) + assert request_data["response_cost"] == 0.0 + + @pytest.mark.asyncio + async def test_dispatched_non_timeout_failure_estimates_input_tokens_with_zero_cost(self): from datetime import datetime from litellm.types.utils import Usage @@ -733,8 +1035,6 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: estimated = request_data["combined_usage_object"] assert isinstance(estimated, Usage) assert estimated.prompt_tokens > 0 - assert estimated.completion_tokens == 0 - assert estimated.total_tokens == estimated.prompt_tokens assert request_data["response_cost"] == 0.0 @pytest.mark.asyncio @@ -1901,7 +2201,355 @@ def test_usage_the_stream_already_recovered_beats_an_estimate(): assert lifted == (recovered, 0.25) -def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost(): +def test_a_timed_out_request_on_an_unmapped_model_keeps_its_row_at_zero_cost(): + from litellm.proxy.utils import _failure_usage_to_lift + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": "my-unmapped-model", + "custom_llm_provider": "openai", + "messages": ONE_USER_MESSAGE, + }, + request_body={}, + dispatched=True, + provider_billed=True, + ) + + assert lifted is not None + usage, response_cost = lifted + assert usage.prompt_tokens > 0 + assert response_cost == 0.0 + + +def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the_deployment(monkeypatch): + """Router._create_deployment registers a deployment's own ``input_cost_per_token`` under its + ``model_info.id``; the failure row prices with that entry through the same + ``deployment_pricing_model_info`` the success path uses.""" + from litellm.proxy.utils import _failure_usage_to_lift + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0}, + ) + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + }, + request_body={}, + dispatched=True, + provider_billed=True, + deployment_id="dep-lit145", + ) + + assert lifted is not None + usage, response_cost = lifted + assert usage.prompt_tokens > 0 + assert response_cost == pytest.approx(usage.prompt_tokens * 0.25) + + +@pytest.mark.parametrize(("prompt_tokens", "expected_rate"), [(1_000, 1e-7), (250_000, 5e-7)]) +def test_a_timed_out_request_is_priced_at_the_deployments_long_context_tier(monkeypatch, prompt_tokens, expected_rate): + """A deployment that declares ``input_cost_per_token_above_200k_tokens`` next to its standard rate + bills a 250k-token prompt at the tier on the success path; the failure row must not flatten it to + the standard rate.""" + from litellm.proxy.utils import _estimated_input_cost + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit-f9-tiered", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-7, + "input_cost_per_token_above_200k_tokens": 5e-7, + "output_cost_per_token": 4e-7, + }, + ) + + response_cost = _estimated_input_cost( + model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"}, + prompt_tokens=prompt_tokens, + deployment_id="dep-lit-f9-tiered", + base_model=None, + ) + + assert response_cost == pytest.approx(prompt_tokens * expected_rate) + + +def test_every_earlier_timed_out_attempt_is_billed_at_its_own_deployment_rate(monkeypatch): + """The router leaves one RetryAttemptRecord per failed attempt in metadata.previous_models, the + final attempt included; the earlier Timeout ones are billed (each at the deployment that served + it), a 429 attempt is not, and the final attempt is left to the failure row itself.""" + from litellm.proxy.utils import _billed_earlier_attempts + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145-b", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0}, + ) + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + public_rate, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=24) + + attempts = _billed_earlier_attempts( + model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"}, + litellm_params={ + "metadata": { + "attempted_retries": 3, + "previous_models": ( + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0}, + {"deployment_id": "dep-lit145-b", "exception_type": "RateLimitError", "attempted_retries": 1}, + {"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 2}, + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 3}, + ), + } + }, + estimated_usage=usage, + base_model=None, + deployment_id="dep-lit145-a", + ) + + assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [(0, "dep-lit145-a"), (2, "dep-lit145-b")] + assert attempts[0].response_cost == pytest.approx(public_rate) + assert attempts[0].response_cost > 0 + assert attempts[1].response_cost == pytest.approx(24 * 0.25) + assert all(a.usage is usage for a in attempts) + + +def test_an_earlier_attempt_on_another_model_is_priced_at_that_models_rate_not_the_final_models(monkeypatch): + """After a fallback hop the final attempt runs on another model group, so the records of the group + that timed out first name their own deployment model (and its ``base_model`` when the deployment + name is not in the cost map). Each is billed at that model's input rate, not at the final model's.""" + from litellm.proxy.utils import _billed_earlier_attempts + + for pricing_model, provider, input_rate in ( + ("lit145-final-model", "openai", 0.5), + ("lit145-first-model", "openai", 0.02), + ("azure/lit145-first-model", "azure", 0.02), + ): + monkeypatch.setitem( + litellm.model_cost, + pricing_model, + {"litellm_provider": provider, "mode": "chat", "input_cost_per_token": input_rate, "output_cost_per_token": 1.0}, + ) + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + + attempts = _billed_earlier_attempts( + model_call_details={"model": "openai/lit145-final-model", "custom_llm_provider": "openai"}, + litellm_params={ + "metadata": { + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-final"}, + "previous_models": ( + { + "deployment_id": "dep-lit145-first", + "model": "openai/lit145-first-model", + "base_model": None, + "exception_type": "Timeout", + "attempted_retries": 0, + }, + { + "deployment_id": "dep-lit145-first-azure", + "model": "azure/lit145-first-deployment", + "base_model": "azure/lit145-first-model", + "exception_type": "Timeout", + "attempted_retries": 1, + }, + { + "deployment_id": "dep-lit145-final", + "model": "openai/lit145-final-model", + "base_model": None, + "exception_type": "Timeout", + "attempted_retries": 0, + }, + { + "deployment_id": "dep-lit145-final", + "model": "openai/lit145-final-model", + "base_model": None, + "exception_type": "Timeout", + "attempted_retries": 1, + }, + ), + } + }, + estimated_usage=usage, + base_model=None, + deployment_id="dep-lit145-final", + ) + + assert [(a.attempted_retries, a.deployment_id, a.response_cost) for a in attempts] == [ + (0, "dep-lit145-first", pytest.approx(24 * 0.02)), + (1, "dep-lit145-first-azure", pytest.approx(24 * 0.02)), + (0, "dep-lit145-final", pytest.approx(24 * 0.5)), + ] + + +def test_earlier_attempts_are_priced_from_the_prompt_estimate_not_the_final_streams_recovered_usage(): + """A final attempt whose stream broke mid-flight lifts its recovered usage (prompt plus the chunks + delivered); the earlier timed-out attempts never produced output, so they are priced from the + input-only estimate and not from that recovered usage.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _estimated_prompt_usage, _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + FAILURE_USAGE_MODEL, + get_litellm_params( + metadata={ + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-final"}, + "previous_models": ( + {"deployment_id": "dep-lit145-final", "exception_type": "Timeout", "attempted_retries": 0}, + ), + } + ), + ) + recovered = litellm.Usage(prompt_tokens=500, completion_tokens=40, total_tokens=540) + logging_obj.model_call_details["combined_usage_object"] = recovered + logging_obj.model_call_details["response_cost"] = 0.01 + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + estimate = _estimated_prompt_usage(logging_obj.model_call_details, {}) + assert lifted["combined_usage_object"] is recovered + assert [(a.usage.prompt_tokens, a.usage.completion_tokens) for a in lifted["billed_failed_attempts"]] == [ + (estimate.prompt_tokens, 0) + ] + assert estimate.prompt_tokens < recovered.prompt_tokens + + +def test_a_timed_out_attempt_from_the_group_a_fallback_left_is_still_billed(): + """After a fallback hop the router restarts ``attempted_retries`` at 0 for the new model group and + re-logs the attempt that exhausted the previous group, so the final attempt (dep-b, retry 1) must be + told from the previous group's attempt with the same count (dep-a, retry 1), and the re-logged + record must bill once.""" + from litellm.proxy.utils import _billed_earlier_attempts + + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + + attempts = _billed_earlier_attempts( + model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"}, + litellm_params={ + "metadata": { + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-b"}, + "previous_models": ( + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0}, + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1}, + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1}, + {"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 0}, + {"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 1}, + ), + } + }, + estimated_usage=usage, + base_model=None, + deployment_id="dep-lit145-b", + ) + + assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [ + (0, "dep-lit145-a"), + (1, "dep-lit145-a"), + (0, "dep-lit145-b"), + ] + assert all(a.response_cost > 0 for a in attempts) + + +def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object: + """A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params() + shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff + instant, so the lift treats the request as dispatched.""" + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model=model, + messages=ONE_USER_MESSAGE, + stream=False, + call_type="acompletion", + start_time=real_datetime.datetime.now(), + litellm_call_id="call-lit145", + function_id="fn-lit145", + ) + logging_obj.update_environment_variables(model=model, user="", optional_params={}, litellm_params=litellm_params) + logging_obj.model_call_details["first_api_call_start_time"] = real_datetime.datetime.now() + logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE + logging_obj.model_call_details["call_type"] = "acompletion" + return logging_obj + + +def test_the_deployment_rate_is_found_on_the_logging_objects_real_litellm_params(monkeypatch): + """``get_litellm_params()`` always carries a ``litellm_metadata`` key (None on chat + completions) next to the router-stamped ``metadata.model_info.id``; the failure row must + still price at the deployment's own rate, like ``Logging.get_router_model_id`` on the + success path.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145-real-shape", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 0.25}, + ) + litellm_params = get_litellm_params(metadata={"model_info": {"id": "dep-lit145-real-shape"}}) + assert "litellm_metadata" in litellm_params and litellm_params["litellm_metadata"] is None + logging_obj = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, litellm_params) + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + assert lifted["response_cost"] == pytest.approx(lifted["combined_usage_object"].prompt_tokens * 0.25) + + +def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path(): + """An Azure deployment name is not in the cost map; the success path prices it through + ``model_info.base_model`` (``get_base_model_from_metadata``) and so must the failure row.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + "azure/lit145-not-in-cost-map", + get_litellm_params(metadata={"model_info": {"base_model": "azure/gpt-4.1"}}), + ) + logging_obj.model_call_details["custom_llm_provider"] = "azure" + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + expected_cost, _ = litellm.cost_per_token( + model="azure/gpt-4.1", prompt_tokens=lifted["combined_usage_object"].prompt_tokens + ) + assert expected_cost > 0 + assert lifted["response_cost"] == pytest.approx(expected_cost) + + +def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price(): + """``model_call_details`` is flattened with the request's optional_params, so a body key + named ``litellm_params`` can land there; the deployment rate must come from the logging + object's own ``litellm_params`` instead.""" + from litellm.proxy.utils import _failure_usage_to_lift + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + "litellm_params": {"input_cost_per_token": 0}, + }, + request_body={}, + dispatched=True, + provider_billed=True, + deployment_id=None, + ) + + assert lifted is not None + usage, response_cost = lifted + expected_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens) + assert response_cost == pytest.approx(expected_cost) + assert response_cost > 0 + + +def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rate(): from litellm.proxy.utils import _failure_usage_to_lift lifted = _failure_usage_to_lift( @@ -1912,13 +2560,16 @@ def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost(): }, request_body={}, dispatched=True, + provider_billed=True, ) assert lifted is not None usage, response_cost = lifted assert usage.prompt_tokens > 0 assert usage.completion_tokens == 0 - assert response_cost == 0.0 + expected_input_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens) + assert expected_input_cost > 0 + assert response_cost == pytest.approx(expected_input_cost) @pytest.mark.parametrize( diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index 7099101db1c..eab52af51fb 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -917,3 +917,53 @@ async def test_update_spend_logs_parks_failed_batch_in_redis_with_wire_safe_date parked = await buffer.get_spend_logs_from_redis_buffer(limit=10) assert mock_prisma_client.spend_log_transactions == [] assert [(row["request_id"], row["startTime"]) for row in parked] == [("a", started.isoformat())] + + +async def _create_many_while_postgres_is_restarting(**_: object) -> int: + from prisma.engine.utils import handle_response_errors + + handle_response_errors( + MagicMock(), + [ + { + "error": "Error in connector: Can't reach database server at `127.0.0.1:5432`", + "user_facing_error": {"error_code": "P1001", "message": "Can't reach database server", "meta": None}, + } + ], + ) + return 0 + + +@pytest.mark.asyncio +async def test_update_spend_logs_retries_and_requeues_batch_when_postgres_restart_tears_down_prisma_engine( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """While Postgres restarts the prisma engine answers the write with a P1001 + payload whose ``meta`` is ``None``, which prisma's own response parser trips + over before it can build a typed error. That outage must be retried like + any other, and once retries exhaust the dequeued batch goes back to the head + of the queue so the rows land when Postgres is back. + """ + + async def _fake_sleep(_: float) -> None: + return None + + monkeypatch.setattr(utils_mod.asyncio, "sleep", _fake_sleep) + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( + side_effect=_create_many_while_postgres_is_restarting + ) + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="c")] + + with pytest.raises(AttributeError): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=2, + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + logs_to_process=[make_spend_log_row(request_id="a"), make_spend_log_row(request_id="b")], + ) + + assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["a", "b", "c"] + assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 3 diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 5b43392b3ed..ba2efb84cf7 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -641,7 +641,7 @@ async def test_calculate_vertex_disable_transform_path(monkeypatch): monkeypatch.setattr( bu, "calculate_vertex_ai_batch_cost_and_usage", - lambda content, model, model_info=None: bu.BatchCostUsageResult( + lambda content, model, model_info=None, vertex_location=None: bu.BatchCostUsageResult( cost=9.9, usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3), models=["gemini-2.0-flash-001"], @@ -672,7 +672,7 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch): monkeypatch.setattr( bu, "calculate_vertex_ai_batch_cost_and_usage", - lambda content, model, model_info=None: pytest.fail("raw vertex path should not run"), + lambda content, model, model_info=None, vertex_location=None: pytest.fail("raw vertex path should not run"), ) result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=[], custom_llm_provider="vertex_ai") @@ -1113,6 +1113,36 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk assert result.failed_requests == 0 +@pytest.mark.asyncio +async def test_handle_completed_vertex_batch_regional_location_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.files.main as files_main + + rows = [_vertex_openai_row("request-1", "gemini-3.6-flash", 10, 5)] + + async def fake_afile_content(**kw): + return type("R", (), {"content": _vertex_jsonl(rows)})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + + global_result = await bu._handle_completed_batch( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={"vertex_project": "proj-1", "vertex_location": "global"}, + vertex_location="global", + ) + regional_result = await bu._handle_completed_batch( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"}, + vertex_location="us-central1", + ) + + uplift = litellm.model_cost["vertex_ai/gemini-3.6-flash"]["regional_endpoint_uplift_multiplier"] + assert global_result.cost > 0 + assert regional_result.cost == pytest.approx(global_result.cost * uplift, rel=1e-9) + assert regional_result.usage == global_result.usage + + @pytest.mark.asyncio async def test_output_file_content_no_output_file_id_raises(): with pytest.raises(ValueError, match="Output file id is None"): @@ -1342,7 +1372,7 @@ async def test_handle_completed_batch_vertex_disable_transform_path(monkeypatch) monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", True, raising=False) seen: dict = {} - def fake_vertex_calc(content, model, model_info=None): + def fake_vertex_calc(content, model, model_info=None, vertex_location=None): seen["content"] = content seen["model"] = model return bu.BatchCostUsageResult( diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py new file mode 100644 index 00000000000..138d9dba906 --- /dev/null +++ b/tests/unit/decisions/test_call.py @@ -0,0 +1,120 @@ +import base64 +import json +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest + +import litellm +from litellm.decisions.main import _prepare_decisions_call +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.systemone.main import _prepare_systemone_call + +QUESTION_TEXT: Final = "Is this about a billing problem?" +CONNECTION: Final = { + "model": "hosted_vllm/jev-fake", + "safety_identifier": None, + "api_key": "fake-provider-key", + "api_base": "http://127.0.0.1:4010", + "timeout": None, + "custom_llm_provider": None, + "extra_headers": None, +} + + +def _logging_obj(call_type: str) -> Logging: + return Logging( + model="hosted_vllm/jev-fake", + messages="placeholder", + stream=False, + call_type=call_type, + start_time=datetime.now(timezone.utc), + litellm_call_id="lit-9659", + function_id="lit-9659", + ) + + +def _logged_request(logging_obj: Logging) -> Mapping[str, object]: + messages: Final = logging_obj.model_call_details["messages"] + assert len(messages) == 1 and messages[0]["role"] == "user" + return json.loads(messages[0]["content"]) + + +def test_systemone_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("asystemone") + + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["state"] == {"message": "My card was charged twice."} + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + assert set(logged) == {"state", "questions"} + + +def test_decisions_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("adecisions") + + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["input"] == "My card was charged twice." + assert logged["questions"][0]["instructions"] == QUESTION_TEXT + assert set(logged) == {"input", "questions"} + + +def test_request_is_logged_even_when_it_is_rejected_before_the_provider_call(): + logging_obj: Final = _logging_obj("asystemone") + + with pytest.raises(litellm.BadRequestError): + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + kwargs={"litellm_logging_obj": logging_obj, "input": "sent to the wrong function"}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + + +def test_decisions_request_is_logged_even_when_its_questions_fail_validation(): + logging_obj: Final = _logging_obj("adecisions") + + with pytest.raises(litellm.BadRequestError): + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "not-a-question-type", "instructions": QUESTION_TEXT}], + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"][0]["instructions"] == QUESTION_TEXT + + +def test_non_utf8_bytes_in_the_request_are_logged_and_rejected_with_a_400(): + logging_obj: Final = _logging_obj("adecisions") + raw_input: Final = b"\xff\xfe not utf-8" + + with pytest.raises(litellm.BadRequestError): + _prepare_decisions_call( + **CONNECTION, + decision_input=raw_input, + questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert base64.urlsafe_b64decode(logged["input"]) == raw_input + assert logged["questions"][0]["instructions"] == QUESTION_TEXT diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 3fc70ffec5c..6c4f6e90c10 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -99,6 +99,31 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re assert completion_cost == pytest.approx(380 * 9.7e-7) +@pytest.mark.parametrize( + ("model_info", "expected_prompt_cost"), + [ + ({"input_cost_per_token": 1e-7, "input_cost_per_character": 2.5e-8, "output_cost_per_token": 0.0}, 35 * 2.5e-8), + ({"input_cost_per_token": 1.5e-7, "output_cost_per_token": 0.0}, 11 * 1.5e-7), + ], + ids=["character_priced_model_bills_characters_instead_of_tokens", "token_priced_model_ignores_the_character_count"], +) +def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( + model_info: ModelInfo, expected_prompt_cost: float +) -> None: + usage = Usage( + prompt_tokens=11, + completion_tokens=0, + total_tokens=11, + prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0) + + def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: openai_info: ModelInfo = {"input_cost_per_token": 2e-6} fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} @@ -450,6 +475,66 @@ def test_video_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12) +def test_prompt_audio_and_video_tokens_without_modality_rates_bill_at_tiered_input_rate(): + """Prompt audio/video tokens with no modality rate are prompt tokens, so a >200k prompt bills them above-200k.""" + from unittest.mock import patch + + mock_model_info = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_above_200k_tokens": 4e-6, + "output_cost_per_token": 1.2e-5, + "output_cost_per_token_above_200k_tokens": 1.8e-5, + } + + usage = Usage( + prompt_tokens=816032, + completion_tokens=4559, + total_tokens=820591, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=9033, audio_tokens=14999, video_tokens=792000), + completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=4559), + ) + + with patch( + "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", + return_value=mock_model_info, + ): + prompt_cost, completion_cost = generic_cost_per_token( + model="test-model", usage=usage, custom_llm_provider="vertex_ai" + ) + + assert round(prompt_cost, 9) == round(816032 * 4e-6, 9) + assert round(completion_cost, 9) == round(4559 * 1.8e-5, 9) + + +def test_prompt_modality_rates_still_win_over_tiered_input_rate(): + """An explicit input_cost_per_audio_token / input_cost_per_video_token beats the fallback, even above 200k.""" + from unittest.mock import patch + + mock_model_info = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_above_200k_tokens": 4e-6, + "input_cost_per_audio_token": 3e-6, + "input_cost_per_video_token": 1e-6, + "output_cost_per_token": 1.2e-5, + } + + usage = Usage( + prompt_tokens=300000, + completion_tokens=10, + total_tokens=300010, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100000, audio_tokens=100000, video_tokens=100000), + completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=10), + ) + + with patch( + "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", + return_value=mock_model_info, + ): + prompt_cost, _ = generic_cost_per_token(model="test-model", usage=usage, custom_llm_provider="vertex_ai") + + assert round(prompt_cost, 9) == round(100000 * 4e-6 + 100000 * 3e-6 + 100000 * 1e-6, 9) + + def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 6f297e6e06a..3f63394b2f5 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -17,6 +17,7 @@ import pytest import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod +from litellm.cost_calculator import _select_model_name_for_cost_calc from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ResponseMetadata, @@ -651,3 +652,38 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_ assert result._response_ms == pytest.approx(2000) assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2) + + +def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model(): + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="azure/ptu-deployment", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="provider-model-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure" + ) + result: Final = ModelResponse( + model="gpt-5.4-nano-2026-03-17", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="azure/ptu-deployment", + kwargs={}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + result.model = "model-group-alias" + + assert ( + _select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure") + == "azure/gpt-5.4-nano-2026-03-17" + ) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 2a524f46731..d9ddc3e950b 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -20,14 +20,14 @@ from openai._legacy_response import HttpxBinaryResponseContent import litellm from litellm._logging import session_id_var, trace_id_var -from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SENTRY_DENYLIST, SENTRY_PII_DENYLIST from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import ( _extract_response_obj_and_hidden_params, - get_standard_logging_object_payload, _get_status_fields, + get_standard_logging_object_payload, set_callbacks, ) from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo @@ -335,6 +335,20 @@ def test_get_masked_api_base(logging_obj): assert type(masked_api_base) == str +@pytest.mark.parametrize("no_upstream_call", [False, True]) +def test_pre_call_stamps_the_provider_handoff_only_when_a_provider_call_is_made(logging_obj, no_upstream_call): + """The proxy fires pre_call for a failure it raised itself (auth, rate limit, client disconnect) + so callbacks see it; that call never reached a provider, so neither handoff instant may be set.""" + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + if no_upstream_call: + logging_obj.model_call_details[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + + logging_obj.pre_call(input="", api_key="") + + assert ("api_call_start_time" in logging_obj.model_call_details) is not no_upstream_call + assert (logging_obj.model_call_details.get("first_api_call_start_time") is not None) is not no_upstream_call + + def test_pre_call_tolerates_missing_api_base(logging_obj): """Presigned batch retrieves (Mistral, Bedrock) build their own URL and pass api_base=None to pre_call; masking must not raise or the request's pre-call logging is silently lost.""" @@ -1318,6 +1332,56 @@ class TestRetrieveBatchCostPassesModelIdentity: assert captured["model_info"] is not None assert captured["model_info"]["input_cost_per_token"] == 0.0 + @pytest.mark.asyncio + async def test_forwards_vertex_location_of_the_deployment(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.litellm_core_utils import litellm_logging as logging_module + + captured: dict[str, object] = {} + + from litellm.batches.batch_utils import BatchCostUsageResult + from litellm.types.utils import LiteLLMBatch, Usage + + async def fake_handle_completed_batch(**kwargs: object) -> BatchCostUsageResult: + captured.update(kwargs) + return BatchCostUsageResult( + cost=1.25, + usage=Usage(prompt_tokens=1800, completion_tokens=1000, total_tokens=2800), + models=["gemini-3.5-flash"], + successful_requests=1, + failed_requests=0, + ) + + monkeypatch.setattr(logging_module, "_handle_completed_batch", fake_handle_completed_batch) + + obj = LitellmLogging( + model="vertex_ai/gemini-3.5-flash", + messages=[{"role": "user", "content": "Hey"}], + stream=False, + call_type="aretrieve_batch", + start_time=time.time(), + litellm_call_id="batch-call-2", + function_id="f", + ) + obj.custom_llm_provider = "vertex_ai" + obj.litellm_params = {"vertex_project": "proj-1", "vertex_location": "us-central1"} + + batch = LiteLLMBatch( + id="batch_vertex", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + object="batch", + status="completed", + output_file_id="file-out", + ) + + with contextlib.suppress(Exception): + await obj._async_success_handler_body(result=batch, start_time=None, end_time=None) + + assert captured, "handle_completed_batch was never called" + assert captured["vertex_location"] == "us-central1" + class TestRetrieveBatchPricesOnlyFinalBatches: """Regression (LIT-7048): retrieving a provider-id batch priced it on every poll. @@ -6783,10 +6847,10 @@ async def test_streaming_success_callbacks_survive_guardrail_logging_hook_failur def _resolve(custom_llm_provider, litellm_params, optional_params, model): from litellm.litellm_core_utils.litellm_logging import ( - _resolve_vertex_location_for_cost, + resolve_vertex_location_for_cost, ) - return _resolve_vertex_location_for_cost( + return resolve_vertex_location_for_cost( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, optional_params=optional_params, @@ -8325,6 +8389,7 @@ class TestAzurePTUSpilloverCost: ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id" SERVED_MODEL: Final = "azure/spill-served-model-ptu" + PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment" PTU_MODEL_INFO: Final = { "id": ROUTER_MODEL_ID, "team_id": "team-1", @@ -8360,7 +8425,9 @@ class TestAzurePTUSpilloverCost: litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None) litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None) - def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging: + def _logging_obj( + self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None + ) -> LitellmLogging: monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag) obj = LitellmLogging( model=self.SERVED_MODEL, @@ -8377,7 +8444,7 @@ class TestAzurePTUSpilloverCost: optional_params={}, litellm_params={ "api_base": "", - "metadata": {"model_info": model_info}, + "metadata": {"model_info": model_info, "deployment": deployment}, "input_cost_per_token": litellm_rate, "output_cost_per_token": litellm_rate, }, @@ -8447,6 +8514,84 @@ class TestAzurePTUSpilloverCost: finally: self._unregister_models() + def test_spillover_payload_reports_the_pay_as_you_go_entry_billing_used(self, monkeypatch) -> None: + self._register_models() + try: + obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch) + response = self._response() + response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"} + response_cost: Final = obj._response_cost_calculator(result=response) + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": self.SERVED_MODEL, + "custom_llm_provider": "azure", + "litellm_params": obj.litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=obj, + status="success", + ) + + assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST) + assert payload is not None + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"] + assert model_map_value["output_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"] + assert response_cost == pytest.approx( + 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"] + ) + finally: + self._unregister_models() + + def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None: + self._register_models() + litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"} + try: + obj = self._logging_obj( + dict(self.PTU_MODEL_INFO), + flag="True", + litellm_rate=0.0, + monkeypatch=monkeypatch, + deployment=self.PTU_DEPLOYMENT, + ) + response = self._response() + response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"} + response._hidden_params["provider_response_model"] = response.model + response_cost: Final = obj._response_cost_calculator(result=response) + response.model = "spill-model-group-alias" + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": self.SERVED_MODEL, + "custom_llm_provider": "azure", + "litellm_params": obj.litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=obj, + status="success", + ) + + assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST) + assert payload is not None + model_map_information: Final = payload["model_map_information"] + assert model_map_information["model_map_key"] != self.PTU_DEPLOYMENT + model_map_value: Final = model_map_information["model_map_value"] + assert model_map_value is not None + assert response_cost == pytest.approx( + 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"] + ) + finally: + litellm.model_cost.pop(self.PTU_DEPLOYMENT, None) + self._unregister_models() + def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None: self._register_models() custom_model_id: Final = "non-ptu-custom-router-model-id" diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 729f46ec57f..f2a7c4f44c2 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -29,7 +29,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transfor from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, ) -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES from litellm.types.utils import ServerToolUse, Usage @@ -6798,3 +6798,91 @@ def test_chat_dummy_tool_result_for_an_orphaned_tool_call_replays_a_byte_identic _assert_prefix_stable(requests) assert [m["role"] for m in requests[0]["messages"]] == ["user", "assistant", "user"] assert requests[0]["messages"][2]["content"][0]["type"] == "tool_result" + + +def _make_transform_request(optional_params: dict, litellm_params: dict) -> dict: + return AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + +def _two_function_tools() -> list[dict]: + return [ + { + "type": "function", + "function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}}, + } + for i in range(2) + ] + + +@pytest.mark.parametrize( + "control, expected_control", + [ + (None, {"type": "ephemeral"}), + ({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}), + ], +) +def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control): + """LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which + Anthropic rejected with 400 Extra inputs are not permitted.""" + tools = _two_function_tools() + tools_before = copy.deepcopy(tools) + point = {"location": "tool_config", **({"control": control} if control is not None else {})} + litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [point]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][-1]["cache_control"] == expected_control + assert "cache_control" not in data["tools"][0] + assert tools == tools_before + assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1" + + +def test_transform_request_tool_config_point_keeps_the_clients_tool_marker(): + tools = _two_function_tools() + tools[0]["cache_control"] = {"type": "ephemeral"} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][1] + + +def test_transform_request_tool_config_point_skips_tool_search_tools(): + tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)) + tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}] + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert data["tools"][1]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][2] + assert "cache_control" not in data["tools"][0] + + +def test_transform_request_tool_config_point_without_tools_drops_the_key(): + litellm_params = {"metadata": {}} + + data = _make_transform_request( + optional_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert "tools" not in data + assert litellm_params["metadata"] == {} diff --git a/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 507467b721f..c1287fa4c17 100644 --- a/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,4 +1,5 @@ import asyncio +import copy import json import os import uuid @@ -1480,6 +1481,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"] + @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" @@ -1594,7 +1596,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict return safeguards, safeguard_results -def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler: +def _upstream_answering_with( + safeguard_results: list[dict[str, object]], captured: dict[str, object] +) -> AsyncHTTPHandler: def upstream_records_the_request(request: httpx.Request) -> httpx.Response: captured["body"] = json.loads(request.content) captured["anthropic-beta"] = request.headers.get("anthropic-beta") @@ -1620,7 +1624,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture _CLIENT_BETA_HEADERS: Final = ( - pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"), + pytest.param( + {"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta" + ), pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"), pytest.param({}, id="client_sends_no_beta_header"), ) @@ -1685,3 +1691,42 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet assert "anthropic_beta" not in captured["body"] assert captured["anthropic-beta"].split(",").count("dangerous-tool-use-2026-09-03") == 1 assert response["safeguard_results"] == safeguard_results + + +def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch): + """LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the + native path dropped it silently, so the tools went out without any cache breakpoint.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + captured = {} + + def fake_native(**kwargs): + captured["params"] = kwargs["anthropic_messages_optional_request_params"] + captured["kwargs"] = kwargs["kwargs"] + captured["messages"] = kwargs["messages"] + return "native-passthrough" + + monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native) + tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)] + tools_before = copy.deepcopy(tools) + + result = handler.anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="anthropic/claude-opus-4-6", + api_key="sk-test", + tools=tools, + litellm_metadata={}, + cache_control_injection_points=[ + {"location": "message", "role": "user"}, + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + ) + + assert result == "native-passthrough" + assert "cache_control_injection_points" not in captured["params"] + assert "cache_control_injection_points" not in captured["kwargs"] + assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in captured["params"]["tools"][0] + assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert tools == tools_before diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 97bea84c7ea..82d1f76a7eb 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -2900,7 +2900,7 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati from litellm.litellm_core_utils.litellm_logging import ( Logging, - _resolve_vertex_location_for_cost, + resolve_vertex_location_for_cost, ) monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") @@ -2949,7 +2949,7 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati global_deployment = await logging_obj_after_handler(GenericLiteLLMParams(vertex_location="global")) assert global_deployment.litellm_params["vertex_location"] == "global" assert ( - _resolve_vertex_location_for_cost( + resolve_vertex_location_for_cost( custom_llm_provider="vertex_ai", litellm_params=global_deployment.litellm_params, optional_params=global_deployment.optional_params, diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/__init__.py b/tests/unit/llms/vertex_ai/vertex_embeddings/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py new file mode 100644 index 00000000000..cbc074b4e94 --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py @@ -0,0 +1,30 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.vertex_embeddings.transformation import VertexAITextEmbeddingConfig +from litellm.types.utils import EmbeddingResponse + + +def _gecko_prediction(values: list[float], token_count: int) -> dict[str, object]: + return {"embeddings": {"values": values, "statistics": {"token_count": token_count, "truncated": False}}} + + +@pytest.mark.parametrize( + ("metadata", "character_count"), + [({"metadata": {"billableCharacterCount": 10}}, 10), ({}, None)], + ids=["billable_character_count_reported", "no_metadata"], +) +def test_vertex_embedding_usage_carries_the_billable_character_count_google_prices_on( + metadata: dict[str, object], character_count: int | None +) -> None: + payload: Final = {"predictions": [_gecko_prediction([0.1, 0.2, 0.3], 3), _gecko_prediction([1, 2.0, 3], 5)], **metadata} + + response: Final = VertexAITextEmbeddingConfig().transform_vertex_response_to_openai( + response=payload, model="text-embedding-005", model_response=EmbeddingResponse() + ) + + assert (response.usage.prompt_tokens, getattr(response.usage.prompt_tokens_details, "character_count", None)) == ( + 8, + character_count, + ) diff --git a/tests/unit/proxy/hooks/test_search_free_tier.py b/tests/unit/proxy/hooks/test_search_free_tier.py new file mode 100644 index 00000000000..b57908378e5 --- /dev/null +++ b/tests/unit/proxy/hooks/test_search_free_tier.py @@ -0,0 +1,210 @@ +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.proxy.hooks.search_free_tier import ( + apply_search_free_tier, + freed_units, + search_free_tier_key, + search_free_tier_ttl, +) +from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage + +NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc) +RATE: Final = 0.01 + + +class _Counter: + def __init__(self) -> None: + self.totals: dict[str, float] = {} + self.ttls: dict[str, int | None] = {} + + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: + self.totals[key] = self.totals.get(key, 0) + value + self.ttls[key] = ttl + return self.totals[key] + + +class _BrokenCounter: + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: + raise ConnectionError("redis is down") + + +def _model_info(billing_unit: str, count: int, period: str = "month", provider: str = "vertex_ai") -> dict: + return { + "litellm_provider": provider, + "mode": "chat", + "web_search_billing_unit": billing_unit, + "search_context_cost_per_query": {"search_context_size_medium": RATE}, + "search_free_requests": {"count": count, "period": period, "shared_group": "test-group"}, + } + + +def _response(queries: int | None) -> ModelResponse: + details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None + return ModelResponse( + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details) + ) + + +def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]: + sl_object = { + "response_cost": response_cost, + "cache_hit": cache_hit, + "custom_llm_provider": model_info["litellm_provider"], + "cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost}, + "model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info}, + } + return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object + + +@pytest.mark.parametrize( + ("allowance", "units", "total_after", "expected"), + [ + (5000, 5, 5, 5), + (5000, 5, 5000, 5), + (5000, 10, 5005, 5), + (5000, 5, 5010, 0), + (2, 5, 5, 2), + (0, 3, 3, 0), + ], +) +def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected): + assert freed_units(allowance, units, total_after) == expected + + +def test_counter_key_is_per_provider_group_and_utc_period(): + assert search_free_tier_key("gemini", {"count": 1, "period": "month", "shared_group": "g3"}, NOW) == ( + "litellm:search_free_tier:gemini:g3:2026-10" + ) + assert search_free_tier_key("vertex_ai", {"count": 1, "period": "day", "shared_group": "g2"}, NOW) == ( + "litellm:search_free_tier:vertex_ai:g2:2026-10-20" + ) + + +@pytest.mark.parametrize( + ("period", "boundary"), + [("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))], +) +def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary): + expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW)) + assert boundary < expiry < boundary + timedelta(hours=2) + + +@pytest.mark.asyncio +async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call(): + counter = _Counter() + model_info = _model_info("per_query", count=7) + reduced = [] + for _ in range(3): + kwargs, sl_object = _call(model_info, response_cost=1.0) + cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"])) + (c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced + assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE) + assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE) + assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5 + assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE) + assert b2["discount_amount"] == pytest.approx(2 * RATE) + assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3 + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 15} + assert counter.ttls["litellm:search_free_tier:vertex_ai:test-group:2026-10"] > 0 + + +@pytest.mark.asyncio +async def test_gemini_api_and_vertex_ai_grounding_draw_on_separate_allowances(): + counter = _Counter() + gemini_api = _model_info("per_query", count=3, provider="gemini") + vertex_ai = _model_info("per_query", count=3, provider="vertex_ai") + kwargs, sl_object = _call(gemini_api, response_cost=1.0) + await apply_search_free_tier(kwargs, sl_object, 1.0, _response(2), counter, NOW) + kwargs, sl_object = _call(vertex_ai, response_cost=1.0) + vertex_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(4), counter, NOW) + kwargs, sl_object = _call(gemini_api, response_cost=1.0) + gemini_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(1), counter, NOW) + assert vertex_cost == pytest.approx(1.0 - 3 * RATE), "vertex ai had its whole allowance left" + assert gemini_cost == pytest.approx(1.0 - RATE), "gemini api had one free query left" + + +@pytest.mark.asyncio +async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count(): + counter = _Counter() + model_info = _model_info("per_prompt", count=1, period="day") + kwargs, sl_object = _call(model_info, response_cost=1.0) + first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + kwargs, sl_object = _call(model_info, response_cost=1.0) + second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + assert first == pytest.approx(1.0 - RATE) + assert second == 1.0 + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10-20": 2} + + +@pytest.mark.asyncio +async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response(): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + response = { + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + "prompt_tokens_details": {"web_search_requests": 3}, + } + } + assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("counter", [None, _BrokenCounter()]) +async def test_full_price_when_no_counter_is_available(counter): + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0 + assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()]) +async def test_requests_without_grounding_never_touch_the_counter(response): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0 + assert counter.totals == {} + + +@pytest.mark.asyncio +async def test_models_without_an_allowance_never_touch_the_counter(): + counter = _Counter() + model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"} + kwargs, sl_object = _call(model_info, response_cost=1.0) + assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0 + assert counter.totals == {} + + +@pytest.mark.asyncio +async def test_a_second_callback_run_on_the_same_request_counts_it_once(): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW) + assert first == second == pytest.approx(1.0 - 5 * RATE) + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5} + + +@pytest.mark.asyncio +async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero(): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True) + assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0 + assert counter.totals == {} + assert "discount_amount" not in sl_object["cost_breakdown"] + + +@pytest.mark.asyncio +async def test_freed_cost_never_exceeds_the_billed_cost(): + counter = _Counter() + billed = 3 * RATE + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed) + assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0 + assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed) + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5} diff --git a/tests/unit/proxy/test_update_spend.py b/tests/unit/proxy/test_update_spend.py index ebe505b3d60..ea80d675142 100644 --- a/tests/unit/proxy/test_update_spend.py +++ b/tests/unit/proxy/test_update_spend.py @@ -321,3 +321,31 @@ async def test_update_spend_logs_multiple_batches_with_failure(): # Verify all logs were cleared from transactions assert len(prisma_client.spend_log_transactions) == 0 + + +@pytest.mark.asyncio +async def test_update_spend_logs_drops_row_prisma_cannot_encode_and_keeps_the_rest(): + """A row the Prisma engine cannot utf-8 encode (a lone surrogate in a string + column) is permanent, so it must be dropped like any other poison row instead + of being requeued at the head of the queue where it would block every tenant.""" + prisma_client = MockPrismaClient() + proxy_logging_obj = create_mock_proxy_logging() + prisma_client.spend_log_transactions = [ + {"request_id": "good-1", "request_tags": '["a"]'}, + {"request_id": "poison", "request_tags": "\ud800"}, + {"request_id": "good-2", "request_tags": '["b"]'}, + ] + written = [] + + async def create_many(data, skip_duplicates): + if any("\ud800" in row["request_tags"] for row in data): + raise UnicodeEncodeError("utf-8", "\ud800", 0, 1, "surrogates not allowed") + written.extend(row["request_id"] for row in data) + + prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=create_many) + + await update_spend(prisma_client, None, proxy_logging_obj) + + assert sorted(written) == ["good-1", "good-2"] + assert prisma_client.spend_log_transactions == [] + proxy_logging_obj.failure_handler.assert_not_called() diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index c13054f1cee..b9f68e6d128 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5311,3 +5311,40 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool ) assert exc_info.value.status_code == 400 assert "tool_choice={'type': 'file_search'}" in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chunks(): + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + + chat_stream: Final = await litellm.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "tell me a story"}], + mock_response="The codename is AZURE-FALCON-42 and the story is long.", + stream=True, + api_key="test-key", + ) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="gpt-4o-mini", + litellm_custom_stream_wrapper=chat_stream, + request_input="tell me a story", + responses_api_request={}, + custom_llm_provider="openai", + ) + assert iterator.partial_response_on_disconnect() is None, "nothing relayed yet, nothing to bill" + events: Final = [await iterator.__anext__() for _ in range(6)] + relayed_text: Final = "".join( + getattr(event, "delta", "") for event in events if event.type == "response.output_text.delta" + ) + assert relayed_text, "the bridge should have relayed some text by now" + + incomplete_event: Final = iterator.partial_response_on_disconnect() + + assert incomplete_event is not None + assert incomplete_event.type == "response.incomplete" + assert incomplete_event.response.status == "incomplete" + assert incomplete_event.response.usage is not None + assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text) + assert incomplete_event.response.usage.input_tokens > 0 diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 9ba7dfcaa80..f8064f9bc88 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -7,7 +7,10 @@ crashed with exit 139 whenever any CustomLogger was registered). """ import asyncio +import json import time +from collections.abc import Mapping +from typing import Final import httpx import pytest @@ -17,6 +20,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module from litellm.responses import streaming_iterator as responses_streaming_iterator_module from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator from litellm.types.llms.openai import ResponsesAPIResponse @@ -156,3 +160,98 @@ async def test_sync_callbacks_run_only_after_async_handler_completes(recording_e submit_times = recording_executor.submit_times_for(logging_obj) assert len(submit_times) == 1 assert submit_times[0] >= recorder.async_hook_finished + + +_CREATED_RESPONSE_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = { + "id": "resp_partial", + "object": "response", + "created_at": 1700000000.0, + "model": "gpt-5.4-nano", + "status": "in_progress", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "service_tier": "priority", +} +_CREATED_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = { + "type": "response.created", + "sequence_number": 0, + "response": _CREATED_RESPONSE_WITH_PRIORITY_TIER, +} + + +def _text_delta(text: str, sequence_number: int) -> Mapping[str, object]: + return { + "type": "response.output_text.delta", + "sequence_number": sequence_number, + "item_id": "msg_partial", + "output_index": 0, + "content_index": 0, + "delta": text, + } + + +def _completed_after(text: str) -> Mapping[str, object]: + return { + "type": "response.completed", + "sequence_number": 9, + "response": { + **_CREATED_RESPONSE_WITH_PRIORITY_TIER, + "status": "completed", + "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, + }, + } + + +def _live_iterator(logging_obj: LitellmLogging) -> ResponsesAPIStreamingIterator: + return ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="gpt-5.4-nano", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + logging_obj=logging_obj, + request_data={"model": "gpt-5.4-nano", "input": "tell me a story"}, + custom_llm_provider="openai", + ) + + +def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier(): + iterator: Final = _live_iterator(_make_logging_obj()) + for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time ", 1), _text_delta("a fox slept.", 2)): + iterator._process_chunk(json.dumps(event)) + + incomplete_event: Final = iterator.partial_response_on_disconnect() + + assert incomplete_event is not None + assert incomplete_event.type == "response.incomplete" + partial: Final = incomplete_event.response + assert partial.status == "incomplete" + assert getattr(partial, "service_tier") == "priority" + assert partial.output[0].content[0].text == "Once upon a time a fox slept." + assert partial.usage.output_tokens == litellm.token_counter( + model="gpt-5.4-nano", text="Once upon a time a fox slept." + ) + assert partial.usage.input_tokens == litellm.token_counter( + model="gpt-5.4-nano", messages=[{"role": "user", "content": "tell me a story"}] + ) + assert partial.usage.total_tokens == partial.usage.input_tokens + partial.usage.output_tokens + + +@pytest.mark.parametrize( + "events", + [ + (), + (_CREATED_WITH_PRIORITY_TIER,), + (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1), _completed_after("Once upon a time")), + ], + ids=["nothing_relayed", "created_only", "already_completed"], +) +@pytest.mark.asyncio +async def test_partial_response_on_disconnect_is_none_when_there_is_nothing_to_bill_or_the_stream_billed_itself( + events: tuple[Mapping[str, object], ...], +): + iterator: Final = _live_iterator(_make_logging_obj()) + for event in events: + iterator._process_chunk(json.dumps(event)) + + assert iterator.partial_response_on_disconnect() is None diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index dbf54ec3b9b..597d0432682 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -437,26 +437,26 @@ def _responses_api_response_with_usage() -> ResponsesAPIResponse: def test_stamp_responses_usage_cost_stamps_computed_cost(): - from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() logging_obj = Mock(spec=LiteLLMLoggingObj) logging_obj._response_cost_calculator.return_value = 0.000704 - _stamp_responses_usage_cost(response, logging_obj) + stamp_responses_usage_cost(response, logging_obj) assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) logging_obj._response_cost_calculator.assert_called_once_with(result=response) def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): - from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() setattr(response.usage, "cost", 0.5) logging_obj = Mock(spec=LiteLLMLoggingObj) - _stamp_responses_usage_cost(response, logging_obj) + stamp_responses_usage_cost(response, logging_obj) assert getattr(response.usage, "cost", None) == pytest.approx(0.5) logging_obj._response_cost_calculator.assert_not_called() @@ -476,7 +476,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): - from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _unvalidated_response_with_dict_usage( { "input_tokens": 29, @@ -488,7 +488,7 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): ) logging_obj = Mock(spec=LiteLLMLoggingObj) - _stamp_responses_usage_cost(response, logging_obj) + stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) @@ -497,12 +497,12 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): - from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) logging_obj = Mock(spec=LiteLLMLoggingObj) logging_obj._response_cost_calculator.return_value = 0.000704 - _stamp_responses_usage_cost(response, logging_obj) + stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(0.000704) @@ -510,13 +510,13 @@ def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): def test_stamp_responses_usage_cost_survives_calculator_failure(): - from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() logging_obj = Mock(spec=LiteLLMLoggingObj) logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable") - _stamp_responses_usage_cost(response, logging_obj) + stamp_responses_usage_cost(response, logging_obj) assert getattr(response.usage, "cost", None) is None diff --git a/tests/unit/router_strategy/test_lowest_cost.py b/tests/unit/router_strategy/test_lowest_cost.py index ab3ef099410..4f7259178f6 100644 --- a/tests/unit/router_strategy/test_lowest_cost.py +++ b/tests/unit/router_strategy/test_lowest_cost.py @@ -101,3 +101,49 @@ async def test_async_log_success_event_counts_a_response_with_no_completion_toke ) assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} + + + + + + + + + +_TWO_DOLLARS_PER_TOKEN = {"input_cost_per_token": 1.0, "output_cost_per_token": 1.0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "azure_litellm_params, base_model", + [ + ({"model": "azure/my-gpt4-deploy"}, "azure/gpt-4.1"), + ({"model": "azure/gpt-4.1"}, "azure/not-a-real-model"), + ], + ids=["base_model_priced", "unmapped_base_model_falls_back_to_deployment_model"], +) +async def test_cost_based_routing_prices_an_azure_deployment_through_its_base_model( + azure_litellm_params: dict, base_model: str +): + handler = LowestCostLoggingHandler(router_cache=DualCache()) + healthy_deployments = [ + { + "model_name": "gpt-4.1", + "litellm_params": {"model": "openai/gpt-4.1", **_TWO_DOLLARS_PER_TOKEN}, + "model_info": {"id": "openai-two-dollars"}, + }, + { + "model_name": "gpt-4.1", + "litellm_params": azure_litellm_params, + "model_info": {"id": "azure-cost-map-priced", "base_model": base_model}, + }, + ] + + picked = await handler.async_get_available_deployments( + model_group="gpt-4.1", healthy_deployments=healthy_deployments + ) + + assert picked is not None and picked["model_info"]["id"] == "azure-cost-map-priced", ( + "a cost-map priced Azure deployment must beat a $2 per token deployment, " + "it loses only when it is scored at the $10 unmapped default" + ) diff --git a/tests/unit/router_utils/test_deployment_pricing.py b/tests/unit/router_utils/test_deployment_pricing.py new file mode 100644 index 00000000000..45cc211a89c --- /dev/null +++ b/tests/unit/router_utils/test_deployment_pricing.py @@ -0,0 +1,53 @@ +import pytest + +from litellm.router_utils.deployment_pricing import deployment_pricing_models, pricing_models + + +@pytest.mark.parametrize( + "base_model, model, expected", + [ + ("azure/gpt-4.1", "azure/my-gpt4-deploy", ("azure/gpt-4.1", "azure/my-gpt4-deploy")), + (None, "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)), + ("", "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)), + ("azure/gpt-4.1", "azure/gpt-4.1", ("azure/gpt-4.1",)), + (None, None, ()), + ], + ids=["base_model_first", "no_base_model", "blank_base_model", "same_name_once", "nothing_named"], +) +def test_pricing_models_walks_base_model_then_deployment_model(base_model, model, expected): + assert pricing_models(base_model, model) == expected + + +@pytest.mark.parametrize( + "deployment, expected", + [ + ( + {"litellm_params": {"model": "azure/my-gpt4-deploy"}, "model_info": {"base_model": "azure/gpt-4.1"}}, + ("azure/gpt-4.1", "azure/my-gpt4-deploy"), + ), + ( + {"litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4.1"}}, + ("azure/gpt-4.1", "azure/my-gpt4-deploy"), + ), + ( + { + "litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4o"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ("azure/gpt-4.1", "azure/my-gpt4-deploy"), + ), + ({"litellm_params": {"model": "openai/gpt-4.1"}, "model_info": {}}, ("openai/gpt-4.1",)), + ({"litellm_params": "not-a-dict", "model_info": None}, ()), + ("not-a-deployment", ()), + ], + ids=[ + "model_info_base_model", + "params_base_model", + "model_info_wins", + "no_base_model", + "malformed_blocks", + "not_a_dict", + ], +) +def test_deployment_pricing_models_reads_the_deployment_dict(deployment, expected): + assert deployment_pricing_models(deployment) == expected diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index fa284ff71cf..407da680006 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -12,6 +12,7 @@ from pydantic import BaseModel import litellm from litellm.cost_calculator import ( + batch_cost_calculator, BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, @@ -3223,6 +3224,29 @@ def _batch_cache_usage() -> Usage: ) +def test_batch_cost_calculator_vertex_regional_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None: + """ + Regression for https://github.com/BerriAI/litellm/issues/46880: a Vertex batch + on a regional location must cost the global batch times the model's + regional_endpoint_uplift_multiplier, the same invariant the online path keeps. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000) + global_prompt, global_completion = batch_cost_calculator( + usage=usage, model="gemini-3.5-flash", custom_llm_provider="vertex_ai", vertex_location="global" + ) + regional_prompt, regional_completion = batch_cost_calculator( + usage=usage, model="gemini-3.5-flash", custom_llm_provider="vertex_ai", vertex_location="us-central1" + ) + + uplift = litellm.get_model_info("vertex_ai/gemini-3.5-flash")["regional_endpoint_uplift_multiplier"] + assert global_prompt + global_completion > 0 + assert regional_prompt == pytest.approx(global_prompt * uplift, rel=1e-9) + assert regional_completion == pytest.approx(global_completion * uplift, rel=1e-9) + + def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates(): from litellm.cost_calculator import batch_cost_calculator diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 709ed93639a..3e37e47aefa 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -10,11 +10,13 @@ from typing import Final import jsonschema import pytest +from pydantic import JsonValue, TypeAdapter import litellm from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts +from litellm.types.utils import SearchFreeRequests REPO_ROOT = Path(__file__).parents[2] GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" @@ -591,3 +593,30 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): "unknown_root-new_family_models", "vertex_ai-new_family_models", ] + + +def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + return MappingProxyType( + {key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row} + ) + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None: + rows = _search_free_requests_rows(path) + assert rows + allowances = { + key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items() + } + for key, row in rows.items(): + assert row.get("supports_web_search") is True, key + assert row.get("search_context_cost_per_query"), key + assert allowances[key]["count"] > 0, key + for group in {allowance["shared_group"] for allowance in allowances.values()}: + same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group} + assert len(same_group) == 1, (group, same_group) + + +def test_search_free_requests_backup_rows_match_the_main_catalog() -> None: + assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 80131534183..c0266492b1e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -11833,7 +11833,7 @@ async def test_cyclic_fallback_graph_does_not_amplify_one_request(): _FLAT_ATTEMPT_RECORD_KEYS = frozenset( - {"model_group", "deployment_id", "exception_type", "exception_string", "attempted_retries"} + {"model_group", "deployment_id", "model", "base_model", "exception_type", "exception_string", "attempted_retries"} ) _BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doNotShip" @@ -18143,3 +18143,68 @@ def test_bare_model_group_served_by_wildcard_deployment_has_provider_prefixed_co assert router._has_content_policy_fallback("claude-sonnet-4-6", {}) is True assert router._has_content_policy_fallback("claude-haiku-4-5", {}) is False + + +@pytest.mark.asyncio +async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens(): + """After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone.""" + import json + + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + def _relayed_iterator(text: str) -> ResponsesAPIStreamingIterator: + logging_obj: Final = MagicMock(spec=LiteLLMLogging) + logging_obj.model_call_details = {"litellm_params": {}} + logging_obj.completion_start_time = None + iterator: Final = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="gpt-5.4", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + logging_obj=logging_obj, + request_data={"model": "gpt-5.4", "input": "Hi"}, + custom_llm_provider="openai", + ) + created: Final = { + "id": "resp_1", + "object": "response", + "created_at": 1700000000.0, + "model": "gpt-5.4", + "status": "in_progress", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + iterator._process_chunk(json.dumps({"type": "response.created", "sequence_number": 0, "response": created})) + iterator._process_chunk( + json.dumps( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": text, + } + ) + ) + return iterator + + router: Final = Router( + model_list=[{"model_name": "primary", "litellm_params": {"model": "openai/gpt-5.4", "api_key": "k"}}] + ) + primary: Final = _relayed_iterator("Once upon a time there was a lighthouse keeper") + fallback: Final = _relayed_iterator("who lit the lamp") + wrapped: Final = await router._aresponses_streaming_iterator( + response=primary, + initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": AsyncMock()}, + ) + primary_tokens: Final = primary.partial_response_on_disconnect().response.usage.output_tokens + fallback_tokens: Final = fallback.partial_response_on_disconnect().response.usage.output_tokens + + assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + + wrapped.adopt_fallback_headers(fallback) + + assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py new file mode 100644 index 00000000000..9520bf57a4c --- /dev/null +++ b/tests/unit/test_router/test_router_timeout.py @@ -0,0 +1,61 @@ +from typing import Final + +import pytest + +from litellm import Router + + +@pytest.mark.parametrize( + "request_kwargs, expected_timeout", + [ + ({"service_tier": "flex"}, 900.0), + ({"service_tier": "default"}, 610.0), + ({}, 610.0), + ({"service_tier": "flex", "timeout": 20}, 20), + ({"service_tier": "flex", "stream": True}, 950.0), + ({"service_tier": "default", "stream": True}, 610.0), + ({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30), + ], +) +def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout( + request_kwargs: dict[str, object], expected_timeout: float +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai-gpt-5.1", + "litellm_params": { + "model": "openai/gpt-5.1", + "api_key": "sk-test", + "timeout": 610, + "stream_timeout": 610, + "timeout_by_service_tier": {"flex": 900}, + "stream_timeout_by_service_tier": {"flex": 950}, + }, + } + ] + ) + deployment_params: Final = router.model_list[0]["litellm_params"] + + assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout + + +def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai-gpt-5.1", + "litellm_params": { + "model": "openai/gpt-5.1", + "api_key": "sk-test", + "timeout": 610, + "stream_timeout": 30, + "timeout_by_service_tier": {"flex": 900}, + }, + } + ] + ) + deployment_params: Final = router.model_list[0]["litellm_params"] + + assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0 + assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0 diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a8385b8340d..cb6a16e0c4c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1068,16 +1068,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "object", "additionalProperties": {"type": "number"}, }, - "search_free_requests": { - "type": "object", - "properties": { - "count": {"type": "integer", "minimum": 1}, - "period": {"type": "string", "enum": ["day", "month"]}, - "shared_group": {"type": "string"}, - }, - "required": ["count", "period", "shared_group"], - "additionalProperties": False, - }, "search_context_cost_per_query": { "type": "object", "properties": { @@ -1091,6 +1081,16 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "string", "enum": ["per_prompt", "per_query"], }, + "search_free_requests": { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": {"type": "string"}, + }, + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + }, "citation_cost_per_token": {"type": "number"}, "supported_modalities": { "type": "array", @@ -3187,9 +3187,7 @@ class TestExtraBodyCannotOverrideModel: } ] - untouched = litellm.get_optional_params( - model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools() - ) + untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()) assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched result = litellm.get_optional_params( @@ -4581,7 +4579,13 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = ( pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"), pytest.param( ResponsesAPIResponse( - id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6" + id="resp_abc", + created_at=1, + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + model="gpt-5.6", ), CallTypes.aresponses, id="responses", @@ -4609,7 +4613,9 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo assert result is response assert second_hook.seen_responses == (response,) - failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message) + failure_logs: Final = tuple( + r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message + ) assert len(failure_logs) == 1 assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message assert str(call_type) in failure_logs[0].message @@ -6448,3 +6454,33 @@ def test_registered_fields_reset_when_model_cost_is_replaced(monkeypatch: pytest monkeypatch.setattr(litellm, "model_cost", {}) assert litellm.utils._registered_fields_for_current_map() == {} + + +def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body(): + """LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no + step that consumes it, so it must not ride along in extra_body and 400 at OpenAI.""" + optional_params = litellm.utils.get_optional_params( + model="gpt-4.1", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}], + extra_body={"kept": True}, + ) + + assert optional_params["extra_body"] == {"kept": True} + assert "cache_control_injection_points" not in optional_params + + +def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy(): + """A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy + applies cache_control_injection_points itself, so the point must ride along in extra_body.""" + point: Final = {"location": "tool_config", "control": {"type": "ephemeral"}} + optional_params: Final[dict[str, object]] = litellm.utils.get_optional_params( + model="claude-sonnet-4-5", + custom_llm_provider="litellm_proxy", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + cache_control_injection_points=[point], + extra_body={"kept": True}, + ) + + assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index e421321aaaa..d88fcce4936 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -74,6 +74,8 @@ CONNECTION_NAMES: Final = ( "request_timeout", "force_timeout", "stream_timeout", + "timeout_by_service_tier", + "stream_timeout_by_service_tier", "max_retries", "tenant_id", "client_id", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 02e2a0dd073..bcd00757227 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32938,6 +32938,10 @@ export interface components { } | null; /** Stream Timeout */ stream_timeout?: number | string | null; + /** Stream Timeout By Service Tier */ + stream_timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tag Regex */ tag_regex?: string[] | null; /** Tags */ @@ -32950,6 +32954,10 @@ export interface components { }[] | null; /** Timeout */ timeout?: number | string | null; + /** Timeout By Service Tier */ + timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tpm */ tpm?: number | null; /** Use Chat Completions Api */ @@ -46554,6 +46562,10 @@ export interface components { } | null; /** Stream Timeout */ stream_timeout?: number | string | null; + /** Stream Timeout By Service Tier */ + stream_timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tag Regex */ tag_regex?: string[] | null; /** Tags */ @@ -46566,6 +46578,10 @@ export interface components { }[] | null; /** Timeout */ timeout?: number | string | null; + /** Timeout By Service Tier */ + timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tpm */ tpm?: number | null; /** Use Chat Completions Api */