From f08742c6f4ef09ab22ac07f96a691bab6c864d25 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:29:48 -0700 Subject: [PATCH 01/54] fix(interactions): track cost and spend for Google Interactions API requests --- litellm/cost_calculator.py | 7 + litellm/litellm_core_utils/litellm_logging.py | 54 +++++-- .../usage_object_transformation.py | 109 ++++++++++++- .../test_usage_object_transformation.py | 121 ++++++++++++++ .../test_litellm_logging.py | 147 ++++++++++++++++++ tests/test_litellm/test_cost_calculator.py | 33 ++++ 6 files changed, 460 insertions(+), 11 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a40a8e1389c..3c541605caa 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + InteractionsUsageObjectTransformation, TranscriptionUsageObjectTransformation, ) from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -899,6 +900,8 @@ def _get_usage_object( usage_obj, ) ) + elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj): + return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj) elif isinstance(usage_obj, dict): return Usage(**usage_obj) elif isinstance(usage_obj, BaseModel): @@ -1267,6 +1270,10 @@ def completion_cost( ) if tr_usage is not None: _usage = tr_usage.model_dump() + elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage): + _usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + _usage + ).model_dump() else: _usage = _usage diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 461ab62b815..0f6e0ccf9c1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -68,6 +68,9 @@ from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) +from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + InteractionsUsageObjectTransformation, +) from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( @@ -93,6 +96,10 @@ from litellm.types.llms.openai import ( ResponseIncompleteEvent, ResponsesAPIResponse, ) +from litellm.types.interactions import ( + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, +) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse @@ -1888,6 +1895,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, FineTuningJob) or isinstance(logging_result, LiteLLMBatch) or isinstance(logging_result, ResponsesAPIResponse) + or isinstance(logging_result, InteractionsAPIResponse) or isinstance(logging_result, OpenAIFileObject) or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) or isinstance(logging_result, OpenAIModerationResponse) @@ -1973,7 +1981,7 @@ class Logging(LiteLLMLoggingBaseClass): try: ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse, InteractionsAPIResponse] ] = None if "complete_streaming_response" in self.model_call_details: return # break out of this. @@ -2428,14 +2436,14 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: return # break out of this. - complete_streaming_response: Optional[Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]] = ( - self._get_assembled_streaming_response( - result=result, - start_time=start_time, - end_time=end_time, - is_async=True, - streaming_chunks=self.streaming_chunks, - ) + complete_streaming_response: ( + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse, InteractionsAPIResponse] | None + ) = self._get_assembled_streaming_response( + result=result, + start_time=start_time, + end_time=end_time, + is_async=True, + streaming_chunks=self.streaming_chunks, ) if complete_streaming_response is not None: @@ -3153,7 +3161,7 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: List[Any], - ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]]: + ) -> Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse, InteractionsAPIResponse] | None: if self.stream is not True: return None if isinstance(result, ModelResponse): @@ -3180,9 +3188,31 @@ class Logging(LiteLLMLoggingBaseClass): ), ) return result.response + elif isinstance(result, InteractionsAPIStreamingResponse): + return self._assemble_completed_interaction_response(result) else: return None + @staticmethod + def _assemble_completed_interaction_response( + result: InteractionsAPIStreamingResponse, + ) -> InteractionsAPIResponse | None: + """ + The Interactions API streaming iterator hands the terminal event to the + success handlers: the new schema (Api-Revision: 2026-05-20) emits + ``interaction.completed`` carrying the full interaction object, the + legacy schema (2026-05-07) emits a chunk with ``status="completed"`` + and usage on the chunk itself. Build the equivalent non-streaming + response so cost calculation and spend tracking see one shape. + """ + if result.event_type == "interaction.completed" and result.interaction is not None: + return InteractionsAPIResponse(**result.interaction) + if result.status == "completed": + return InteractionsAPIResponse( + **result.model_dump(exclude={"event_type", "delta", "index", "step", "interaction_id", "interaction"}) + ) + return None + def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse: """ Handles logging for Anthropic messages responses. @@ -4687,6 +4717,8 @@ class StandardLoggingPayloadSetup: elif isinstance(usage, dict): if ResponseAPILoggingUtils._is_response_api_usage(usage): return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage): + return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage) return Usage(**usage) raise ValueError(f"usage is required, got={usage} of type {type(usage)}") @@ -4713,6 +4745,8 @@ class StandardLoggingPayloadSetup: if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump() + if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw): + return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump() return _raw if isinstance(_raw, Usage): return _raw.model_dump() diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index 1c6adbec174..3f591180b77 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -1,6 +1,7 @@ -from typing import Any, Optional, Union +from typing import Any, Mapping, Optional, Sequence, Union from litellm.types.utils import ( + CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -34,3 +35,109 @@ class TranscriptionUsageObjectTransformation: ), ) return None + + +_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = { + "text": "text_tokens", + "audio": "audio_tokens", + "image": "image_tokens", + "video": "video_tokens", + "document": "text_tokens", +} + + +def _modality_field(entry: Mapping[str, Any]) -> str | None: + return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower()) + + +def _token_count(value: Any) -> int: + return value if isinstance(value, int) else 0 + + +def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]: + fields = {field for entry in entries if (field := _modality_field(entry)) is not None} + return { + field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field) + for field in fields + } + + +def _subtract_cached_from_input( + input_sums: Mapping[str, int], + cached_sums: Mapping[str, int], + total_cached_tokens: int, +) -> Mapping[str, int]: + if cached_sums: + return {field: max(0, tokens - cached_sums.get(field, 0)) for field, tokens in input_sums.items()} + if total_cached_tokens and "text_tokens" in input_sums: + return { + **input_sums, + "text_tokens": max(0, input_sums["text_tokens"] - total_cached_tokens), + } + return input_sums + + +class InteractionsUsageObjectTransformation: + """ + Maps the Google Interactions API usage block (total_input_tokens, + output_tokens_by_modality, ...) into LiteLLM's chat-format ``Usage`` so the + generic cost calculator and spend tracking can bill it. + """ + + @staticmethod + def is_interactions_usage_object(usage_object: Any) -> bool: + if not isinstance(usage_object, dict): + return False + if "prompt_tokens" in usage_object or "input_tokens" in usage_object: + return False + return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object + + @staticmethod + def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage: + input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple( + usage_object.get("tool_use_tokens_by_modality") or () + ) + cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ())) + output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ())) + + total_cached_tokens = _token_count(usage_object.get("total_cached_tokens")) + input_sums = _subtract_cached_from_input( + input_sums=_modality_token_sums(input_entries), + cached_sums=cached_sums, + total_cached_tokens=total_cached_tokens, + ) + + reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count( + usage_object.get("total_thought_tokens") + ) + prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count( + usage_object.get("total_tool_use_tokens") + ) + completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens + total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens) + + prompt_tokens_details = ( + PromptTokensDetailsWrapper( + cached_tokens=total_cached_tokens or None, + **input_sums, + ) + if input_sums or total_cached_tokens + else None + ) + completion_tokens_details = ( + CompletionTokensDetailsWrapper( + reasoning_tokens=reasoning_tokens or None, + **output_sums, + ) + if output_sums or reasoning_tokens + else None + ) + + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=completion_tokens_details, + cache_read_input_tokens=total_cached_tokens or None, + ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py new file mode 100644 index 00000000000..01241cf260d --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py @@ -0,0 +1,121 @@ +from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + InteractionsUsageObjectTransformation, +) +from litellm.types.utils import Usage + +OMNI_VIDEO_USAGE = { + "total_tokens": 18247, + "total_input_tokens": 16, + "input_tokens_by_modality": [{"modality": "text", "tokens": 16}], + "total_cached_tokens": 0, + "total_output_tokens": 17937, + "output_tokens_by_modality": [{"modality": "video", "tokens": 17376}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 294, +} + + +def test_detects_interactions_usage_object(): + assert InteractionsUsageObjectTransformation.is_interactions_usage_object(OMNI_VIDEO_USAGE) is True + + +def test_rejects_chat_and_responses_api_usage_objects(): + chat_usage = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + responses_api_usage = {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30} + assert InteractionsUsageObjectTransformation.is_interactions_usage_object(chat_usage) is False + assert InteractionsUsageObjectTransformation.is_interactions_usage_object(responses_api_usage) is False + assert InteractionsUsageObjectTransformation.is_interactions_usage_object(None) is False + assert InteractionsUsageObjectTransformation.is_interactions_usage_object("usage") is False + + +def test_transforms_real_omni_video_usage_block(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(OMNI_VIDEO_USAGE) + + assert isinstance(usage, Usage) + assert usage.prompt_tokens == 16 + assert usage.completion_tokens == 17937 + 294 + assert usage.total_tokens == 18247 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.text_tokens == 16 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.video_tokens == 17376 + assert usage.completion_tokens_details.reasoning_tokens == 294 + + +def test_transforms_reasoning_tokens_spec_field_name(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 10, + "total_output_tokens": 20, + "total_reasoning_tokens": 5, + } + ) + assert usage.completion_tokens == 25 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 5 + assert usage.total_tokens == 35 + + +def test_cached_tokens_subtracted_from_text_input(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 1000, + "input_tokens_by_modality": [{"modality": "text", "tokens": 1000}], + "total_cached_tokens": 400, + "total_output_tokens": 50, + } + ) + assert usage.prompt_tokens == 1000 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.text_tokens == 600 + assert usage.prompt_tokens_details.cached_tokens == 400 + assert usage._cache_read_input_tokens == 400 + + +def test_cached_tokens_subtracted_per_modality_when_breakdown_present(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 1500, + "input_tokens_by_modality": [ + {"modality": "text", "tokens": 1000}, + {"modality": "audio", "tokens": 500}, + ], + "total_cached_tokens": 300, + "cached_tokens_by_modality": [{"modality": "audio", "tokens": 300}], + "total_output_tokens": 50, + } + ) + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.text_tokens == 1000 + assert usage.prompt_tokens_details.audio_tokens == 200 + assert usage.prompt_tokens_details.cached_tokens == 300 + + +def test_tool_use_tokens_billed_as_input(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 100, + "input_tokens_by_modality": [{"modality": "text", "tokens": 100}], + "total_tool_use_tokens": 40, + "tool_use_tokens_by_modality": [{"modality": "text", "tokens": 40}], + "total_output_tokens": 10, + } + ) + assert usage.prompt_tokens == 140 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.text_tokens == 140 + + +def test_document_modality_folds_into_text(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 80, + "input_tokens_by_modality": [ + {"modality": "text", "tokens": 30}, + {"modality": "document", "tokens": 50}, + ], + "total_output_tokens": 10, + } + ) + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.text_tokens == 80 diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index ade2c677745..3769c656b0c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3773,3 +3773,150 @@ def test_zero_token_video_usage_preserves_duration_seconds(logging_obj): assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0 assert payload["total_tokens"] == 0 assert payload["completion_tokens"] == 0 + + +INTERACTIONS_USAGE_BLOCK = { + "total_tokens": 175, + "total_input_tokens": 100, + "input_tokens_by_modality": [{"modality": "text", "tokens": 100}], + "total_cached_tokens": 0, + "total_output_tokens": 50, + "output_tokens_by_modality": [{"modality": "text", "tokens": 50}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 25, +} + + +def _interactions_logging_obj(stream: bool): + logging_obj = LitellmLogging( + model="gemini-2.5-flash", + messages=[], + stream=stream, + call_type="acreate", + start_time=time.time(), + litellm_call_id="interactions-call-id", + function_id="interactions-fn-id", + ) + logging_obj.update_environment_variables( + litellm_params={}, + optional_params={}, + model="gemini-2.5-flash", + custom_llm_provider="gemini", + input="hi", + ) + return logging_obj + + +def test_interactions_response_is_recognized_for_logging(): + from litellm.types.interactions import InteractionsAPIResponse + + logging_obj = _interactions_logging_obj(stream=False) + response = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="completed") + assert logging_obj._is_recognized_call_type_for_logging(logging_result=response) is True + + +def test_non_streaming_interactions_success_sets_response_cost_and_usage(): + import datetime as dt + + from litellm.types.interactions import InteractionsAPIResponse + + logging_obj = _interactions_logging_obj(stream=False) + response = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + + logging_obj._success_handler_helper_fn( + result=response, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["response_cost"] > 0 + standard_logging_object = logging_obj.model_call_details["standard_logging_object"] + assert standard_logging_object["prompt_tokens"] == 100 + assert standard_logging_object["completion_tokens"] == 75 + assert standard_logging_object["total_tokens"] == 175 + assert standard_logging_object["response_cost"] == logging_obj.model_call_details["response_cost"] + + +def test_assembled_streaming_response_from_completed_interaction_event(): + import datetime as dt + + from litellm.types.interactions import ( + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, + ) + + logging_obj = _interactions_logging_obj(stream=True) + completed_event = InteractionsAPIStreamingResponse( + event_type="interaction.completed", + interaction={ + "id": "interactions/abc", + "model": "gemini-2.5-flash", + "status": "completed", + "steps": [], + "usage": dict(INTERACTIONS_USAGE_BLOCK), + }, + ) + + assembled = logging_obj._get_assembled_streaming_response( + result=completed_event, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + is_async=True, + streaming_chunks=[], + ) + + assert isinstance(assembled, InteractionsAPIResponse) + assert assembled.usage == INTERACTIONS_USAGE_BLOCK + + in_progress_event = InteractionsAPIStreamingResponse(event_type="interaction.in_progress") + assert ( + logging_obj._get_assembled_streaming_response( + result=in_progress_event, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + is_async=True, + streaming_chunks=[], + ) + is None + ) + + +def test_assembled_streaming_response_from_legacy_completed_chunk(): + from litellm.types.interactions import ( + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, + ) + + legacy_chunk = InteractionsAPIStreamingResponse( + event_type="interaction.complete", + id="interactions/legacy", + model="gemini-2.5-flash", + status="completed", + outputs=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + + assembled = LitellmLogging._assemble_completed_interaction_response(legacy_chunk) + + assert isinstance(assembled, InteractionsAPIResponse) + assert assembled.id == "interactions/legacy" + assert assembled.usage == INTERACTIONS_USAGE_BLOCK + + +def test_standard_logging_payload_maps_interactions_usage(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + usage = StandardLoggingPayloadSetup.get_usage_from_response_obj( + response_obj={"usage": dict(INTERACTIONS_USAGE_BLOCK)} + ) + + assert usage.prompt_tokens == 100 + assert usage.completion_tokens == 75 + assert usage.total_tokens == 175 diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 9636db4f4cd..c76439e5ad1 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3479,3 +3479,36 @@ def test_batch_cost_calculator_cache_creation_falls_back_to_input_rate(): ) assert prompt_cost == pytest.approx((1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3e-6) / 2) + + +def test_completion_cost_bills_interactions_api_response(): + from litellm.types.interactions import InteractionsAPIResponse + + model_info = litellm.get_model_info(model="gemini-2.5-flash", custom_llm_provider="gemini") + response = InteractionsAPIResponse( + id="interactions/abc123", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage={ + "total_tokens": 175, + "total_input_tokens": 100, + "input_tokens_by_modality": [{"modality": "text", "tokens": 100}], + "total_cached_tokens": 0, + "total_output_tokens": 50, + "output_tokens_by_modality": [{"modality": "text", "tokens": 50}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 25, + }, + ) + + cost = completion_cost(completion_response=response, custom_llm_provider="gemini") + + reasoning_rate = model_info.get("output_cost_per_reasoning_token") or model_info["output_cost_per_token"] + expected = ( + 100 * model_info["input_cost_per_token"] + + 50 * model_info["output_cost_per_token"] + + 25 * reasoning_rate + ) + assert cost == pytest.approx(expected) + assert cost > 0 From f1a5054a16a503c5e087121abf87c0246aa8b6d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:11:20 -0700 Subject: [PATCH 02/54] fix(interactions): bill only interaction creation, not GET polls --- litellm/litellm_core_utils/litellm_logging.py | 18 ++++++++- litellm/types/utils.py | 6 +++ .../test_litellm_logging.py | 40 +++++++++++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0f6e0ccf9c1..94ee595a169 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1895,7 +1895,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, FineTuningJob) or isinstance(logging_result, LiteLLMBatch) or isinstance(logging_result, ResponsesAPIResponse) - or isinstance(logging_result, InteractionsAPIResponse) + or (isinstance(logging_result, InteractionsAPIResponse) and self._is_interactions_create_call_type()) or isinstance(logging_result, OpenAIFileObject) or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) or isinstance(logging_result, OpenAIModerationResponse) @@ -1913,6 +1913,22 @@ class Logging(LiteLLMLoggingBaseClass): return True return False + def _is_interactions_create_call_type(self) -> bool: + """ + Only interaction creation is billable. GET polls, deletes, and cancels + also return an ``InteractionsAPIResponse`` (with usage once completed), + so recognizing those would write spend on every poll of a background + interaction. The proxy sets ``call_type`` from its route_type + (``create_interaction``/``acreate_interaction``); the SDK sets it from + the decorated function name (``create``/``acreate``). + """ + return self.call_type in ( + CallTypes.create_interaction.value, + CallTypes.acreate_interaction.value, + "create", + "acreate", + ) + def _flush_passthrough_collected_chunks_helper( self, raw_bytes: List[bytes], diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 90ea99ceb23..0fafd4a27fe 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -396,6 +396,12 @@ class CallTypes(str, Enum): vector_store_search = "vector_store_search" avector_store_search = "avector_store_search" + ######################################################### + # Google Interactions API Call Types + ######################################################### + create_interaction = "create_interaction" + acreate_interaction = "acreate_interaction" + ######################################################### # Container Call Types ######################################################### diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 3769c656b0c..2bf9f54d7c9 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3787,12 +3787,12 @@ INTERACTIONS_USAGE_BLOCK = { } -def _interactions_logging_obj(stream: bool): +def _interactions_logging_obj(stream: bool, call_type: str = "acreate"): logging_obj = LitellmLogging( model="gemini-2.5-flash", messages=[], stream=stream, - call_type="acreate", + call_type=call_type, start_time=time.time(), litellm_call_id="interactions-call-id", function_id="interactions-fn-id", @@ -3807,14 +3807,46 @@ def _interactions_logging_obj(stream: bool): return logging_obj -def test_interactions_response_is_recognized_for_logging(): +@pytest.mark.parametrize("call_type", ["create", "acreate", "create_interaction", "acreate_interaction"]) +def test_interactions_response_is_recognized_for_logging(call_type): from litellm.types.interactions import InteractionsAPIResponse - logging_obj = _interactions_logging_obj(stream=False) + logging_obj = _interactions_logging_obj(stream=False, call_type=call_type) response = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="completed") assert logging_obj._is_recognized_call_type_for_logging(logging_result=response) is True +@pytest.mark.parametrize( + "call_type", + ["aget", "get", "aget_interaction", "adelete_interaction", "acancel_interaction"], +) +def test_interactions_get_poll_is_not_billed(call_type): + import datetime as dt + + from litellm.types.interactions import InteractionsAPIResponse + + logging_obj = _interactions_logging_obj(stream=False, call_type=call_type) + response = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + + assert logging_obj._is_recognized_call_type_for_logging(logging_result=response) is False + + logging_obj._success_handler_helper_fn( + result=response, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details.get("response_cost") is None + assert logging_obj.model_call_details.get("standard_logging_object") is None + + def test_non_streaming_interactions_success_sets_response_cost_and_usage(): import datetime as dt diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5629cd6c8d4..59c0532ccbe 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21685,7 +21685,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ From 59d4e52a3d2b5e088af0e894e34bbd0ed5fbfe9a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:05:33 -0700 Subject: [PATCH 03/54] fix(interactions): bill background interactions once completed via cost polling --- litellm/constants.py | 11 ++ .../interactions/background_cost_polling.py | 131 +++++++++++++ litellm/interactions/main.py | 9 + litellm/litellm_core_utils/litellm_logging.py | 27 ++- .../proxy/hooks/proxy_track_cost_callback.py | 12 ++ .../credential_migration.py | 73 ++----- .../test_background_cost_polling.py | 184 ++++++++++++++++++ .../test_litellm_logging.py | 59 +++++- .../hooks/test_proxy_track_cost_callback.py | 43 ++++ tests/test_litellm/test_cost_calculator.py | 29 +++ 10 files changed, 519 insertions(+), 59 deletions(-) create mode 100644 litellm/interactions/background_cost_polling.py create mode 100644 tests/test_litellm/interactions/test_background_cost_polling.py diff --git a/litellm/constants.py b/litellm/constants.py index 715d57e594d..be880b72854 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1457,6 +1457,17 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE = max(1, int(os.getenv("STALE_OBJECT_CLEANUP_BAT # installations with large numbers of stale managed objects). _batch_polling_env = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower() PROXY_BATCH_POLLING_ENABLED = _batch_polling_env == "true" +BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS = float( + os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", 5) +) +BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS = float( + os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", 60) +) +BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS = float( + os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 3600) +) +_background_interaction_cost_polling_env = os.getenv("BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true").lower() +BACKGROUND_INTERACTION_COST_POLLING_ENABLED = _background_interaction_cost_polling_env == "true" PROXY_BUDGET_RESCHEDULER_MAX_TIME = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)) PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10 diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py new file mode 100644 index 00000000000..fefbf8715cb --- /dev/null +++ b/litellm/interactions/background_cost_polling.py @@ -0,0 +1,131 @@ +""" +Cost tracking for background interactions. + +A create request with ``background=true`` returns ``in_progress`` with no +usage block, and GET polls are deliberately never billed (billing them would +double-charge every poll; the GET response also does not echo ``background``, +so a poll cannot be told apart from a re-fetch of an already-billed +interaction). The create call is therefore the only place that can own +billing: it schedules a poll task that fetches the interaction until it +reaches a terminal status and logs the final usage as a single success event +attributed to the original request. +""" + +import asyncio +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Iterator, Optional + +from litellm._logging import verbose_logger +from litellm.constants import ( + BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS, + BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS, + BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS, + BACKGROUND_INTERACTION_COST_POLLING_ENABLED, +) +from litellm.types.interactions import InteractionsAPIResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +_TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete", "budget_exceeded"}) + + +@dataclass(frozen=True, slots=True) +class BackgroundInteractionPollContext: + interaction_id: str + custom_llm_provider: str + logging_obj: "LiteLLMLoggingObj" + api_key: Optional[str] = None + api_base: Optional[str] = None + initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS + max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS + timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS + + +FetchInteraction = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]] + + +async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse: + from litellm.interactions import aget + + return await aget( + interaction_id=context.interaction_id, + custom_llm_provider=context.custom_llm_provider, + **{"api_key": context.api_key, "api_base": context.api_base, "no-log": True}, + ) + + +def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]: + elapsed = 0.0 + interval = initial + while elapsed + interval <= timeout: + yield interval + elapsed += interval + interval = min(interval * 2, maximum) + + +async def poll_and_log_background_interaction_cost( + context: BackgroundInteractionPollContext, + fetch_interaction: FetchInteraction = _fetch_interaction, +) -> None: + for interval in _poll_intervals( + initial=context.initial_interval_seconds, + maximum=context.max_interval_seconds, + timeout=context.timeout_seconds, + ): + await asyncio.sleep(interval) + try: + response = await fetch_interaction(context) + except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop + verbose_logger.debug( + "Background interaction cost poll for %s failed, will retry: %s", + context.interaction_id, + e, + ) + continue + if response.status not in _TERMINAL_STATUSES: + continue + if response.usage is not None: + await context.logging_obj.async_log_background_interaction_completion(result=response) + return + verbose_logger.warning( + "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked", + context.interaction_id, + context.timeout_seconds, + ) + + +_ACTIVE_POLL_TASKS: set["asyncio.Task[None]"] = set() # mutable-ok: asyncio requires strong refs to running tasks + + +def maybe_schedule_background_interaction_cost_polling( + response: Any, + create_kwargs: dict[str, Any], + custom_llm_provider: str, +) -> Optional["asyncio.Task[None]"]: + from litellm.litellm_core_utils.litellm_logging import Logging + + if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED: + return None + if not isinstance(response, InteractionsAPIResponse): + return None + if response.status != "in_progress" or not response.id: + return None + logging_obj = create_kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return None + try: + asyncio.get_running_loop() + except RuntimeError: + return None + context = BackgroundInteractionPollContext( + interaction_id=response.id, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + api_key=create_kwargs.get("api_key"), + api_base=create_kwargs.get("api_base"), + ) + task = asyncio.create_task(poll_and_log_background_interaction_cost(context)) + _ACTIVE_POLL_TASKS.add(task) + task.add_done_callback(_ACTIVE_POLL_TASKS.discard) + return task diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index 8634269ee94..5a0ea9280b2 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -39,6 +39,9 @@ from typing import Any, AsyncIterator, Coroutine, Dict, Iterator, List, Optional import httpx import litellm +from litellm.interactions.background_cost_polling import ( + maybe_schedule_background_interaction_cost_polling, +) from litellm.interactions.http_handler import interactions_http_handler from litellm.interactions.utils import ( InteractionsAPIRequestUtils, @@ -170,6 +173,12 @@ async def acreate( else: response = init_response + maybe_schedule_background_interaction_cost_polling( + response=response, + create_kwargs=kwargs, + custom_llm_provider=custom_llm_provider, + ) + return response # type: ignore except Exception as e: raise litellm.exception_type( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 94ee595a169..ac6334bea76 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1895,7 +1895,11 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, FineTuningJob) or isinstance(logging_result, LiteLLMBatch) or isinstance(logging_result, ResponsesAPIResponse) - or (isinstance(logging_result, InteractionsAPIResponse) and self._is_interactions_create_call_type()) + or ( + isinstance(logging_result, InteractionsAPIResponse) + and logging_result.usage is not None + and self._is_interactions_create_call_type() + ) or isinstance(logging_result, OpenAIFileObject) or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) or isinstance(logging_result, OpenAIModerationResponse) @@ -1921,6 +1925,13 @@ class Logging(LiteLLMLoggingBaseClass): interaction. The proxy sets ``call_type`` from its route_type (``create_interaction``/``acreate_interaction``); the SDK sets it from the decorated function name (``create``/``acreate``). + + Recognition additionally requires a usage block (checked at the call + site): a ``background=true`` create returns ``in_progress`` without + usage, and billing it would write a $0 spend log under the interaction + id that collides with the row the background poll task writes once the + interaction completes (see + ``litellm.interactions.background_cost_polling``). """ return self.call_type in ( CallTypes.create_interaction.value, @@ -1929,6 +1940,20 @@ class Logging(LiteLLMLoggingBaseClass): "acreate", ) + async def async_log_background_interaction_completion( + self, + result: InteractionsAPIResponse, + ) -> None: + """ + Log the terminal result of a background interaction as a fresh success + event. The create request already ran success logging for its + ``in_progress`` response (no usage, so no cost was tracked); clearing + the dedup flag lets the completed result flow through cost calculation + and spend tracking exactly once, spanning create to completion. + """ + self.model_call_details.pop("has_logged_async_success", None) + await self.async_success_handler(result=result) + def _flush_passthrough_collected_chunks_helper( self, raw_bytes: List[bytes], diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index b839426fcda..6359eae0fa4 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -279,6 +279,12 @@ class _ProxyDBLogger(CustomLogger): await _release_budget_reservation(budget_reservation=budget_reservation) else: await _release_budget_reservation(budget_reservation=budget_reservation) + if _is_unbilled_in_progress_interaction(completion_response): + verbose_proxy_logger.debug( + "Cost tracking deferred for in-progress background interaction; " + "a poll task logs the final usage once it completes" + ) + return # Non-model call types (health checks, afile_delete) have no model or standard_logging_object. # Use .get() for "stream" to avoid KeyError on health checks. # WS session wrappers (_aresponses_websocket, _arealtime) also reach here with @@ -418,6 +424,12 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: bucket[key] = value +def _is_unbilled_in_progress_interaction(completion_response: Any) -> bool: + from litellm.types.interactions import InteractionsAPIResponse + + return isinstance(completion_response, InteractionsAPIResponse) and completion_response.usage is None + + def _should_track_cost_callback( user_api_key: Optional[str], user_id: Optional[str], diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 4d51295f8dc..6f79a39c883 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -130,9 +130,7 @@ def classify_value(value: object, key: str = "scan") -> ValueClass: return "plaintext" if value.startswith(_V2_GCM_PREFIX): return "migrated" - decrypted = decrypt_value_helper( - value=value, key=key, exception_type="debug", return_original_value=False - ) + decrypted = decrypt_value_helper(value=value, key=key, exception_type="debug", return_original_value=False) if decrypted is None: # Did not decrypt under nacl and has no v2 marker: legacy plaintext. return "plaintext" @@ -151,9 +149,7 @@ def reencrypt_value(value: object, key: str = "migrate") -> object: return value if value.startswith(_V2_GCM_PREFIX): return value # idempotent: already migrated - decrypted = decrypt_value_helper( - value=value, key=key, exception_type="debug", return_original_value=False - ) + decrypted = decrypt_value_helper(value=value, key=key, exception_type="debug", return_original_value=False) if decrypted is None: # Either legacy plaintext (no ciphertext to migrate) or corrupt. Either # way, do not overwrite — preserve the value as stored. @@ -161,9 +157,7 @@ def reencrypt_value(value: object, key: str = "migrate") -> object: return encrypt_value_helper(decrypted) -def reencrypt_selective_dict( - data: dict[str, object], sensitive_keys: list[str] -) -> dict[str, object]: +def reencrypt_selective_dict(data: dict[str, object], sensitive_keys: list[str]) -> dict[str, object]: """Return a copy of ``data`` with only ``sensitive_keys`` re-encrypted. Non-sensitive fields (e.g. ``base_url``, ``connection_id``) are left as-is. @@ -212,9 +206,7 @@ async def _migrate_config_settings_row( dict with selected sensitive fields (vantage_settings / cloudzero_settings). """ report = LocationReport(location=param_name) - record = await prisma_client.db.litellm_config.find_unique( - where={"param_name": param_name} - ) + record = await prisma_client.db.litellm_config.find_unique(where={"param_name": param_name}) if record is None or record.param_value is None: return report @@ -266,9 +258,7 @@ async def _migrate_sso_config(prisma_client: object, dry_run: bool) -> LocationR every present string field. """ report = LocationReport(location="sso_config") - record = await prisma_client.db.litellm_ssoconfig.find_unique( - where={"id": "sso_config"} - ) + record = await prisma_client.db.litellm_ssoconfig.find_unique(where={"id": "sso_config"}) if record is None or record.sso_settings is None: return report @@ -344,9 +334,7 @@ async def _migrate_callback_vars_table( rows = await table.find_many() for row in rows or []: metadata = getattr(row, "metadata", None) - if not isinstance(metadata, dict) or ( - "logging" not in metadata and "callback_settings" not in metadata - ): + if not isinstance(metadata, dict) or ("logging" not in metadata and "callback_settings" not in metadata): continue # Classify every callback-var value directly (strip the litellm_enc:: @@ -534,9 +522,7 @@ async def _scan_config_env_vars(prisma_client: object) -> LocationReport: """Scan the ``environment_variables`` config row (``param_value`` dict).""" report = LocationReport(location="config_environment_variables") try: - record = await prisma_client.db.litellm_config.find_unique( - where={"param_name": "environment_variables"} - ) + record = await prisma_client.db.litellm_config.find_unique(where={"param_name": "environment_variables"}) except Exception as e: # pragma: no cover - defensive verbose_proxy_logger.debug("scan: config env vars unavailable: %s", str(e)) return report @@ -557,11 +543,7 @@ async def _scan_covered_tables(prisma_client: object) -> list[LocationReport]: """Read-only classification of every rotation-covered table. No writes.""" reports: list[LocationReport] = [] for location, db_attr, json_cols, scalar_cols in _COVERED_TABLE_SPECS: - reports.append( - await _scan_one_table( - prisma_client, location, db_attr, json_cols, scalar_cols - ) - ) + reports.append(await _scan_one_table(prisma_client, location, db_attr, json_cols, scalar_cols)) reports.append(await _scan_config_env_vars(prisma_client)) return reports @@ -575,9 +557,7 @@ _VANTAGE_SENSITIVE = ["api_key", "integration_token"] _CLOUDZERO_SENSITIVE = ["api_key"] -async def _migrate_covered_tables( - prisma_client: object, user_api_key_dict: object -) -> list[LocationReport]: +async def _migrate_covered_tables(prisma_client: object, user_api_key_dict: object) -> list[LocationReport]: """Re-encrypt the tables already covered by ``_rotate_master_key`` (model table, credentials, MCP credential/env tables, config environment_variables) by running that orchestrator in *same-key* mode. With the AES gate on, the @@ -597,8 +577,7 @@ async def _migrate_covered_tables( current_key = _get_salt_key() if current_key is None: raise RuntimeError( - "Cannot migrate covered tables: no salt key / master key is set. " - "Set LITELLM_SALT_KEY before migrating." + "Cannot migrate covered tables: no salt key / master key is set. Set LITELLM_SALT_KEY before migrating." ) await _rotate_master_key( prisma_client=cast("PrismaClient", prisma_client), @@ -648,19 +627,9 @@ async def migrate_encryption( # Net-new walkers (items 3, 4, 11, 12, 13). report.add(await _migrate_callback_vars_table(prisma_client, "team", dry_run)) - report.add( - await _migrate_callback_vars_table(prisma_client, "verification_token", dry_run) - ) - report.add( - await _migrate_config_settings_row( - prisma_client, "vantage_settings", _VANTAGE_SENSITIVE, dry_run - ) - ) - report.add( - await _migrate_config_settings_row( - prisma_client, "cloudzero_settings", _CLOUDZERO_SENSITIVE, dry_run - ) - ) + report.add(await _migrate_callback_vars_table(prisma_client, "verification_token", dry_run)) + report.add(await _migrate_config_settings_row(prisma_client, "vantage_settings", _VANTAGE_SENSITIVE, dry_run)) + report.add(await _migrate_config_settings_row(prisma_client, "cloudzero_settings", _CLOUDZERO_SENSITIVE, dry_run)) report.add(await _migrate_sso_config(prisma_client, dry_run)) return report @@ -683,20 +652,10 @@ async def check_encryption(prisma_client: object) -> MigrationReport: # Net-new walker locations, in dry-run (read-only) mode. report.add(await _migrate_callback_vars_table(prisma_client, "team", dry_run=True)) + report.add(await _migrate_callback_vars_table(prisma_client, "verification_token", dry_run=True)) + report.add(await _migrate_config_settings_row(prisma_client, "vantage_settings", _VANTAGE_SENSITIVE, dry_run=True)) report.add( - await _migrate_callback_vars_table( - prisma_client, "verification_token", dry_run=True - ) - ) - report.add( - await _migrate_config_settings_row( - prisma_client, "vantage_settings", _VANTAGE_SENSITIVE, dry_run=True - ) - ) - report.add( - await _migrate_config_settings_row( - prisma_client, "cloudzero_settings", _CLOUDZERO_SENSITIVE, dry_run=True - ) + await _migrate_config_settings_row(prisma_client, "cloudzero_settings", _CLOUDZERO_SENSITIVE, dry_run=True) ) report.add(await _migrate_sso_config(prisma_client, dry_run=True)) return report diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py new file mode 100644 index 00000000000..aac97826b02 --- /dev/null +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -0,0 +1,184 @@ +import asyncio +import time + +import pytest + +from litellm.interactions.background_cost_polling import ( + BackgroundInteractionPollContext, + maybe_schedule_background_interaction_cost_polling, + poll_and_log_background_interaction_cost, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.types.interactions import InteractionsAPIResponse + +USAGE_BLOCK = { + "total_tokens": 175, + "total_input_tokens": 100, + "input_tokens_by_modality": [{"modality": "text", "tokens": 100}], + "total_cached_tokens": 0, + "total_output_tokens": 50, + "output_tokens_by_modality": [{"modality": "text", "tokens": 50}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 25, +} + + +def _logging_obj(call_type: str = "acreate_interaction") -> LitellmLogging: + logging_obj = LitellmLogging( + model="gemini-2.5-flash", + messages=[], + stream=False, + call_type=call_type, + start_time=time.time(), + litellm_call_id="bg-interactions-call-id", + function_id="bg-interactions-fn-id", + ) + logging_obj.update_environment_variables( + litellm_params={}, + optional_params={}, + model="gemini-2.5-flash", + custom_llm_provider="gemini", + input="hi", + ) + return logging_obj + + +def _context(logging_obj: LitellmLogging, timeout_seconds: float = 1.0) -> BackgroundInteractionPollContext: + return BackgroundInteractionPollContext( + interaction_id="interactions/bg-abc", + custom_llm_provider="gemini", + logging_obj=logging_obj, + initial_interval_seconds=0.001, + max_interval_seconds=0.002, + timeout_seconds=timeout_seconds, + ) + + +def _response(status: str, with_usage: bool) -> InteractionsAPIResponse: + return InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-2.5-flash", + status=status, + steps=[], + usage=dict(USAGE_BLOCK) if with_usage else None, + ) + + +def _fetch_sequence(*responses): + remaining = list(responses) + calls = [] + + async def fetch(context): + calls.append(context.interaction_id) + item = remaining.pop(0) if len(remaining) > 1 else remaining[0] + if isinstance(item, Exception): + raise item + return item + + return fetch, calls + + +@pytest.mark.asyncio +async def test_poller_bills_once_when_interaction_completes(): + logging_obj = _logging_obj() + fetch, calls = _fetch_sequence( + _response("in_progress", with_usage=False), + _response("completed", with_usage=True), + ) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert len(calls) == 2 + assert logging_obj.model_call_details["response_cost"] > 0 + assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + + +@pytest.mark.asyncio +async def test_poller_stops_without_billing_on_terminal_status_without_usage(): + logging_obj = _logging_obj() + fetch, calls = _fetch_sequence(_response("failed", with_usage=False)) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert len(calls) == 1 + assert logging_obj.model_call_details.get("response_cost") is None + + +@pytest.mark.asyncio +async def test_poller_gives_up_after_timeout_without_billing(): + logging_obj = _logging_obj() + fetch, calls = _fetch_sequence(_response("in_progress", with_usage=False)) + + await poll_and_log_background_interaction_cost( + _context(logging_obj, timeout_seconds=0.01), + fetch_interaction=fetch, + ) + + assert len(calls) >= 2 + assert logging_obj.model_call_details.get("response_cost") is None + + +@pytest.mark.asyncio +async def test_poller_retries_after_fetch_error_and_still_bills(): + logging_obj = _logging_obj() + fetch, calls = _fetch_sequence( + RuntimeError("transient network error"), + _response("completed", with_usage=True), + ) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert len(calls) == 2 + assert logging_obj.model_call_details["response_cost"] > 0 + + +@pytest.mark.asyncio +async def test_schedule_creates_poll_task_for_in_progress_create(): + logging_obj = _logging_obj() + task = maybe_schedule_background_interaction_cost_polling( + response=_response("in_progress", with_usage=False), + create_kwargs={"litellm_logging_obj": logging_obj}, + custom_llm_provider="gemini", + ) + + assert isinstance(task, asyncio.Task) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response,create_kwargs", + [ + (_response("completed", with_usage=True), {"litellm_logging_obj": "placeholder"}), + (_response("in_progress", with_usage=False), {}), + ("not a response", {"litellm_logging_obj": "placeholder"}), + ], +) +async def test_schedule_skips_non_pollable_results(response, create_kwargs): + if create_kwargs.get("litellm_logging_obj") == "placeholder": + create_kwargs = {"litellm_logging_obj": _logging_obj()} + + task = maybe_schedule_background_interaction_cost_polling( + response=response, + create_kwargs=create_kwargs, + custom_llm_provider="gemini", + ) + + assert task is None + + +@pytest.mark.asyncio +async def test_schedule_respects_kill_switch(monkeypatch): + import litellm.interactions.background_cost_polling as module + + monkeypatch.setattr(module, "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", False) + + task = maybe_schedule_background_interaction_cost_polling( + response=_response("in_progress", with_usage=False), + create_kwargs={"litellm_logging_obj": _logging_obj()}, + custom_llm_provider="gemini", + ) + + assert task is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 2bf9f54d7c9..4d8b19150f7 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3812,10 +3812,67 @@ def test_interactions_response_is_recognized_for_logging(call_type): from litellm.types.interactions import InteractionsAPIResponse logging_obj = _interactions_logging_obj(stream=False, call_type=call_type) - response = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="completed") + response = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) assert logging_obj._is_recognized_call_type_for_logging(logging_result=response) is True +@pytest.mark.parametrize("call_type", ["acreate", "acreate_interaction"]) +def test_in_progress_background_create_is_not_billed(call_type): + import datetime as dt + + from litellm.types.interactions import InteractionsAPIResponse + + logging_obj = _interactions_logging_obj(stream=False, call_type=call_type) + response = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress") + + assert logging_obj._is_recognized_call_type_for_logging(logging_result=response) is False + + logging_obj._success_handler_helper_fn( + result=response, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details.get("response_cost") is None + assert logging_obj.model_call_details.get("standard_logging_object") is None + + +@pytest.mark.asyncio +async def test_background_interaction_completion_rebills_after_in_progress_success(): + import datetime as dt + + from litellm.types.interactions import InteractionsAPIResponse + + logging_obj = _interactions_logging_obj(stream=False) + in_progress = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress") + await logging_obj.async_success_handler( + result=in_progress, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + ) + + assert logging_obj.model_call_details.get("response_cost") is None + assert logging_obj.should_run_logging(event_type="async_success") is False + + completed = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + await logging_obj.async_log_background_interaction_completion(result=completed) + + assert logging_obj.model_call_details["response_cost"] > 0 + assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + + @pytest.mark.parametrize( "call_type", ["aget", "get", "aget_interaction", "adelete_interaction", "acancel_interaction"], diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index f289148101a..02a586a3dcc 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -604,6 +604,49 @@ async def test_track_cost_callback_skips_when_no_standard_logging_object(): mock_proxy_logging.failed_tracking_alert.assert_not_called() +@pytest.mark.asyncio +async def test_track_cost_callback_defers_in_progress_background_interaction(): + """ + A background=true interaction create returns in_progress with no usage + block, so its success event has a model but no standard_logging_object. + The callback must skip quietly (billing happens later via the background + poll task) instead of raising 'Cost tracking failed' and alerting. + """ + from litellm.types.interactions import InteractionsAPIResponse + + logger = _ProxyDBLogger() + + kwargs = { + "call_type": "acreate_interaction", + "model": "gemini/gemini-3-flash-preview", + "litellm_call_id": "test-call-id", + "litellm_params": {}, + "stream": False, + } + in_progress_response = InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-3-flash-preview", + status="in_progress", + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response=in_progress_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + mock_proxy_logging.db_spend_update_writer.update_database.assert_not_called() + mock_proxy_logging.failed_tracking_alert.assert_not_called() + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_propagates_trace_id_from_logging_obj(): """ diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index c76439e5ad1..42fef833944 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3512,3 +3512,32 @@ def test_completion_cost_bills_interactions_api_response(): ) assert cost == pytest.approx(expected) assert cost > 0 + + +def test_completion_cost_bills_interactions_video_output_at_video_rate(): + from litellm.types.interactions import InteractionsAPIResponse + + model_info = litellm.get_model_info(model="gemini-omni-flash-preview", custom_llm_provider="gemini") + video_tokens = 5792 * 8 + response = InteractionsAPIResponse( + id="interactions/video123", + model="gemini-omni-flash-preview", + status="completed", + steps=[], + usage={ + "total_tokens": 10 + video_tokens, + "total_input_tokens": 10, + "input_tokens_by_modality": [{"modality": "text", "tokens": 10}], + "total_cached_tokens": 0, + "total_output_tokens": video_tokens, + "output_tokens_by_modality": [{"modality": "video", "tokens": video_tokens}], + "total_tool_use_tokens": 0, + "total_thought_tokens": 0, + }, + ) + + cost = completion_cost(completion_response=response, custom_llm_provider="gemini") + + expected = 10 * model_info["input_cost_per_token"] + video_tokens * model_info["output_cost_per_video_token"] + assert model_info["output_cost_per_video_token"] != model_info["output_cost_per_token"] + assert cost == pytest.approx(expected) From bbb156577245a9a27ae529276bcaa771e3d06266 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 06:32:16 -0700 Subject: [PATCH 04/54] fix(interactions): hold budget reservation open until background interaction completes --- .../interactions/background_cost_polling.py | 26 ++++++ .../proxy/hooks/proxy_track_cost_callback.py | 14 ++- .../test_background_cost_polling.py | 56 +++++++++++- .../hooks/test_proxy_track_cost_callback.py | 85 +++++++++++++++++++ 4 files changed, 176 insertions(+), 5 deletions(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index fefbf8715cb..846b26f28b9 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -22,6 +22,7 @@ from litellm.constants import ( BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS, BACKGROUND_INTERACTION_COST_POLLING_ENABLED, ) +from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.types.interactions import InteractionsAPIResponse if TYPE_CHECKING: @@ -87,12 +88,37 @@ async def poll_and_log_background_interaction_cost( continue if response.usage is not None: await context.logging_obj.async_log_background_interaction_completion(result=response) + else: + await _release_open_budget_reservation(logging_obj=context.logging_obj) return verbose_logger.warning( "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked", context.interaction_id, context.timeout_seconds, ) + await _release_open_budget_reservation(logging_obj=context.logging_obj) + + +async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None: + """ + The proxy keeps the pre-call budget reservation open for an in-progress + background interaction so concurrent creates cannot stack past the budget. + The completion success event reconciles it to the actual cost; when the + interaction terminates without billable usage (or polling gives up), no + such event fires, so the poller must release the reservation here or the + spend counters stay pinned at the estimated cost. + """ + metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details) + budget_reservation = metadata.get("user_api_key_budget_reservation") + if not isinstance(budget_reservation, dict): + return + + from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation + + try: + await release_budget_reservation(budget_reservation=budget_reservation) + except Exception: # noqa: BLE001 # a failed release must not crash the poll task; counters expire via TTL + verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction") _ACTIVE_POLL_TASKS: set["asyncio.Task[None]"] = set() # mutable-ok: asyncio requires strong refs to running tasks diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 6359eae0fa4..a645bb9929f 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -5,6 +5,7 @@ from typing import Any, List, Optional, Union, cast import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, @@ -278,13 +279,20 @@ class _ProxyDBLogger(CustomLogger): elif budget_reservation is not None: await _release_budget_reservation(budget_reservation=budget_reservation) else: - await _release_budget_reservation(budget_reservation=budget_reservation) if _is_unbilled_in_progress_interaction(completion_response): + if BACKGROUND_INTERACTION_COST_POLLING_ENABLED: + verbose_proxy_logger.debug( + "Cost tracking deferred for in-progress background interaction; " + "the budget reservation stays open until the poll task logs the final usage" + ) + return + await _release_budget_reservation(budget_reservation=budget_reservation) verbose_proxy_logger.debug( - "Cost tracking deferred for in-progress background interaction; " - "a poll task logs the final usage once it completes" + "Background interaction cost polling is disabled; released the budget " + "reservation for an in-progress interaction that will not be billed" ) return + await _release_budget_reservation(budget_reservation=budget_reservation) # Non-model call types (health checks, afile_delete) have no model or standard_logging_object. # Use .get() for "stream" to avoid KeyError on health checks. # WS session wrappers (_aresponses_websocket, _arealtime) also reach here with diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py index aac97826b02..24abeec1053 100644 --- a/tests/test_litellm/interactions/test_background_cost_polling.py +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -1,5 +1,6 @@ import asyncio import time +from typing import Optional import pytest @@ -23,7 +24,10 @@ USAGE_BLOCK = { } -def _logging_obj(call_type: str = "acreate_interaction") -> LitellmLogging: +def _logging_obj( + call_type: str = "acreate_interaction", + litellm_params: Optional[dict] = None, +) -> LitellmLogging: logging_obj = LitellmLogging( model="gemini-2.5-flash", messages=[], @@ -34,7 +38,7 @@ def _logging_obj(call_type: str = "acreate_interaction") -> LitellmLogging: function_id="bg-interactions-fn-id", ) logging_obj.update_environment_variables( - litellm_params={}, + litellm_params=litellm_params or {}, optional_params={}, model="gemini-2.5-flash", custom_llm_provider="gemini", @@ -43,6 +47,14 @@ def _logging_obj(call_type: str = "acreate_interaction") -> LitellmLogging: return logging_obj +def _reservation() -> dict: + return {"reserved_cost": 0.05, "entries": [], "finalized": False, "input_cost": 0.001} + + +def _logging_obj_with_reservation(reservation: dict) -> LitellmLogging: + return _logging_obj(litellm_params={"metadata": {"user_api_key_budget_reservation": reservation}}) + + def _context(logging_obj: LitellmLogging, timeout_seconds: float = 1.0) -> BackgroundInteractionPollContext: return BackgroundInteractionPollContext( interaction_id="interactions/bg-abc", @@ -118,6 +130,46 @@ async def test_poller_gives_up_after_timeout_without_billing(): assert logging_obj.model_call_details.get("response_cost") is None +@pytest.mark.asyncio +async def test_poller_releases_budget_reservation_when_interaction_ends_without_usage(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + fetch, _ = _fetch_sequence(_response("failed", with_usage=False)) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_poller_releases_budget_reservation_on_timeout_give_up(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + fetch, _ = _fetch_sequence(_response("in_progress", with_usage=False)) + + await poll_and_log_background_interaction_cost( + _context(logging_obj, timeout_seconds=0.01), + fetch_interaction=fetch, + ) + + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_poller_leaves_reservation_reconciliation_to_the_completion_event(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + fetch, _ = _fetch_sequence( + _response("in_progress", with_usage=False), + _response("completed", with_usage=True), + ) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert logging_obj.model_call_details["response_cost"] > 0 + assert reservation["finalized"] is False + + @pytest.mark.asyncio async def test_poller_retries_after_fetch_error_and_still_bills(): logging_obj = _logging_obj() diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 02a586a3dcc..725e7f22f7d 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -647,6 +647,91 @@ async def test_track_cost_callback_defers_in_progress_background_interaction(): mock_proxy_logging.failed_tracking_alert.assert_not_called() +def _in_progress_interaction_kwargs(reservation: dict) -> dict: + return { + "call_type": "acreate_interaction", + "model": "gemini/gemini-3-flash-preview", + "litellm_call_id": "test-call-id", + "litellm_params": {"metadata": {"user_api_key_budget_reservation": reservation}}, + "stream": False, + } + + +@pytest.mark.asyncio +async def test_track_cost_callback_keeps_reservation_open_for_in_progress_background_interaction(): + """ + The pre-call budget reservation must stay open while a background + interaction is in flight, so concurrent creates cannot stack past the + budget; the poll task's completion event reconciles it to the actual cost. + """ + from litellm.types.interactions import InteractionsAPIResponse + + logger = _ProxyDBLogger() + reservation = {"reserved_cost": 0.05, "entries": [], "finalized": False} + in_progress_response = InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-3-flash-preview", + status="in_progress", + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=_in_progress_interaction_kwargs(reservation), + completion_response=in_progress_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert reservation["finalized"] is False + mock_proxy_logging.failed_tracking_alert.assert_not_called() + + +@pytest.mark.asyncio +async def test_track_cost_callback_releases_reservation_for_in_progress_interaction_when_polling_disabled( + monkeypatch, +): + """ + With the poll task kill switch off nothing will ever reconcile the + reservation, so the callback must release it or the spend counters stay + pinned at the estimated cost forever. + """ + import litellm.proxy.hooks.proxy_track_cost_callback as callback_module + from litellm.types.interactions import InteractionsAPIResponse + + monkeypatch.setattr(callback_module, "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", False) + + logger = _ProxyDBLogger() + reservation = {"reserved_cost": 0.05, "entries": [], "finalized": False} + in_progress_response = InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-3-flash-preview", + status="in_progress", + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=_in_progress_interaction_kwargs(reservation), + completion_response=in_progress_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert reservation["finalized"] is True + mock_proxy_logging.failed_tracking_alert.assert_not_called() + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_propagates_trace_id_from_logging_obj(): """ From 194dca7dd0d67ef23795fc13668092af1e2a86ef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:15:50 -0700 Subject: [PATCH 05/54] fix(interactions): settle pending background interaction billing before delete --- .../interactions/background_cost_polling.py | 91 ++++++++++++- litellm/interactions/main.py | 3 + .../test_background_cost_polling.py | 123 ++++++++++++++++++ 3 files changed, 211 insertions(+), 6 deletions(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 846b26f28b9..ed30d2573bd 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -9,6 +9,16 @@ interaction). The create call is therefore the only place that can own billing: it schedules a poll task that fetches the interaction until it reaches a terminal status and logs the final usage as a single success event attributed to the original request. + +Deleting an interaction makes every subsequent poll fail, which would let a +caller retrieve the completed output themselves and then delete it before the +poll task settles, leaving the work unbilled and the budget reservation +refunded at the poll timeout. ``adelete`` therefore settles any pending poll +for the interaction before dispatching the delete: it fetches the current +state with the create's credentials, bills it if it is terminal with usage, +and releases the reservation otherwise. A settlement gate on the create's +logging object makes the poll task and the delete path mutually exclusive, so +the interaction is billed exactly once no matter who settles first. """ import asyncio @@ -65,6 +75,25 @@ def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[ interval = min(interval * 2, maximum) +_SETTLED_KEY = "background_interaction_settled" + + +def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool: + return logging_obj.model_call_details.get(_SETTLED_KEY) is True + + +def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool: + """ + Exactly-once gate between the poll task and the delete-time settlement: + both run on the same event loop and neither awaits between reading and + setting the flag, so whichever claims first owns billing or release. + """ + if _is_settled(logging_obj): + return False + logging_obj.model_call_details[_SETTLED_KEY] = True + return True + + async def poll_and_log_background_interaction_cost( context: BackgroundInteractionPollContext, fetch_interaction: FetchInteraction = _fetch_interaction, @@ -75,6 +104,8 @@ async def poll_and_log_background_interaction_cost( timeout=context.timeout_seconds, ): await asyncio.sleep(interval) + if _is_settled(context.logging_obj): + return try: response = await fetch_interaction(context) except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop @@ -86,11 +117,15 @@ async def poll_and_log_background_interaction_cost( continue if response.status not in _TERMINAL_STATUSES: continue + if not _claim_settlement(context.logging_obj): + return if response.usage is not None: await context.logging_obj.async_log_background_interaction_completion(result=response) else: await _release_open_budget_reservation(logging_obj=context.logging_obj) return + if not _claim_settlement(context.logging_obj): + return verbose_logger.warning( "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked", context.interaction_id, @@ -104,9 +139,10 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> The proxy keeps the pre-call budget reservation open for an in-progress background interaction so concurrent creates cannot stack past the budget. The completion success event reconciles it to the actual cost; when the - interaction terminates without billable usage (or polling gives up), no - such event fires, so the poller must release the reservation here or the - spend counters stay pinned at the estimated cost. + interaction terminates without billable usage (or polling gives up, or it + is deleted before settling), no such event fires, so whoever claims the + settlement must release the reservation here or the spend counters stay + pinned at the estimated cost. """ metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details) budget_reservation = metadata.get("user_api_key_budget_reservation") @@ -121,7 +157,21 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction") -_ACTIVE_POLL_TASKS: set["asyncio.Task[None]"] = set() # mutable-ok: asyncio requires strong refs to running tasks +@dataclass(frozen=True, slots=True) +class _ActiveBackgroundPoll: + task: "asyncio.Task[None]" + context: BackgroundInteractionPollContext + + +_ACTIVE_POLLS: dict[ + str, _ActiveBackgroundPoll +] = {} # mutable-ok: asyncio requires strong refs to running tasks, and delete settlement looks polls up by interaction id + + +def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None: + entry = _ACTIVE_POLLS.get(interaction_id) + if entry is not None and entry.task is task: + del _ACTIVE_POLLS[interaction_id] def maybe_schedule_background_interaction_cost_polling( @@ -152,6 +202,35 @@ def maybe_schedule_background_interaction_cost_polling( api_base=create_kwargs.get("api_base"), ) task = asyncio.create_task(poll_and_log_background_interaction_cost(context)) - _ACTIVE_POLL_TASKS.add(task) - task.add_done_callback(_ACTIVE_POLL_TASKS.discard) + _ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context) + task.add_done_callback( + lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished) + ) return task + + +async def maybe_settle_background_interaction_before_delete( + interaction_id: str, + fetch_interaction: FetchInteraction = _fetch_interaction, +) -> None: + entry = _ACTIVE_POLLS.get(interaction_id) + if entry is None: + return + context = entry.context + try: + response = await fetch_interaction(context) + except Exception as e: # noqa: BLE001 # unfetchable pre-delete state settles by releasing the reservation + verbose_logger.debug( + "Could not fetch background interaction %s before delete, releasing its reservation: %s", + interaction_id, + e, + ) + if _claim_settlement(context.logging_obj): + await _release_open_budget_reservation(logging_obj=context.logging_obj) + return + if not _claim_settlement(context.logging_obj): + return + if response.status in _TERMINAL_STATUSES and response.usage is not None: + await context.logging_obj.async_log_background_interaction_completion(result=response) + return + await _release_open_budget_reservation(logging_obj=context.logging_obj) diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index 5a0ea9280b2..985c60c2cc2 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -41,6 +41,7 @@ import httpx import litellm from litellm.interactions.background_cost_polling import ( maybe_schedule_background_interaction_cost_polling, + maybe_settle_background_interaction_before_delete, ) from litellm.interactions.http_handler import interactions_http_handler from litellm.interactions.utils import ( @@ -474,6 +475,8 @@ async def adelete( loop = asyncio.get_event_loop() kwargs["adelete_interaction"] = True + await maybe_settle_background_interaction_before_delete(interaction_id=interaction_id) + func = partial( delete, interaction_id=interaction_id, diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py index 24abeec1053..64436ebe4a7 100644 --- a/tests/test_litellm/interactions/test_background_cost_polling.py +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -5,8 +5,10 @@ from typing import Optional import pytest from litellm.interactions.background_cost_polling import ( + _SETTLED_KEY, BackgroundInteractionPollContext, maybe_schedule_background_interaction_cost_polling, + maybe_settle_background_interaction_before_delete, poll_and_log_background_interaction_cost, ) from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging @@ -221,6 +223,127 @@ async def test_schedule_skips_non_pollable_results(response, create_kwargs): assert task is None +def _register_poll(logging_obj: LitellmLogging, poll_fetch=None) -> asyncio.Task: + import litellm.interactions.background_cost_polling as bg + + if poll_fetch is None: + poll_fetch, _ = _fetch_sequence(_response("in_progress", with_usage=False)) + context = _context(logging_obj) + task = asyncio.create_task(poll_and_log_background_interaction_cost(context, fetch_interaction=poll_fetch)) + bg._ACTIVE_POLLS[context.interaction_id] = bg._ActiveBackgroundPoll(task=task, context=context) + task.add_done_callback(lambda finished: bg._discard_poll(context.interaction_id, finished)) + return task + + +@pytest.mark.asyncio +async def test_delete_settlement_bills_pending_background_interaction(): + logging_obj = _logging_obj() + task = _register_poll(logging_obj) + fetch, calls = _fetch_sequence(_response("completed", with_usage=True)) + + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=fetch, + ) + + assert len(calls) == 1 + assert logging_obj.model_call_details["response_cost"] > 0 + assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + await asyncio.wait_for(task, timeout=5) + + +@pytest.mark.asyncio +async def test_delete_settlement_releases_reservation_when_still_in_progress(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + task = _register_poll(logging_obj) + fetch, _ = _fetch_sequence(_response("in_progress", with_usage=False)) + + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=fetch, + ) + + assert reservation["finalized"] is True + assert logging_obj.model_call_details.get("response_cost") is None + await asyncio.wait_for(task, timeout=5) + + +@pytest.mark.asyncio +async def test_delete_settlement_releases_reservation_when_prefetch_fails(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + task = _register_poll(logging_obj) + fetch, _ = _fetch_sequence(RuntimeError("interaction already deleted")) + + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=fetch, + ) + + assert reservation["finalized"] is True + assert logging_obj.model_call_details.get("response_cost") is None + await asyncio.wait_for(task, timeout=5) + + +@pytest.mark.asyncio +async def test_delete_settlement_ignores_interactions_without_pending_poll(): + fetch, calls = _fetch_sequence(_response("completed", with_usage=True)) + + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/never-polled", + fetch_interaction=fetch, + ) + + assert calls == [] + + +@pytest.mark.asyncio +async def test_delete_settlement_noop_after_poll_task_finished(): + logging_obj = _logging_obj() + poll_fetch, _ = _fetch_sequence(_response("completed", with_usage=True)) + task = _register_poll(logging_obj, poll_fetch=poll_fetch) + await asyncio.wait_for(task, timeout=5) + assert logging_obj.model_call_details["response_cost"] > 0 + + settle_fetch, settle_calls = _fetch_sequence(_response("completed", with_usage=True)) + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=settle_fetch, + ) + + assert settle_calls == [] + + +@pytest.mark.asyncio +async def test_delete_settlement_does_not_rebill_when_gate_already_claimed(): + logging_obj = _logging_obj() + logging_obj.model_call_details[_SETTLED_KEY] = True + task = _register_poll(logging_obj) + fetch, calls = _fetch_sequence(_response("completed", with_usage=True)) + + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=fetch, + ) + + assert len(calls) == 1 + assert logging_obj.model_call_details.get("response_cost") is None + await asyncio.wait_for(task, timeout=5) + + +@pytest.mark.asyncio +async def test_poller_exits_without_billing_once_settled_elsewhere(): + logging_obj = _logging_obj() + logging_obj.model_call_details[_SETTLED_KEY] = True + fetch, calls = _fetch_sequence(_response("completed", with_usage=True)) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert calls == [] + assert logging_obj.model_call_details.get("response_cost") is None + + @pytest.mark.asyncio async def test_schedule_respects_kill_switch(monkeypatch): import litellm.interactions.background_cost_polling as module From e458aa1230a1e23a06aa6ebff88561f5fa510dda Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:12:49 -0700 Subject: [PATCH 06/54] fix(interactions): bill google_search grounding queries per query --- .../usage_object_transformation.py | 12 ++++++- .../test_usage_object_transformation.py | 29 +++++++++++++++ tests/test_litellm/test_cost_calculator.py | 36 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index 3f591180b77..23fafc4132d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -62,6 +62,14 @@ def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, i } +def _google_search_query_count(usage_object: Mapping[str, Any]) -> int: + return sum( + _token_count(entry.get("count")) + for entry in tuple(usage_object.get("grounding_tool_count") or ()) + if isinstance(entry, Mapping) and entry.get("type") == "google_search" + ) + + def _subtract_cached_from_input( input_sums: Mapping[str, int], cached_sums: Mapping[str, int], @@ -116,12 +124,14 @@ class InteractionsUsageObjectTransformation: completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens) + web_search_requests = _google_search_query_count(usage_object) prompt_tokens_details = ( PromptTokensDetailsWrapper( cached_tokens=total_cached_tokens or None, + web_search_requests=web_search_requests or None, **input_sums, ) - if input_sums or total_cached_tokens + if input_sums or total_cached_tokens or web_search_requests else None ) completion_tokens_details = ( diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py index 01241cf260d..2d8092959ce 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py @@ -106,6 +106,35 @@ def test_tool_use_tokens_billed_as_input(): assert usage.prompt_tokens_details.text_tokens == 140 +def test_google_search_grounding_count_maps_to_web_search_requests(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 103, + "input_tokens_by_modality": [{"modality": "text", "tokens": 103}], + "total_output_tokens": 226, + "total_thought_tokens": 351, + "grounding_tool_count": [ + {"type": "google_search", "count": 3}, + {"type": "url_context", "count": 2}, + ], + } + ) + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.web_search_requests == 3 + + +def test_no_grounding_leaves_web_search_requests_unset(): + usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + { + "total_input_tokens": 10, + "input_tokens_by_modality": [{"modality": "text", "tokens": 10}], + "total_output_tokens": 5, + } + ) + assert usage.prompt_tokens_details is not None + assert getattr(usage.prompt_tokens_details, "web_search_requests", None) is None + + def test_document_modality_folds_into_text(): usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( { diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 42fef833944..b144c614474 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3514,6 +3514,42 @@ def test_completion_cost_bills_interactions_api_response(): assert cost > 0 +def test_completion_cost_bills_interactions_google_search_per_query(): + from litellm.types.interactions import InteractionsAPIResponse + + model_info = litellm.get_model_info(model="gemini-3-flash-preview", custom_llm_provider="gemini") + response = InteractionsAPIResponse( + id="interactions/search123", + model="gemini-3-flash-preview", + status="completed", + steps=[], + usage={ + "total_tokens": 680, + "total_input_tokens": 103, + "input_tokens_by_modality": [{"modality": "text", "tokens": 103}], + "total_cached_tokens": 0, + "total_output_tokens": 226, + "total_tool_use_tokens": 0, + "total_thought_tokens": 351, + "grounding_tool_count": [{"type": "google_search", "count": 3}], + }, + ) + + cost = completion_cost(completion_response=response, custom_llm_provider="gemini") + + per_query_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"] + reasoning_rate = model_info.get("output_cost_per_reasoning_token") or model_info["output_cost_per_token"] + expected = ( + 103 * model_info["input_cost_per_token"] + + 226 * model_info["output_cost_per_token"] + + 351 * reasoning_rate + + 3 * per_query_cost + ) + assert model_info.get("web_search_billing_unit") == "per_query" + assert cost == pytest.approx(expected) + assert cost > 3 * per_query_cost + + def test_completion_cost_bills_interactions_video_output_at_video_rate(): from litellm.types.interactions import InteractionsAPIResponse From abf7dab0c2822edf8c3b2bc78618e62e5e6941f8 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 31 Jul 2026 21:14:18 +0000 Subject: [PATCH 07/54] feat(azure_ai): support entra id / oauth auth on every azure ai foundry route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/images/main.py | 27 ++-- litellm/llms/azure/common_utils.py | 29 +++- litellm/llms/azure_ai/common_utils.py | 47 +++++- .../image_edit/flux2_transformation.py | 18 +-- .../azure_ai/image_edit/mai_transformation.py | 19 +-- .../azure_ai/image_edit/transformation.py | 22 ++- .../document_intelligence/transformation.py | 16 +- litellm/llms/azure_ai/ocr/transformation.py | 10 +- .../llms/azure_ai/rerank/transformation.py | 8 +- .../llms/base_llm/rerank/transformation.py | 2 + litellm/llms/cohere/rerank/transformation.py | 2 + litellm/llms/custom_httpx/llm_http_handler.py | 1 + .../llms/dashscope/rerank/transformation.py | 2 + .../llms/deepinfra/rerank/transformation.py | 2 + .../fireworks_ai/rerank/transformation.py | 2 + .../llms/hosted_vllm/rerank/transformation.py | 2 + .../llms/huggingface/rerank/transformation.py | 2 + .../llms/infinity/rerank/transformation.py | 4 +- litellm/llms/jina_ai/rerank/transformation.py | 2 + .../llms/nvidia_nim/rerank/transformation.py | 2 + .../llms/vertex_ai/rerank/transformation.py | 8 +- litellm/llms/voyage/rerank/transformation.py | 2 + litellm/llms/watsonx/rerank/transformation.py | 2 + litellm/main.py | 4 +- .../llms/azure/test_azure_common_utils.py | 63 +++++++- ...test_azure_ai_image_edit_transformation.py | 33 ++++ .../test_mai_image_edit_transformation.py | 14 ++ .../test_azure_ai_rerank_transformation.py | 24 +++ .../llms/azure_ai/test_azure_ai_entra_auth.py | 153 ++++++++++++++++++ ...ocument_intelligence_ocr_transformation.py | 27 ++++ 30 files changed, 480 insertions(+), 69 deletions(-) create mode 100644 tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py diff --git a/litellm/images/main.py b/litellm/images/main.py index 17ea9aa177b..4c88eb52cd8 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -430,24 +430,31 @@ def image_generation( aimg_generation=aimg_generation, ) elif custom_llm_provider == "azure_ai": - from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + from litellm.llms.azure_ai.common_utils import ( + AzureFoundryModelInfo, + get_azure_ai_auth_headers, + ) api_base = AzureFoundryModelInfo.get_api_base(api_base) api_key = AzureFoundryModelInfo.get_api_key(api_key) if extra_headers is not None: optional_params["extra_headers"] = extra_headers - default_headers = { + caller_set_auth = "api-key" in headers or "Authorization" in headers + auth_headers = ( + headers + if caller_set_auth + else get_azure_ai_auth_headers( + api_key=api_key, + litellm_params=litellm_params_dict, + api_key_header="api-key", + ) + ) + headers = { "Content-Type": "application/json", + **auth_headers, + **headers, } - # Only add api-key header if api_key is not None - # Azure AD authentication will use Authorization header instead - if api_key is not None: - default_headers["api-key"] = api_key - - for k, v in default_headers.items(): - if k not in headers: - headers[k] = v model_response = azure_chat_completions.image_generation( model=model, diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 91f5793e269..85100e595e6 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -2,6 +2,7 @@ import asyncio import hashlib import json import os +from functools import lru_cache from typing import Any, Callable, Dict, Literal, NamedTuple, Optional, Union, cast import httpx @@ -57,6 +58,24 @@ def process_azure_headers(headers: Union[httpx.Headers, dict]) -> dict: return {**llm_response_headers, **openai_headers} +@lru_cache(maxsize=128) +def _cached_entra_id_token_provider( + tenant_id: str, + client_id: str, + client_secret: str, + scope: str, +) -> Callable[[], str]: + """Build (once per credential set) a bearer token provider backed by a `ClientSecretCredential`. + + The credential caches the access token internally and only talks to Entra ID when it is close + to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one + per request. + """ + from azure.identity import ClientSecretCredential, get_bearer_token_provider + + return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope) + + def get_azure_ad_token_from_entra_id( tenant_id: str, client_id: str, @@ -75,8 +94,6 @@ def get_azure_ad_token_from_entra_id( Returns: callable that returns a bearer token. """ - from azure.identity import ClientSecretCredential, get_bearer_token_provider - verbose_logger.debug("Getting Azure AD Token from Entra ID") if tenant_id.startswith("os.environ/"): @@ -102,9 +119,13 @@ def get_azure_ad_token_from_entra_id( ) if _tenant_id is None or _client_id is None or _client_secret is None: raise ValueError("tenant_id, client_id, and client_secret must be provided") - credential = ClientSecretCredential(_tenant_id, _client_id, _client_secret) - token_provider = get_bearer_token_provider(credential, scope) + token_provider = _cached_entra_id_token_provider( + tenant_id=_tenant_id, + client_id=_client_id, + client_secret=_client_secret, + scope=scope, + ) verbose_logger.debug("token_provider %s", token_provider) diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index 9965aa693c3..5dd5f5c78cc 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -1,9 +1,54 @@ +from collections.abc import Mapping from typing import List, Literal, Optional import litellm from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams + +AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"] + + +def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None: + """ + Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment. + + Accepts the same credential set as the `azure` provider: service principal + (`tenant_id` / `client_id` / `client_secret`), a pre-fetched `azure_ad_token`, an OIDC + federated token, username/password, or `DefaultAzureCredential` / managed identity. + """ + from litellm.llms.azure.common_utils import get_azure_ad_token + + params = GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams() + + return get_azure_ad_token(params) + + +def get_azure_ai_auth_headers( + api_key: str | None, + litellm_params: Mapping[str, object] | None = None, + api_key_header: AzureAIApiKeyHeader = "Authorization", + api_key_env_var: str = "AZURE_AI_API_KEY", +) -> dict[str, str]: + """ + Build the auth headers for an Azure AI Foundry route. + + Prefers the API key when one is configured, and otherwise falls back to Entra ID / OAuth, + sending the access token as a bearer token. + """ + if api_key: + return {api_key_header: f"Bearer {api_key}" if api_key_header == "Authorization" else api_key} + + azure_ad_token = get_azure_ai_entra_token(litellm_params=litellm_params) + if azure_ad_token: + return {"Authorization": f"Bearer {azure_ad_token}"} + + raise ValueError( + f"Missing Azure AI credentials - set an API key (`api_key` or {api_key_env_var}), or Entra ID / OAuth " + "credentials (`tenant_id` + `client_id` + `client_secret`, `azure_ad_token`, an OIDC token, or a managed " + "identity with `litellm.enable_azure_ad_token_refresh = True`)" + ) class AzureFoundryModelInfo(BaseLLMModelInfo): @@ -43,7 +88,7 @@ class AzureFoundryModelInfo(BaseLLMModelInfo): @staticmethod def get_api_key(api_key: Optional[str] = None) -> Optional[str]: - return api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_AI_API_KEY") + return api_key or litellm.api_key or get_secret_str("AZURE_AI_API_KEY") @property def api_version(self, api_version: Optional[str] = None) -> Optional[str]: diff --git a/litellm/llms/azure_ai/image_edit/flux2_transformation.py b/litellm/llms/azure_ai/image_edit/flux2_transformation.py index 1bc3bdcddc1..db429b85082 100644 --- a/litellm/llms/azure_ai/image_edit/flux2_transformation.py +++ b/litellm/llms/azure_ai/image_edit/flux2_transformation.py @@ -5,7 +5,10 @@ from typing import Any, Dict, Optional, Tuple from httpx._types import RequestFiles import litellm -from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo +from litellm.llms.azure_ai.common_utils import ( + AzureFoundryModelInfo, + get_azure_ai_auth_headers, +) from litellm.llms.azure_ai.image_generation.flux_transformation import ( AzureFoundryFluxImageGenerationConfig, ) @@ -71,16 +74,13 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig): """ Validate Azure AI Foundry environment and set up authentication """ - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - if not api_key: - raise ValueError( - f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter." - ) - headers.update( { - "Api-Key": api_key, + **get_azure_ai_auth_headers( + api_key=AzureFoundryModelInfo.get_api_key(api_key), + litellm_params=litellm_params, + api_key_header="Api-Key", + ), "Content-Type": "application/json", } ) diff --git a/litellm/llms/azure_ai/image_edit/mai_transformation.py b/litellm/llms/azure_ai/image_edit/mai_transformation.py index aa1092b0a53..fdac9912193 100644 --- a/litellm/llms/azure_ai/image_edit/mai_transformation.py +++ b/litellm/llms/azure_ai/image_edit/mai_transformation.py @@ -3,7 +3,10 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast import httpx from httpx._types import RequestFiles -from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo +from litellm.llms.azure_ai.common_utils import ( + AzureFoundryModelInfo, + get_azure_ai_auth_headers, +) from litellm.llms.azure_ai.image_generation.mai_transformation import ( AzureFoundryMAIImageGenerationConfig, ) @@ -91,15 +94,13 @@ class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig): litellm_params: Optional[dict] = None, api_base: Optional[str] = None, ) -> dict: - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - if not api_key: - raise ValueError( - f"Azure AI API key is required for model {model}. " - "Set AZURE_AI_API_KEY environment variable or pass api_key parameter." + headers.update( + get_azure_ai_auth_headers( + api_key=AzureFoundryModelInfo.get_api_key(api_key), + litellm_params=litellm_params, + api_key_header="api-key", ) - - headers.update({"api-key": api_key}) + ) return headers def get_complete_url( diff --git a/litellm/llms/azure_ai/image_edit/transformation.py b/litellm/llms/azure_ai/image_edit/transformation.py index 5393a0ba55f..22b0b169faf 100644 --- a/litellm/llms/azure_ai/image_edit/transformation.py +++ b/litellm/llms/azure_ai/image_edit/transformation.py @@ -3,7 +3,10 @@ from typing import Optional import httpx import litellm -from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo +from litellm.llms.azure_ai.common_utils import ( + AzureFoundryModelInfo, + get_azure_ai_auth_headers, +) from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.secret_managers.main import get_secret_str from litellm.utils import _add_path_to_api_base @@ -30,19 +33,14 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig): ) -> dict: """ Validate Azure AI Foundry environment and set up authentication - Uses Api-Key header format + Uses the Api-Key header format, or an Entra ID / OAuth bearer token when no key is set """ - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - if not api_key: - raise ValueError( - f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter." - ) - headers.update( - { - "Api-Key": api_key, # Azure AI Foundry uses Api-Key header format - } + get_azure_ai_auth_headers( + api_key=AzureFoundryModelInfo.get_api_key(api_key), + litellm_params=litellm_params, + api_key_header="Api-Key", + ) ) return headers diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py index 7d915892a28..4db4472dfc2 100644 --- a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py +++ b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py @@ -25,6 +25,7 @@ from litellm.constants import ( AZURE_OPERATION_POLLING_TIMEOUT, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers from litellm.llms.base_llm.ocr.transformation import ( BaseOCRConfig, DocumentType, @@ -215,17 +216,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): """ Validate environment and return headers for Azure Document Intelligence. - Authentication uses Ocp-Apim-Subscription-Key header. + Authentication uses the Ocp-Apim-Subscription-Key header, or an Entra ID / OAuth bearer + token when no subscription key is set. """ # Get API key from environment if not provided if api_key is None: api_key = get_secret_str(AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR) - if api_key is None: - raise ValueError( - "Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter" - ) - # Validate API base/endpoint is provided if api_base is None: api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") @@ -236,7 +233,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): ) headers = { - "Ocp-Apim-Subscription-Key": api_key, + **get_azure_ai_auth_headers( + api_key=api_key, + litellm_params=litellm_params, + api_key_header="Ocp-Apim-Subscription-Key", + api_key_env_var=AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR, + ), "Content-Type": "application/json", **headers, } diff --git a/litellm/llms/azure_ai/ocr/transformation.py b/litellm/llms/azure_ai/ocr/transformation.py index a57e3e869cf..abc23008f6a 100644 --- a/litellm/llms/azure_ai/ocr/transformation.py +++ b/litellm/llms/azure_ai/ocr/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, convert_url_to_base64, ) +from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData from litellm.llms.mistral.ocr.transformation import MistralOCRConfig from litellm.secret_managers.main import get_secret_str @@ -47,17 +48,12 @@ class AzureAIOCRConfig(MistralOCRConfig): """ Validate environment and return headers for Azure AI OCR. - Azure AI uses Bearer token authentication with AZURE_AI_API_KEY. + Authenticates with AZURE_AI_API_KEY, or with an Entra ID / OAuth token when no key is set. """ # Get API key from environment if not provided if api_key is None: api_key = get_secret_str(AZURE_AI_OCR_API_KEY_ENV_VAR) - if api_key is None: - raise ValueError( - "Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params" - ) - # Validate API base is provided if api_base is None: api_base = get_secret_str("AZURE_AI_API_BASE") @@ -68,7 +64,7 @@ class AzureAIOCRConfig(MistralOCRConfig): ) headers = { - "Authorization": f"Bearer {api_key}", + **get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params), "Content-Type": "application/json", **headers, } diff --git a/litellm/llms/azure_ai/rerank/transformation.py b/litellm/llms/azure_ai/rerank/transformation.py index 928f53bd485..24cdc67a23b 100644 --- a/litellm/llms/azure_ai/rerank/transformation.py +++ b/litellm/llms/azure_ai/rerank/transformation.py @@ -2,12 +2,14 @@ Translate between Cohere's `/rerank` format and Azure AI's `/rerank` format. """ +from collections.abc import Mapping from typing import Optional import httpx import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers from litellm.llms.cohere.rerank.transformation import CohereRerankConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import RerankResponse @@ -64,15 +66,13 @@ class AzureAIRerankConfig(CohereRerankConfig): model: str, api_key: Optional[str] = None, optional_params: Optional[dict] = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: if api_key is None: api_key = get_secret_str("AZURE_AI_API_KEY") or litellm.azure_key - if api_key is None: - raise ValueError("Azure AI API key is required. Please set 'AZURE_AI_API_KEY' or 'litellm.azure_key'") - default_headers = { - "Authorization": f"Bearer {api_key}", + **get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params), "accept": "application/json", "content-type": "application/json", } diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index eac44ba85c5..e9f210fb31c 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union import httpx @@ -24,6 +25,7 @@ class BaseRerankConfig(ABC): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: pass diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index e494e89fbf2..86d9a3d224d 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Dict, List, Union import httpx @@ -81,6 +82,7 @@ class CohereRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: if api_key is None: api_key = get_secret_str("COHERE_API_KEY") or get_secret_str("CO_API_KEY") or litellm.cohere_key diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..e41fbea94d6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1063,6 +1063,7 @@ class BaseLLMHTTPHandler: headers=headers or {}, model=model, optional_params=optional_rerank_params, + litellm_params=litellm_params, ) api_base = provider_config.get_complete_url( diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 365e15fdd7a..b8c369b892c 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -22,6 +22,7 @@ as supported only for gte-rerank-v2 / qwen3-vl-rerank. Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api """ +from collections.abc import Mapping from typing import Any, Dict, List, Union import httpx @@ -85,6 +86,7 @@ class DashScopeRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: if api_key is None: api_key = get_secret_str("DASHSCOPE_API_KEY") diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index 82069e4e195..87a6ecc7120 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -2,6 +2,7 @@ Translate between Cohere's `/rerank` format and Deepinfra's `/rerank` format. """ +from collections.abc import Mapping from typing import Any, Dict, List, Union import httpx @@ -67,6 +68,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: if api_key is None: api_key = get_secret_str("DEEPINFRA_API_KEY") diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index 393a6c5a8e5..e727f5c1d2b 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -4,6 +4,7 @@ Fireworks AI Rerank API transformation Reference: https://docs.fireworks.ai/inference-api-reference/rerank """ +from collections.abc import Mapping from typing import Any, Dict, List, Union import httpx @@ -104,6 +105,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: api_key = self._get_api_key(api_key) if api_key is None: diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 77504eba04a..cd35cc72492 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -2,6 +2,7 @@ Transformation logic for Hosted VLLM rerank """ +from collections.abc import Mapping from typing import Any, Dict, List, Union import httpx @@ -107,6 +108,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: if api_key is None: api_key = get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key" diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index cdad77a9815..245551cf4f2 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -1,4 +1,5 @@ import os +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union import httpx @@ -125,6 +126,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): api_key: str | None = None, optional_params: dict | None = None, api_base: str | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: # Get API credentials api_key, api_base = self.get_api_credentials(api_key=api_key, api_base=api_base) diff --git a/litellm/llms/infinity/rerank/transformation.py b/litellm/llms/infinity/rerank/transformation.py index 94746da4609..7451b06c01a 100644 --- a/litellm/llms/infinity/rerank/transformation.py +++ b/litellm/llms/infinity/rerank/transformation.py @@ -4,12 +4,13 @@ Transformation logic from Cohere's /v1/rerank format to Infinity's `/v1/rerank` Why separate file? Make it easy to see how transformation works """ -from litellm._uuid import uuid +from collections.abc import Mapping from typing import List, Optional import httpx import litellm +from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.cohere.rerank.transformation import CohereRerankConfig from litellm.secret_managers.main import get_secret_str @@ -46,6 +47,7 @@ class InfinityRerankConfig(CohereRerankConfig): model: str, api_key: Optional[str] = None, optional_params: Optional[dict] = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: if api_key is None: api_key = get_secret_str("INFINITY_API_KEY") or get_secret_str("INFINITY_API_KEY") or litellm.infinity_key diff --git a/litellm/llms/jina_ai/rerank/transformation.py b/litellm/llms/jina_ai/rerank/transformation.py index 7f4c0709bdd..903e629803b 100644 --- a/litellm/llms/jina_ai/rerank/transformation.py +++ b/litellm/llms/jina_ai/rerank/transformation.py @@ -6,6 +6,7 @@ Why separate file? Make it easy to see how transformation works Docs - https://jina.ai/reranker """ +from collections.abc import Mapping from typing import Any, Dict, List, Tuple, Union from httpx import URL, Response @@ -139,6 +140,7 @@ class JinaAIRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> Dict: if api_key is None: raise ValueError("api_key is required. Set via `api_key` parameter or `JINA_API_KEY` environment variable.") diff --git a/litellm/llms/nvidia_nim/rerank/transformation.py b/litellm/llms/nvidia_nim/rerank/transformation.py index 2d72d52f991..07b792468c9 100644 --- a/litellm/llms/nvidia_nim/rerank/transformation.py +++ b/litellm/llms/nvidia_nim/rerank/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Dict, List, Literal, Union import httpx @@ -148,6 +149,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: """ Validate that the Nvidia NIM API key is present. diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index b9680af20cc..055a02aa40d 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -4,6 +4,7 @@ Translates from Cohere's `/v1/rerank` input format to Vertex AI Discovery Engine Why separate file? Make it easy to see how transformation works """ +from collections.abc import Mapping from typing import Any, Dict, List, Union import httpx @@ -74,14 +75,15 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): model: str, api_key: str | None = None, optional_params: Dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> dict: """ Validate and set up authentication for Vertex AI Discovery Engine API """ # Get credentials and project info from optional_params (which contains vertex_credentials, etc.) - litellm_params = optional_params.copy() if optional_params else {} - vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params) - vertex_project = self.safe_get_vertex_ai_project(litellm_params) + vertex_params = optional_params.copy() if optional_params else {} + vertex_credentials = self.safe_get_vertex_ai_credentials(vertex_params) + vertex_project = self.safe_get_vertex_ai_project(vertex_params) # Get access token using the base class method access_token, project_id = self._ensure_access_token( diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index e426e39962b..df9f32dd96d 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -4,6 +4,7 @@ Transformation logic for Voyage AI's /v1/rerank endpoint. Docs - https://docs.voyageai.com/docs/reranker """ +from collections.abc import Mapping from typing import Any, Dict, List, Tuple, Union import httpx @@ -137,6 +138,7 @@ class VoyageRerankConfig(BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> Dict: if api_key is None: api_key = get_secret_str("VOYAGE_API_KEY") or get_secret_str("VOYAGE_AI_API_KEY") diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 25b593f1c0a..549ccca4748 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -5,6 +5,7 @@ Docs - https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank """ import uuid +from collections.abc import Mapping from typing import Any, Dict, List, Union, cast import httpx @@ -60,6 +61,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): model: str, api_key: str | None = None, optional_params: dict | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> Dict: optional_params = optional_params or {} diff --git a/litellm/main.py b/litellm/main.py index acdec7385da..b167a257d18 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6739,6 +6739,8 @@ def embedding( aembedding=aembedding, ) elif custom_llm_provider == "azure_ai": + from litellm.llms.azure_ai.common_utils import get_azure_ai_entra_token + api_base = ( api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there or litellm.api_base @@ -6748,8 +6750,8 @@ def embedding( api_key = ( api_key or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key or get_secret_str("AZURE_AI_API_KEY") + or get_azure_ai_entra_token(litellm_params=litellm_params_dict) ) ## EMBEDDING CALL diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index a3280b90fe3..450920f1f44 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -11,7 +11,12 @@ sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path import litellm -from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token +from litellm.llms.azure.common_utils import ( + BaseAzureLLM, + _cached_entra_id_token_provider, + get_azure_ad_token, + get_azure_ad_token_from_entra_id, +) from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, ) @@ -2034,3 +2039,59 @@ def test_azure_traditional_api_uses_azure_openai_client(): assert isinstance( async_client, AsyncAzureOpenAI ), f"Expected AsyncAzureOpenAI client for api_version={api_version}" + + +class TestEntraIdTokenProviderCache: + def setup_method(self): + _cached_entra_id_token_provider.cache_clear() + + def teardown_method(self): + _cached_entra_id_token_provider.cache_clear() + + def test_reuses_credential_for_the_same_service_principal(self): + with ( + patch("azure.identity.ClientSecretCredential") as mock_credential, + patch("azure.identity.get_bearer_token_provider", side_effect=lambda credential, scope: lambda: "token"), + ): + first = get_azure_ad_token_from_entra_id( + tenant_id="tenant", + client_id="client", + client_secret="secret", + scope="https://cognitiveservices.azure.com/.default", + ) + second = get_azure_ad_token_from_entra_id( + tenant_id="tenant", + client_id="client", + client_secret="secret", + scope="https://cognitiveservices.azure.com/.default", + ) + + assert first is second + assert mock_credential.call_count == 1 + + @pytest.mark.parametrize( + "second_call_kwargs", + [ + {"tenant_id": "other-tenant"}, + {"client_id": "other-client"}, + {"client_secret": "other-secret"}, + {"scope": "https://ai.azure.com/.default"}, + ], + ) + def test_does_not_share_a_provider_across_credentials_or_scopes(self, second_call_kwargs): + base_kwargs = { + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + "scope": "https://cognitiveservices.azure.com/.default", + } + + with ( + patch("azure.identity.ClientSecretCredential") as mock_credential, + patch("azure.identity.get_bearer_token_provider", side_effect=lambda credential, scope: lambda: "token"), + ): + first = get_azure_ad_token_from_entra_id(**base_kwargs) + second = get_azure_ad_token_from_entra_id(**{**base_kwargs, **second_call_kwargs}) + + assert first is not second + assert mock_credential.call_count == 2 diff --git a/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py b/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py index da1041f3d60..9c9401fa8e9 100644 --- a/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py +++ b/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py @@ -5,6 +5,10 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +import litellm +from litellm.llms.azure_ai.image_edit.flux2_transformation import ( + AzureFoundryFlux2ImageEditConfig, +) from litellm.llms.azure_ai.image_edit.transformation import ( AzureFoundryFluxImageEditConfig, ) @@ -32,3 +36,32 @@ def test_azure_ai_url_generation(): ) expected_url = f"{api_base}/openai/deployments/FLUX.1-Kontext-pro/images/edits?api-version=2025-04-01-preview" assert complete_url == expected_url + + +def test_azure_ai_validate_environment_with_entra_token(monkeypatch): + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_key", None) + config = AzureFoundryFluxImageEditConfig() + + headers = config.validate_environment( + {}, + "FLUX.1-Kontext-pro", + litellm_params={"azure_ad_token": "entra-token"}, + ) + + assert headers == {"Authorization": "Bearer entra-token"} + + +def test_flux2_validate_environment_with_entra_token(monkeypatch): + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_key", None) + config = AzureFoundryFlux2ImageEditConfig() + + headers = config.validate_environment( + {}, + "flux.2-pro", + litellm_params={"azure_ad_token": "entra-token"}, + ) + + assert headers["Authorization"] == "Bearer entra-token" + assert headers["Content-Type"] == "application/json" diff --git a/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py b/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py index d5256be02d7..c4e39a26aeb 100644 --- a/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py +++ b/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py @@ -8,6 +8,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../../..")) +import litellm from litellm.llms.azure_ai.image_edit import ( AzureFoundryMAIImageEditConfig, get_azure_ai_image_edit_config, @@ -169,3 +170,16 @@ class TestAzureMAIImageEdit: assert image_response.data[0].b64_json == "abc123" assert image_response.usage.output_tokens == 1024 assert image_response.usage.total_tokens == 1024 + + +def test_mai_validate_environment_with_entra_token(monkeypatch): + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_key", None) + + headers = AzureFoundryMAIImageEditConfig().validate_environment( + headers={}, + model="MAI-Image-2.5", + litellm_params={"azure_ad_token": "entra-token"}, + ) + + assert headers == {"Authorization": "Bearer entra-token"} diff --git a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py index ffabce6e00c..150f5794ab1 100644 --- a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py +++ b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py @@ -7,6 +7,7 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +import litellm from litellm.llms.azure_ai.rerank.transformation import AzureAIRerankConfig @@ -97,3 +98,26 @@ class TestAzureAIRerankConfigGetCompleteUrl: model=self.model, ) assert url == "https://my-resource.services.ai.azure.com/v1/rerank?r=1" + + +class TestAzureAIRerankConfigValidateEnvironment: + def test_uses_api_key_when_set(self): + headers = AzureAIRerankConfig().validate_environment( + headers={}, + model="azure_ai/cohere-rerank-v3-english", + api_key="my-key", + ) + + assert headers["Authorization"] == "Bearer my-key" + + def test_falls_back_to_entra_token(self, monkeypatch): + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "azure_key", None) + + headers = AzureAIRerankConfig().validate_environment( + headers={}, + model="azure_ai/cohere-rerank-v3-english", + litellm_params={"azure_ad_token": "entra-token"}, + ) + + assert headers["Authorization"] == "Bearer entra-token" diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py new file mode 100644 index 00000000000..8ac37feee4b --- /dev/null +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py @@ -0,0 +1,153 @@ +""" +Entra ID / OAuth auth for Azure AI Foundry routes. + +Every azure_ai route must authenticate with an Entra ID token when no API key is configured, +instead of requiring an API key. +""" + +from unittest.mock import patch + +import pytest + +import litellm +from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers +from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig + +ENTRA_PARAMS = {"azure_ad_token": "entra-token"} + + +@pytest.fixture(autouse=True) +def clear_azure_env(monkeypatch): + for env_var in ( + "AZURE_AI_API_KEY", + "AZURE_API_KEY", + "AZURE_AD_TOKEN", + "AZURE_TENANT_ID", + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_SCOPE", + "OPENAI_API_KEY", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY", + ): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + + +def test_api_key_wins_over_entra_credentials(): + headers = get_azure_ai_auth_headers(api_key="my-key", litellm_params=ENTRA_PARAMS, api_key_header="Api-Key") + + assert headers == {"Api-Key": "my-key"} + + +def test_entra_token_used_when_no_api_key(): + headers = get_azure_ai_auth_headers(api_key=None, litellm_params=ENTRA_PARAMS, api_key_header="Api-Key") + + assert headers == {"Authorization": "Bearer entra-token"} + + +def test_service_principal_token_is_requested_with_the_configured_scope(): + with patch("litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id") as mock_entra_id: + mock_entra_id.return_value = lambda: "sp-token" + + headers = get_azure_ai_auth_headers( + api_key=None, + litellm_params={ + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + "azure_scope": "https://ai.azure.com/.default", + }, + ) + + mock_entra_id.assert_called_once_with( + tenant_id="tenant", + client_id="client", + client_secret="secret", + scope="https://ai.azure.com/.default", + ) + assert headers == {"Authorization": "Bearer sp-token"} + + +def test_error_mentions_both_credential_types_when_nothing_is_configured(): + with pytest.raises(ValueError) as exc_info: + get_azure_ai_auth_headers(api_key=None, litellm_params={}) + + message = str(exc_info.value) + assert "AZURE_AI_API_KEY" in message + assert "client_secret" in message + + +def test_ocr_authenticates_with_entra_token(): + headers = AzureAIOCRConfig().validate_environment( + headers={}, + model="azure_ai/mistral-ocr", + api_base="https://my-resource.services.ai.azure.com", + litellm_params=ENTRA_PARAMS, + ) + + assert headers["Authorization"] == "Bearer entra-token" + + +def test_embedding_falls_back_to_entra_token_instead_of_openai_key(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-openai-key") + + with patch.object(litellm.main.azure_ai_embedding, "embedding") as mock_embedding: + mock_embedding.return_value = litellm.EmbeddingResponse() + + litellm.embedding( + model="azure_ai/cohere-embed-v3-english", + input=["hello"], + api_base="https://my-resource.services.ai.azure.com", + azure_ad_token="entra-token", + ) + + assert mock_embedding.call_args.kwargs["api_key"] == "entra-token" + + +def test_image_generation_authenticates_with_entra_token(): + with patch.object(litellm.images.main.azure_chat_completions, "image_generation") as mock_image_generation: + mock_image_generation.return_value = litellm.ImageResponse() + + litellm.image_generation( + model="azure_ai/FLUX-1.1-pro", + prompt="a red circle", + api_base="https://my-resource.services.ai.azure.com", + azure_ad_token="entra-token", + ) + + headers = mock_image_generation.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer entra-token" + assert "api-key" not in headers + + +def test_image_generation_keeps_caller_supplied_authorization_header(): + with patch.object(litellm.images.main.azure_chat_completions, "image_generation") as mock_image_generation: + mock_image_generation.return_value = litellm.ImageResponse() + + litellm.image_generation( + model="azure_ai/FLUX-1.1-pro", + prompt="a red circle", + api_base="https://my-resource.services.ai.azure.com", + headers={"Authorization": "Bearer caller-token"}, + ) + + headers = mock_image_generation.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer caller-token" + assert "api-key" not in headers + + +def test_image_generation_still_uses_api_key_header(): + with patch.object(litellm.images.main.azure_chat_completions, "image_generation") as mock_image_generation: + mock_image_generation.return_value = litellm.ImageResponse() + + litellm.image_generation( + model="azure_ai/FLUX-1.1-pro", + prompt="a red circle", + api_base="https://my-resource.services.ai.azure.com", + api_key="my-key", + ) + + headers = mock_image_generation.call_args.kwargs["headers"] + assert headers["api-key"] == "my-key" + assert "Authorization" not in headers diff --git a/tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py b/tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py index 39d6f1dc355..b8e11a0bfcb 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py @@ -248,3 +248,30 @@ def test_get_complete_url_combines_pages_and_features(): assert "&pages=1,2,3" in url assert "&features=keyValuePairs,languages" in url + + +def test_validate_environment_uses_subscription_key(monkeypatch): + monkeypatch.delenv("AZURE_DOCUMENT_INTELLIGENCE_API_KEY", raising=False) + + headers = AzureDocumentIntelligenceOCRConfig().validate_environment( + headers={}, + model="prebuilt-layout", + api_key="my-key", + api_base="https://example.cognitiveservices.azure.com", + ) + + assert headers["Ocp-Apim-Subscription-Key"] == "my-key" + + +def test_validate_environment_falls_back_to_entra_token(monkeypatch): + monkeypatch.delenv("AZURE_DOCUMENT_INTELLIGENCE_API_KEY", raising=False) + + headers = AzureDocumentIntelligenceOCRConfig().validate_environment( + headers={}, + model="prebuilt-layout", + api_base="https://example.cognitiveservices.azure.com", + litellm_params={"azure_ad_token": "entra-token"}, + ) + + assert headers["Authorization"] == "Bearer entra-token" + assert "Ocp-Apim-Subscription-Key" not in headers From c5d50817a70373f3443423fa8a6a97980ad156a9 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 31 Jul 2026 22:01:28 +0000 Subject: [PATCH 08/54] fix(azure_ai): detect caller auth headers case-insensitively in image generation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/images/main.py | 3 ++- .../llms/azure_ai/test_azure_ai_entra_auth.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/images/main.py b/litellm/images/main.py index 4c88eb52cd8..3bee6000d3f 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -440,7 +440,8 @@ def image_generation( if extra_headers is not None: optional_params["extra_headers"] = extra_headers - caller_set_auth = "api-key" in headers or "Authorization" in headers + caller_header_names = frozenset(name.lower() for name in headers) + caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names auth_headers = ( headers if caller_set_auth diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py index 8ac37feee4b..1145439a7b4 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py @@ -121,7 +121,8 @@ def test_image_generation_authenticates_with_entra_token(): assert "api-key" not in headers -def test_image_generation_keeps_caller_supplied_authorization_header(): +@pytest.mark.parametrize("header_name", ["Authorization", "authorization", "api-key", "API-KEY"]) +def test_image_generation_keeps_caller_supplied_auth_header(header_name): with patch.object(litellm.images.main.azure_chat_completions, "image_generation") as mock_image_generation: mock_image_generation.return_value = litellm.ImageResponse() @@ -129,12 +130,12 @@ def test_image_generation_keeps_caller_supplied_authorization_header(): model="azure_ai/FLUX-1.1-pro", prompt="a red circle", api_base="https://my-resource.services.ai.azure.com", - headers={"Authorization": "Bearer caller-token"}, + headers={header_name: "caller-credential"}, ) headers = mock_image_generation.call_args.kwargs["headers"] - assert headers["Authorization"] == "Bearer caller-token" - assert "api-key" not in headers + assert headers[header_name] == "caller-credential" + assert len(headers) == 2 def test_image_generation_still_uses_api_key_header(): From e3da917e679ff68aaa86f5ed67dcb6528117f072 Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:29:07 +0000 Subject: [PATCH 09/54] fix(proxy): parse form-encoded video edit/extension bodies after auth Fixes #36487 video_edit, video_extension, and video_remix called request.body() after user_api_key_auth had already parsed multipart/form bodies via _read_request_body(), causing RuntimeError Stream consumed and 500s for OpenAI SDK clients. Use _read_request_body consistently and normalize bare-string or JSON-string video references from form fields into video_id. --- litellm/proxy/video_endpoints/endpoints.py | 28 ++++------- litellm/proxy/video_endpoints/utils.py | 21 +++++++++ .../proxy/video_endpoints/test_endpoints.py | 24 ++++++++++ tests/test_litellm/test_video_generation.py | 47 +++++++++++++++++++ 4 files changed, 100 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 6c6b004fd17..cb014bcceee 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -2,7 +2,6 @@ from typing import Any, Final -import orjson from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse @@ -20,6 +19,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + pop_video_reference_to_video_id, ) from litellm.types.videos.utils import ( decode_character_id_with_provider, @@ -451,9 +451,7 @@ async def video_remix( version, ) - # Read request body - body: Final = await request.body() - data: Final = orjson.loads(body) + data: Final = await _read_request_body(request=request) data["video_id"] = video_id decoded: Final = decode_video_id_with_provider(video_id) @@ -760,15 +758,10 @@ async def video_edit( version, ) - body: Final = await request.body() - data: Final = orjson.loads(body) + data: Final = await _read_request_body(request=request) + pop_video_reference_to_video_id(data) - # Extract video_id from nested video object - video_ref: Final = data.pop("video", {}) - video_id: Final = video_ref.get("id", "") if isinstance(video_ref, dict) else "" - data["video_id"] = video_id - - decoded: Final = decode_video_id_with_provider(video_id) + decoded: Final = decode_video_id_with_provider(data["video_id"]) provider_from_id: Final = decoded.get("custom_llm_provider") model_id_from_decoded: Final = decoded.get("model_id") @@ -860,15 +853,10 @@ async def video_extension( version, ) - body: Final = await request.body() - data: Final = orjson.loads(body) + data: Final = await _read_request_body(request=request) + pop_video_reference_to_video_id(data) - # Extract video_id from nested video object - video_ref: Final = data.pop("video", {}) - video_id: Final = video_ref.get("id", "") if isinstance(video_ref, dict) else "" - data["video_id"] = video_id - - decoded: Final = decode_video_id_with_provider(video_id) + decoded: Final = decode_video_id_with_provider(data["video_id"]) provider_from_id: Final = decoded.get("custom_llm_provider") model_id_from_decoded: Final = decoded.get("model_id") diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index d6b398e3476..5f508cd02ca 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -13,6 +13,27 @@ def extract_model_from_target_model_names(target_model_names: Any) -> str | None return target_model_names[0] if target_model_names else None +def pop_video_reference_to_video_id(data: dict[str, Any]) -> None: + """ + Normalize OpenAI video edit/extension payloads into ``video_id``. + + JSON bodies use ``video: {"id": ...}``. Multipart and form-urlencoded bodies + may send a bare id string or a JSON-encoded reference object as a string field. + """ + video_ref: Final = data.pop("video", {}) + if isinstance(video_ref, dict): + video_id: Final = video_ref.get("id", "") + elif isinstance(video_ref, str): + try: + parsed_ref: Final = orjson.loads(video_ref) + except orjson.JSONDecodeError: + parsed_ref = None + video_id = parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref + else: + video_id = "" + data["video_id"] = video_id + + def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: custom_llm_provider: Final = data.get("custom_llm_provider") if custom_llm_provider: diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 40a26fad3c3..78f32600f73 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -375,6 +375,7 @@ async def test_content__model_encoded_id(harness): async def call_edit( harness: Harness, *, body: Dict[str, Any], headers=None, query=None ): + harness.read_body.return_value = dict(body) return await endpoints.video_edit( request=FakeRequest(headers=headers, query=query, raw_body=orjson.dumps(body)), fastapi_response=Response(), @@ -431,6 +432,27 @@ async def test_edit__missing_video_object_defaults_to_openai(harness): assert "video" not in data +@pytest.mark.asyncio +async def test_edit__bare_string_video_id_from_form_field(harness): + await call_edit(harness, body={"prompt": "brighter", "video": "video_plain"}) + + assert harness.processor_data() == { + "prompt": "brighter", + "video_id": "video_plain", + "custom_llm_provider": "openai", + } + + +@pytest.mark.asyncio +async def test_edit__json_string_video_reference_from_form_field(harness): + await call_edit( + harness, + body={"prompt": "brighter", "video": orjson.dumps({"id": "video_plain"}).decode()}, + ) + + assert harness.processor_data()["video_id"] == "video_plain" + + # =========================================================================== # # GET /v1/videos - video_list # # =========================================================================== # @@ -474,6 +496,7 @@ async def test_list__provider_from_header(harness): async def call_remix( harness: Harness, video_id: str, *, body, headers=None, query=None ): + harness.read_body.return_value = dict(body) return await endpoints.video_remix( video_id=video_id, request=FakeRequest(headers=headers, query=query, raw_body=orjson.dumps(body)), @@ -632,6 +655,7 @@ async def test_get_character__plain_id_defaults_openai_no_encode(harness): async def call_extension(harness: Harness, *, body, headers=None, query=None): + harness.read_body.return_value = dict(body) return await endpoints.video_extension( request=FakeRequest(headers=headers, query=query, raw_body=orjson.dumps(body)), fastapi_response=Response(), diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 3d0472ef96e..fc7ba773c38 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -2321,6 +2321,53 @@ def test_edit_and_extension_support_custom_provider_from_extra_body( assert captured_data["custom_llm_provider"] == "vertex_ai" +@pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) +def test_edit_and_extension_accept_form_encoded_after_auth_reads_body( + video_proxy_test_client, endpoint +): + from fastapi import Request + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "video_resp_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + } + + async def auth_that_reads_body_first(request: Request): + await _read_request_body(request=request) + return MagicMock() + + app = video_proxy_test_client.app + app.dependency_overrides[user_api_key_auth] = auth_that_reads_body_first + + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + endpoint, + headers={"Authorization": "Bearer sk-1234"}, + data={ + "model": "my-video-model", + "prompt": "brighter", + "video": "video_123", + }, + ) + + assert response.status_code == 200, response.text + assert captured_data["video_id"] == "video_123" + assert captured_data["prompt"] == "brighter" + + @pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) def test_edit_and_extension_route_with_encoded_video_ids( video_proxy_test_client, endpoint From 3db1759d04ba8d888a08dbe8308cc75a348141d8 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 13 Aug 2026 19:45:32 +0000 Subject: [PATCH 10/54] fix(bedrock): stop emitting an empty assistant delta after the finish_reason chunk Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/chat/invoke_handler.py | 8 +++- .../llms/bedrock/chat/test_invoke_handler.py | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 8d2b3dae71b..595884ae630 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -561,6 +561,10 @@ class AWSEventStreamDecoder: elif "usage" in chunk_data: usage = converse_config._transform_usage(chunk_data.get("usage", {})) + carries_message_content: Final = any( + key in chunk_data for key in ("start", "delta", "contentBlockIndex", "stopReason") + ) + model_response_provider_specific_fields: Final = {} if "trace" in chunk_data: trace: Final = chunk_data.get("trace") @@ -571,8 +575,8 @@ class AWSEventStreamDecoder: finish_reason=finish_reason, index=0, # Always 0 - Bedrock never returns multiple choices delta=Delta( - content=text, - role="assistant", + content=text if carries_message_content else None, + role="assistant" if carries_message_content else None, tool_calls=[tool_use] if tool_use else None, provider_specific_fields=(provider_specific_fields if provider_specific_fields else None), thinking_blocks=thinking_blocks, diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index ee50b9db015..9783976db4a 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -1,3 +1,4 @@ +import datetime import os import sys from unittest.mock import AsyncMock, MagicMock @@ -8,6 +9,8 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.bedrock.chat.invoke_handler import ( AWSEventStreamDecoder, make_call, @@ -293,3 +296,46 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) + +@pytest.mark.asyncio +async def test_converse_stream_ends_on_finish_reason_chunk(): + """The usage-only metadata event Bedrock sends after messageStop must not reach the caller as an extra + assistant delta following the finish_reason chunk.""" + model = "anthropic.claude-sonnet-4-6" + events = ( + {"role": "assistant"}, + {"contentBlockIndex": 0, "delta": {"text": "Hello"}}, + {"contentBlockIndex": 0, "delta": {"text": " world"}}, + {"contentBlockIndex": 0}, + {"stopReason": "end_turn"}, + {"usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, "metrics": {"latencyMs": 100}}, + ) + + async def bedrock_stream(): + decoder = AWSEventStreamDecoder(model=model) + for event in events: + yield decoder._chunk_parser(chunk_data=event) + + wrapper = CustomStreamWrapper( + completion_stream=bedrock_stream(), + model=model, + custom_llm_provider="bedrock", + logging_obj=LiteLLMLoggingObj( + model=model, + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=datetime.datetime.now(), + litellm_call_id="1234", + function_id="1234", + ), + ) + + chunks = [chunk async for chunk in wrapper] + + assert [choice.finish_reason for chunk in chunks for choice in chunk.choices].count("stop") == 1 + assert chunks[-1].choices[0].finish_reason == "stop", ( + f"stream must end on the finish_reason chunk, got trailing {chunks[-1].model_dump(exclude_none=True)}" + ) + assert any(getattr(chunk, "usage", None) is not None for chunk in wrapper.chunks) + From b573679384282e8a9ffa265a5af0b47e61a8bf89 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 21 Aug 2026 18:44:46 -0700 Subject: [PATCH 11/54] fix(proxy): keep every value of a repeated form key in get_form_data get_form_data collapsed the FormData multidict with dict(form) before the loop that rebuilds `foo[]` arrays ever ran, so a request sending timestamp_granularities[]=word and timestamp_granularities[]=segment reached the provider as ["segment"] with the first value silently dropped. Read the multidict with multi_items() instead. The test could not catch it because its mock was a plain dict carrying the same key twice, which Python collapses exactly the way the bug did. Every request.form mock that fed get_form_data now returns real FormData. --- .../proxy/common_utils/http_parsing_utils.py | 6 +- tests/test_litellm/ocr/test_ocr_file_input.py | 3 +- .../common_utils/test_http_parsing_utils.py | 74 ++++++++----------- .../test_llm_pass_through_endpoints.py | 3 +- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1e4344a71f4..4cb55f6966e 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -274,10 +274,10 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - form_data: Final = dict(form) parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form_data.items(): - # OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` + # multi_items(), not dict(form): a dict drops every value but the last of a repeated key, + # which is the whole array this function exists to rebuild + for key, value in form.multi_items(): if key.endswith("[]"): clean_key = key[:-2] parsed_form_data.setdefault(clean_key, []).append(value) diff --git a/tests/test_litellm/ocr/test_ocr_file_input.py b/tests/test_litellm/ocr/test_ocr_file_input.py index e6216d7c580..feb98d14c03 100644 --- a/tests/test_litellm/ocr/test_ocr_file_input.py +++ b/tests/test_litellm/ocr/test_ocr_file_input.py @@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock import orjson import pytest +from starlette.datastructures import FormData from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type @@ -470,7 +471,7 @@ class TestProxySecurityGuard: mock_request = MagicMock() mock_request.headers = {"content-type": "multipart/form-data; boundary=---"} - mock_request.form = AsyncMock(return_value=mock_form) + mock_request.form = AsyncMock(return_value=FormData(mock_form)) result = await self._parse_multipart(mock_request) diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 869d228d5a4..98af1fb7ce2 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -7,6 +7,7 @@ import orjson import pytest from fastapi import Request from fastapi.testclient import TestClient +from starlette.datastructures import FormData sys.path.insert( 0, os.path.abspath("../../../..") @@ -73,7 +74,7 @@ async def test_form_data_parsing(): test_data = {"name": "test_user", "message": "hello world"} # Mock the form method to return the test data as an awaitable - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -124,7 +125,7 @@ async def test_form_data_with_json_metadata(): } # Mock the form method to return the test data as an awaitable - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -165,7 +166,7 @@ async def test_form_data_with_invalid_json_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -188,7 +189,7 @@ async def test_form_data_without_metadata(): test_data = {"model": "whisper-1", "file": "audio.mp3", "language": "en"} # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -219,7 +220,7 @@ async def test_form_data_with_empty_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -254,7 +255,7 @@ async def test_form_data_with_dict_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -285,7 +286,7 @@ async def test_form_data_with_none_metadata(): } # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=test_data) + mock_request.form = AsyncMock(return_value=FormData(test_data)) mock_request.headers = {"content-type": "multipart/form-data"} mock_request.scope = {} mock_request.state._cached_headers = None @@ -500,33 +501,29 @@ async def test_surrogate_repair_skipped_above_size_limit(monkeypatch): @pytest.mark.asyncio async def test_get_form_data(): """ - Test that get_form_data correctly handles form data with array notation. - Tests audio transcription parameters as a specific example. + A repeated `foo[]` key is how the OpenAI SDKs send a list, so every value has to + survive. `FormData`, not a dict: a dict cannot even hold the duplicate key. """ - # Create a mock request with transcription form data mock_request = MagicMock() + mock_request.form = AsyncMock( + return_value=FormData( + [ + ("file", "file_object"), + ("model", "gpt-4o-transcribe"), + ("include[]", "logprobs"), + ("language", "en"), + ("prompt", "Transcribe this audio file"), + ("response_format", "json"), + ("stream", "false"), + ("temperature", "0.2"), + ("timestamp_granularities[]", "word"), + ("timestamp_granularities[]", "segment"), + ] + ) + ) - # Create mock form data with array notation for timestamp_granularities - mock_form_data = { - "file": "file_object", # In a real request this would be an UploadFile - "model": "gpt-4o-transcribe", - "include[]": "logprobs", # Array notation - "language": "en", - "prompt": "Transcribe this audio file", - "response_format": "json", - "stream": "false", - "temperature": "0.2", - "timestamp_granularities[]": "word", # First array item - "timestamp_granularities[]": "segment", # Second array item (would overwrite in dict, but handled by the function) - } - - # Mock the form method to return the test data - mock_request.form = AsyncMock(return_value=mock_form_data) - - # Call the function being tested result = await get_form_data(mock_request) - # Verify regular form fields are preserved assert result["file"] == "file_object" assert result["model"] == "gpt-4o-transcribe" assert result["language"] == "en" @@ -534,17 +531,8 @@ async def test_get_form_data(): assert result["response_format"] == "json" assert result["stream"] == "false" assert result["temperature"] == "0.2" - - # Verify array fields are correctly parsed - assert "include" in result - assert isinstance(result["include"], list) - assert "logprobs" in result["include"] - - assert "timestamp_granularities" in result - assert isinstance(result["timestamp_granularities"], list) - # Note: In a real MultiDict, both values would be present - # But in our mock dictionary the second value overwrites the first - assert "segment" in result["timestamp_granularities"] + assert result["include"] == ["logprobs"] + assert result["timestamp_granularities"] == ["word", "segment"] def test_get_tags_from_request_body_with_metadata_tags(): @@ -958,7 +946,7 @@ class TestReadRequestBodyNonCanonicalContentType: mock_request = MagicMock() mock_request.body = AsyncMock(return_value=orjson.dumps(payload)) - mock_request.form = AsyncMock(return_value={}) + mock_request.form = AsyncMock(return_value=FormData({})) mock_request.headers = {"content-type": content_type} mock_request.scope = {} @@ -969,7 +957,7 @@ class TestReadRequestBodyNonCanonicalContentType: @pytest.mark.asyncio async def test_real_form_post_still_parsed_as_form(self): mock_request = MagicMock() - mock_request.form = AsyncMock(return_value={"k": "v"}) + mock_request.form = AsyncMock(return_value=FormData({"k": "v"})) mock_request.body = AsyncMock(return_value=b"") mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} mock_request.scope = {} @@ -1025,7 +1013,7 @@ class TestGetRequestBody: mock_request = MagicMock() mock_request.method = "POST" mock_request.headers = {"content-type": "multipart/form-data; boundary=x"} - mock_request.form = AsyncMock(return_value={"k": "v"}) + mock_request.form = AsyncMock(return_value=FormData({"k": "v"})) mock_request.scope = {} result = await get_request_body(mock_request) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 6568f6aeacf..88141fd1c90 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -13,6 +13,7 @@ import httpx import pytest from fastapi import HTTPException, Request, Response from fastapi.testclient import TestClient +from starlette.datastructures import FormData sys.path.insert( 0, os.path.abspath("../../../..") @@ -1384,7 +1385,7 @@ async def test_is_streaming_request_fn(): mock_request = Mock() mock_request.method = "POST" mock_request.headers = {"content-type": "multipart/form-data"} - mock_request.form = AsyncMock(return_value={"stream": "true"}) + mock_request.form = AsyncMock(return_value=FormData({"stream": "true"})) assert await is_streaming_request_fn(mock_request) is True From b7f8016002c080f64ca76a1136fc3fe103a5ee75 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 21 Aug 2026 18:44:57 -0700 Subject: [PATCH 12/54] test: gate the test suite on F601, B023, B025 and F632 Four more ruff rules for code the test suite runs but never checks. F601 is the one that paid: the duplicate key it flagged in a get_form_data fixture was the mock reproducing the production bug fixed in the previous commit. B025 removed two unreachable handlers, one of them a pytest.skip shadowed by an earlier `pass`, so an upstream Vertex flake reported green having asserted nothing. F632 turned an `is ""` identity check, which passes only on CPython interning, into the `== ""` it meant. B023 fixed three closures over loop variables, all latent today but one iteration-order change away from checking the last case N times. --- ruff-tests.toml | 15 ++++++++++++++ tests/code_coverage_tests/bedrock_pricing.py | 2 +- tests/load_tests/test_langsmith_load_test.py | 5 ----- .../test_amazing_vertex_completion.py | 10 ++-------- .../logging_callback_tests/test_spend_logs.py | 1 - .../test_key_generate_prisma.py | 20 ++++++++----------- .../test_ollama_completion_transformation.py | 2 +- .../llms/watsonx/test_watsonx_common_utils.py | 6 +----- tests/test_litellm/test_utils.py | 2 -- 9 files changed, 28 insertions(+), 35 deletions(-) diff --git a/ruff-tests.toml b/ruff-tests.toml index de0931f5e69..8f21ca31a6f 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -36,6 +36,17 @@ # `re.search`, so a `.` copied out of an error message is a wildcard and the block # accepts messages the author never meant to accept. Mark a real regex raw, wrap a # literal message in `re.escape`, and the pattern says which one it is +# F601 the same key literal twice in one dict. Python keeps the last value, so the +# first is dropped before the test ever runs, and a fixture that looks like it +# covers two cases covers one +# B023 a closure over a loop variable. Every closure sees the last iteration's value, +# so a per-case callback built in a loop checks the last case N times. Bind the +# value as a parameter instead +# B025 an `except` for a type an earlier `except` already catches. The second handler +# is unreachable, so the recovery or skip written there never happens +# F632 `is` against a literal. It compares identity, so it passes only where CPython +# happens to intern the value and stops meaning what it says the moment the +# value is built at runtime # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -58,4 +69,8 @@ lint.select = [ "PLR0133", "PLW0127", "RUF043", + "F601", + "B023", + "B025", + "F632", ] diff --git a/tests/code_coverage_tests/bedrock_pricing.py b/tests/code_coverage_tests/bedrock_pricing.py index b2c9e78b06c..e6219109f6d 100644 --- a/tests/code_coverage_tests/bedrock_pricing.py +++ b/tests/code_coverage_tests/bedrock_pricing.py @@ -95,7 +95,7 @@ def get_bedrock_pricing(url, providers): else: # General logic for other providers section = soup.find( - "h2", text=lambda t: t and provider.lower() in t.lower() + "h2", text=lambda t, needle=provider.lower(): t and needle in t.lower() ) if not section: pricing_data[provider] = "Provider section not found" diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py index cf9fe526b74..40b976541a5 100644 --- a/tests/load_tests/test_langsmith_load_test.py +++ b/tests/load_tests/test_langsmith_load_test.py @@ -66,11 +66,6 @@ def test_langsmith_logging_async(): except Exception as e: pytest.fail(f"An exception occurred - {e}") - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {e}") - async def make_async_calls(metadata=None, **completion_kwargs): total_tasks = 300 diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index a52b5975f6e..53b3b2d6071 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -4207,13 +4207,7 @@ def test_gemini_google_maps_tool_simple(): ) print(f"Response: {response.model_dump_json(indent=4)}") assert response.choices[0].message.content is not None - except (litellm.RateLimitError, litellm.InternalServerError): - # Transient Vertex-side failures (rate limiting, 500 INTERNAL from the - # Google Maps grounding backend) are not LiteLLM bugs — don't fail CI. - pass - except litellm.InternalServerError: - pytest.skip( - "Google Maps Platform returned a transient 500 (upstream flake); skipping." - ) + except (litellm.RateLimitError, litellm.InternalServerError) as e: + pytest.skip(f"Transient Vertex-side failure, not a LiteLLM bug: {e}") except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/logging_callback_tests/test_spend_logs.py b/tests/logging_callback_tests/test_spend_logs.py index 709aa81f421..3aa0b3ebd90 100644 --- a/tests/logging_callback_tests/test_spend_logs.py +++ b/tests/logging_callback_tests/test_spend_logs.py @@ -148,7 +148,6 @@ def test_spend_logs_payload(model_id: Optional[str]): "completion_start_time": datetime.datetime(2024, 6, 7, 12, 43, 30, 954146), "max_tokens": 10, "extra_body": {}, - "custom_llm_provider": "azure", "input": [ {"role": "system", "content": "you are a helpful assistant.\n"}, {"role": "user", "content": "bom dia"}, diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index efedc156429..4037107c474 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -3327,16 +3327,17 @@ async def test_team_access_groups(prisma_client): request._url = URL(url="/chat/completions") + def body_reader(requested_model: str): + async def return_body() -> bytes: + return f'{{"model": "{requested_model}"}}'.encode() + + return return_body + for model in ["gpt-4o", "gemini-pro-vision"]: # Expect these to pass - async def return_body(): - return_string = f'{{"model": "{model}"}}' - # return string as bytes - return return_string.encode() - request = Request(scope={"type": "http"}) request._url = URL(url="/chat/completions") - request.body = return_body + request.body = body_reader(model) # use generated key to auth in print( @@ -3346,14 +3347,9 @@ async def test_team_access_groups(prisma_client): for model in ["gpt-4", "gpt-4o-mini", "gemini-experimental"]: # Expect these to fail - async def return_body_2(): - return_string = f'{{"model": "{model}"}}' - # return string as bytes - return return_string.encode() - request = Request(scope={"type": "http"}) request._url = URL(url="/chat/completions") - request.body = return_body_2 + request.body = body_reader(model) # use generated key to auth in print( diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index dd59cdcac1c..e746c1bfd6b 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -480,7 +480,7 @@ class TestOllamaTextCompletionResponseIterator: assert isinstance(result, ModelResponseStream) assert result.choices and result.choices[0].delta is not None assert result.choices[0].delta.content == None - assert getattr(result.choices[0].delta, "reasoning_content", None) is "" + assert getattr(result.choices[0].delta, "reasoning_content", None) == "" def test_chunk_parser_done_chunk(self): """Test that done chunks work correctly.""" diff --git a/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py b/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py index be74dc40eda..0f3a6bae1ff 100644 --- a/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py +++ b/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py @@ -129,11 +129,7 @@ class TestGenerateIAMToken: mock_client.reset_mock() mock_cache.reset_mock() - # Configure mock to return values based on env_keys - def get_secret_side_effect(key): - return env_keys.get(key) - - mock_get_secret_str.side_effect = get_secret_side_effect + mock_get_secret_str.side_effect = env_keys.get mock_response = MagicMock() mock_response.json.return_value = { diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index bd23ca11fbe..a01540c51e8 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -864,7 +864,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_character_above_128k_tokens": {"type": "number"}, "input_cost_per_image": {"type": "number"}, "input_cost_per_image_above_128k_tokens": {"type": "number"}, - "input_cost_per_image_token": {"type": "number"}, "input_cost_per_video_token": {"type": "number"}, "input_cost_per_token_above_200k_tokens": {"type": "number"}, "input_cost_per_token_above_256k_tokens": {"type": "number"}, @@ -1008,7 +1007,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, "bedrock_converse_supports_strict_tools": {"type": "boolean"}, "tpm": {"type": "number"}, - "provider_specific_entry": {"type": "object"}, "supported_endpoints": { "type": "array", "items": { From d6f9bce4bb7c0783cb994f3ef165c182069557dc Mon Sep 17 00:00:00 2001 From: milan Date: Sat, 22 Aug 2026 04:05:02 +0000 Subject: [PATCH 13/54] fix(a2a): normalize agent card protocolBinding casing before transport match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/a2a_protocol/card_resolver.py | 28 ++++++++++++ litellm/a2a_protocol/main.py | 13 +++--- tests/test_litellm/a2a_protocol/test_main.py | 45 +++++++++++++++++--- 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/litellm/a2a_protocol/card_resolver.py b/litellm/a2a_protocol/card_resolver.py index d14d892256b..8070d88b761 100644 --- a/litellm/a2a_protocol/card_resolver.py +++ b/litellm/a2a_protocol/card_resolver.py @@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM. Extends the A2A SDK's card resolver to support multiple well-known paths. """ +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger @@ -48,6 +49,33 @@ def is_localhost_or_internal_url(url: str | None) -> bool: return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS) +_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType( + { + "jsonrpc": "JSONRPC", + "http+json": "HTTP+JSON", + "grpc": "GRPC", + } +) + + +def normalize_agent_card_protocol_bindings(agent_card: "AgentCard") -> "AgentCard": + """ + Canonicalize protocolBinding casing on the card's supported interfaces. + + Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase + bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings + case-sensitively against its uppercase TransportProtocol constants and fails + with "no compatible transports found." for spec-adjacent casings. + """ + interfaces: Final = getattr(agent_card, "supported_interfaces", None) or () + for interface in interfaces: + binding: str = getattr(interface, "protocol_binding", "") or "" + canonical = _CANONICAL_PROTOCOL_BINDINGS.get(binding.lower()) + if canonical is not None and binding != canonical: + interface.protocol_binding = canonical + return agent_card + + def get_agent_card_url(agent_card: "AgentCard") -> str | None: """Return the agent endpoint URL from the resolved SDK card.""" url: Final = getattr(agent_card, "url", None) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 1c6ebf0b95c..2d4267cc0db 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -73,6 +73,7 @@ except ImportError: from litellm.a2a_protocol.card_resolver import ( LiteLLMA2ACardResolver, get_agent_card_url, + normalize_agent_card_protocol_bindings, ) from litellm.a2a_protocol.exception_mapping_utils import ( handle_a2a_localhost_retry, @@ -782,13 +783,17 @@ async def create_a2a_client( if extra_headers: verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys())) + resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url) + agent_card: Final = normalize_agent_card_protocol_bindings( + await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None) + ) + a2a_client: Final = await create_client( # pyright: ignore[reportOptionalCall] - base_url, + agent_card, client_config=ClientConfig( # pyright: ignore[reportOptionalCall] httpx_client=httpx_client, streaming=streaming, ), - resolver_http_kwargs={"headers": extra_headers} if extra_headers else None, ) # Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse # the configured httpx client and this agent's headers without excavating @@ -799,9 +804,7 @@ async def create_a2a_client( if extra_headers else None ) - agent_card: Final = getattr(a2a_client, "_card", None) - if agent_card is not None: - a2a_client._litellm_agent_card = agent_card + a2a_client._litellm_agent_card = agent_card verbose_logger.info("A2A client created for %s", base_url) diff --git a/tests/test_litellm/a2a_protocol/test_main.py b/tests/test_litellm/a2a_protocol/test_main.py index 08f6b9f25bb..29aa3aaabd7 100644 --- a/tests/test_litellm/a2a_protocol/test_main.py +++ b/tests/test_litellm/a2a_protocol/test_main.py @@ -176,10 +176,32 @@ _AGENT_A_HEADERS = {"x-agent-token": "token-for-a", "x-tenant": "tenant-a"} _AGENT_B_HEADERS = {"x-agent-token": "token-for-b", "x-tenant": "tenant-b"} +_V1_RPC_REPLY = { + "jsonrpc": "2.0", + "id": "reply", + "result": {"message": {"messageId": "reply-1", "role": "ROLE_AGENT", "parts": [{"text": "pong"}]}}, +} + + +_LOWERCASE_BINDING_CARD = { + "name": "langgraph-agent", + "version": "1.0.0", + "capabilities": {"streaming": True}, + "defaultInputModes": ["text/plain"], + "defaultOutputModes": ["text/plain"], + "skills": [], + "supportedInterfaces": [ + {"url": "http://127.0.0.1:9/", "protocolBinding": "jsonrpc", "protocolVersion": "1.0"} + ], +} + + class _RequestRecorder: """Records the headers httpx put on the wire, per outbound request.""" - def __init__(self): + def __init__(self, card=_AGENT_CARD, rpc_reply=_RPC_REPLY): + self.card = card + self.rpc_reply = rpc_reply self.card_requests = [] self.rpc_requests = [] self.client = None @@ -188,23 +210,23 @@ class _RequestRecorder: headers = {k.lower(): v for k, v in request.headers.items()} if request.method == "GET": self.card_requests.append(headers) - return httpx.Response(200, json=_AGENT_CARD) + return httpx.Response(200, json=self.card) self.rpc_requests.append(headers) - return httpx.Response(200, json=_RPC_REPLY) + return httpx.Response(200, json=self.rpc_reply) def _a2a_client_cache_key(timeout: float) -> str: return "async_httpx_client" + f"timeout_{timeout}" + httpxSpecialProvider.A2AProvider -async def _seed_shared_a2a_client() -> _RequestRecorder: +async def _seed_shared_a2a_client(card=_AGENT_CARD, rpc_reply=_RPC_REPLY) -> _RequestRecorder: """Put the one A2A client the cache will hand out behind a mock transport. Seeding has to happen on the test's own event loop, because the client cache keys on it. The injected client is a real httpx.AsyncClient, so the merge of per-request headers over client defaults, which is what these tests are about, stays real. """ - recorder = _RequestRecorder() + recorder = _RequestRecorder(card=card, rpc_reply=rpc_reply) handler = AsyncHTTPHandler(timeout=DEFAULT_A2A_AGENT_TIMEOUT) owned_client = handler.client handler.client = httpx.AsyncClient(transport=httpx.MockTransport(recorder)) @@ -311,6 +333,19 @@ async def test_streaming_send_carries_only_its_own_caller_headers(isolated_clien assert received["b"]["x-tenant"] == "tenant-b" +@pytest.mark.asyncio +async def test_lowercase_protocol_binding_in_agent_card_still_gets_a_client(isolated_client_cache): + """LangGraph Platform serves cards with protocolBinding "jsonrpc"; a2a-sdk matches + bindings case-sensitively, so without normalization client creation raises + ValueError("no compatible transports found.").""" + await _seed_shared_a2a_client(card=_LOWERCASE_BINDING_CARD, rpc_reply=_V1_RPC_REPLY) + + a2a_client = await create_a2a_client(base_url="http://127.0.0.1:9") + response = await _send_message(a2a_client, _send_request("lc")) + + assert type(response.root.result).__name__ == "Message" + + @pytest.mark.asyncio async def test_agent_card_fetch_carries_the_callers_headers(isolated_client_cache): """Agent cards can sit behind the same auth as the agent, so the card fetch must stay From 329202004ef15d77e79b4c7c234c208567c22108 Mon Sep 17 00:00:00 2001 From: milan Date: Sat, 22 Aug 2026 04:17:16 +0000 Subject: [PATCH 14/54] test(a2a): stub card resolver in create_a2a_client unit tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_agent_header_isolation.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_header_isolation.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_header_isolation.py index 5193ad989dc..30095796a7f 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_header_isolation.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_header_isolation.py @@ -245,13 +245,21 @@ def _fake_get_async_httpx_client_factory(captured_calls: list): return _fake_get_async_httpx_client -async def _fake_create_client(base_url, client_config=None, **kwargs): +async def _fake_create_client(agent_card, client_config=None, **kwargs): client = MagicMock() if client_config is not None: client._litellm_httpx_client = client_config.httpx_client return client +def _fake_card_resolver(httpx_client, base_url, **kwargs): + resolver = MagicMock() + card = MagicMock() + card.supported_interfaces = () + resolver.get_agent_card = AsyncMock(return_value=card) + return resolver + + @pytest.mark.asyncio async def test_create_a2a_client_leaves_the_shared_client_untouched(): """ @@ -276,6 +284,10 @@ async def test_create_a2a_client_leaves_the_shared_client_untouched(): "litellm.a2a_protocol.main.create_client", new=AsyncMock(side_effect=_fake_create_client), ), + patch( + "litellm.a2a_protocol.main.A2ACardResolver", + side_effect=_fake_card_resolver, + ), ): await create_a2a_client( base_url="http://agent-a:9999", @@ -321,6 +333,10 @@ async def test_create_a2a_client_default_timeout_matches_constant(): "litellm.a2a_protocol.main.create_client", new=AsyncMock(side_effect=_fake_create_client), ), + patch( + "litellm.a2a_protocol.main.A2ACardResolver", + side_effect=_fake_card_resolver, + ), ): await create_a2a_client(base_url="http://127.0.0.1:9") @@ -352,6 +368,10 @@ async def test_create_a2a_client_explicit_timeout_overrides_default(): "litellm.a2a_protocol.main.create_client", new=AsyncMock(side_effect=_fake_create_client), ), + patch( + "litellm.a2a_protocol.main.A2ACardResolver", + side_effect=_fake_card_resolver, + ), ): await create_a2a_client(base_url="http://127.0.0.1:9", timeout=42.5) From 766f72f1d7c1dc7b7b1d1d93f8dde46090f76f9e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:19:42 -0700 Subject: [PATCH 15/54] fix(anthropic): round-trip thinking blocks to OpenAI backends The experimental /v1/messages adapters lost prior-turn reasoning three different ways once the request left for an OpenAI-shaped backend. On the Responses path, thinking blocks were flattened into output_text inside the assistant message, so the model read its own private reasoning back as visible prose and no reasoning item was ever sent. They now become Responses reasoning input items, grouped by signature so summary parts that arrived as one item go back as one item. The response direction stops hardcoding signature=None and carries the reasoning item id, which is what lets the next turn regroup them; the streaming wrapper emits the matching signature_delta. On the chat completions path the adapter attached thinking_blocks but never set reasoning_content, so Moonshot and DeepSeek substituted a single-space placeholder and other providers sent nothing. It is now derived from the thinking blocks. With use_chat_completions_url_for_anthropic_messages and a model that itself bridges to /v1/responses, the assistant message was dropped whole: reasoning, text, and all. That branch now emits the reasoning items and the message content alongside the tool calls. Fixes #24985 --- .../transformation.py | 32 +++++- .../prompt_templates/common_utils.py | 32 ++++++ .../adapters/transformation.py | 4 + .../responses_adapters/streaming_iterator.py | 13 ++- .../responses_adapters/transformation.py | 107 ++++++++++++------ ...responses_transformation_transformation.py | 87 ++++++++++++++ ...al_pass_through_adapters_transformation.py | 42 +++++++ ...t_responses_adapters_streaming_iterator.py | 14 ++- .../test_responses_adapters_transformation.py | 90 ++++++++++++++- 9 files changed, 379 insertions(+), 42 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6103b1bf484..ea54bd83e12 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -21,6 +21,9 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse from litellm._logging import verbose_logger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + responses_reasoning_item_from_thinking_blocks, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, @@ -85,6 +88,21 @@ def _get_reasoning_items( return [] +def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload + """Reasoning input items for an assistant message. + + Stored reasoning items win because they carry an id the Responses API minted; thinking + blocks are the fallback for turns that arrived over another API surface. + """ + stored: Final = [_reasoning_item_to_response_input(r_item) for r_item in _get_reasoning_items(msg)] + if stored: + return stored + from_thinking: Final = responses_reasoning_item_from_thinking_blocks( + cast(Iterable[Mapping[str, Any]], msg.get("thinking_blocks") or ()) # cast-ok: untyped client thinking blocks + ) + return [from_thinking] if from_thinking is not None else [] + + def _build_reasoning_item( item_id: str, encrypted_content: str | None, @@ -372,8 +390,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) ) elif role == "assistant" and tool_calls and isinstance(tool_calls, list): - for r_item in _get_reasoning_items(msg): - input_items.append(_reasoning_item_to_response_input(r_item)) + input_items.extend(_reasoning_input_items(msg)) + if content: + input_items.append( + { + "type": "message", + "role": "assistant", + "content": self._convert_content_to_responses_format(content, "assistant"), + } + ) for tool_call in tool_calls: function = tool_call.get("function") custom = tool_call.get("custom") @@ -400,8 +425,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raise ValueError(f"tool call not supported: {tool_call}") elif content is not None: if role == "assistant": - for r_item in _get_reasoning_items(msg): - input_items.append(_reasoning_item_to_response_input(r_item)) + input_items.extend(_reasoning_input_items(msg)) input_items.append( { "type": "message", diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2db5776047b..4211fbe610a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1549,6 +1549,38 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]: return None, message_content +def reasoning_content_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, Any]], +) -> str: + """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect. + + Redacted blocks carry no readable text, so they contribute nothing. + """ + return "\n".join( + text + for block in thinking_blocks + if block.get("type") == "thinking" and (text := str(block.get("thinking") or "")) + ) + + +def responses_reasoning_item_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, Any]], +) -> dict[str, Any] | None: # mutable-ok: API message payload + """Build a Responses API `reasoning` input item from Anthropic thinking blocks. + + The item carries no `id`: the Responses API rejects an empty one and 404s on any id it + did not mint itself, while an item without an id is always accepted. + """ + summary: Final[list[dict[str, Any]]] = [ # mutable-ok: API message payload + {"type": "summary_text", "text": text} # mutable-ok: API message payload + for block in thinking_blocks + if block.get("type") == "thinking" and (text := str(block.get("thinking") or "")) + ] + if not summary: + return None + return {"type": "reasoning", "summary": summary} # mutable-ok: API message payload + + def _parse_content_for_reasoning( message_text: str | None, ) -> tuple[str | None, str | None]: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 34c2d837127..7c89da81fe6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -64,6 +64,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho from litellm.litellm_core_utils.prompt_templates.common_utils import ( parse_tool_call_arguments, + reasoning_content_from_thinking_blocks, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -592,6 +593,9 @@ class LiteLLMAnthropicMessagesAdapter: assistant_message["tool_calls"] = tool_calls if len(thinking_blocks) > 0: assistant_message["thinking_blocks"] = thinking_blocks + reasoning_content = reasoning_content_from_thinking_blocks(thinking_blocks) + if reasoning_content: + assistant_message["reasoning_content"] = reasoning_content new_messages.append(assistant_message) return new_messages diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index e2ad9c9c6d3..286edb24b9b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -152,7 +152,7 @@ class AnthropicResponsesStreamWrapper: if block_idx < 0: if not delta: return - block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""}) + block_idx = self._open_block(item_id, {"type": "thinking", "thinking": "", "signature": ""}) self._chunk_queue.append( { "type": "content_block_delta", @@ -189,6 +189,17 @@ class AnthropicResponsesStreamWrapper: block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: return + done_item_type: Final = ( + getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) if item else None + ) + if done_item_type == "reasoning" and item_id: + self._chunk_queue.append( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "signature_delta", "signature": item_id}, + } + ) self._chunk_queue.append( { "type": "content_block_stop", diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 25d729d8606..b1061f86d0b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,12 +6,14 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable +from collections.abc import Iterable, Mapping +from itertools import groupby from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + responses_reasoning_item_from_thinking_blocks, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( @@ -100,6 +102,58 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload ] + @staticmethod + def _summary_part_text(part: object) -> str: + if isinstance(part, Mapping): + mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json + return str(mapping.get("text", "")) + return str(getattr(part, "text", "")) + + @classmethod + def _thinking_blocks_from_reasoning_item( + cls, + item_id: str | None, + summary: Iterable[object], + ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload + """Anthropic thinking blocks for one Responses reasoning item. + + The reasoning item id rides along as the signature so that a follow-up turn can + regroup the summary parts into the single reasoning item they came from. + """ + return tuple( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=text, + signature=item_id or None, + ).model_dump() + for part in summary + if (text := cls._summary_part_text(part)) + ) + + @staticmethod + def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str: + """Group consecutive thinking blocks sharing a signature; keep every other block alone.""" + index, block = indexed_block + signature: Final = block.get("signature") or "" + return f"thinking:{signature}" if block.get("type") == "thinking" else f"block:{index}" + + @classmethod + def _assistant_group_to_input_item( + cls, group: tuple[Mapping[str, Any], ...] + ) -> dict[str, Any] | None: # mutable-ok: API message payload + first: Final = group[0] + btype: Final = first.get("type") + if btype == "thinking": + return responses_reasoning_item_from_thinking_blocks(group) + if btype == "tool_use": + return { # mutable-ok: API message payload + "type": "function_call", + "call_id": first.get("id", ""), + "name": first.get("name", ""), + "arguments": json.dumps(first.get("input", {})), + } + return None + def translate_messages_to_responses_input( self, messages: list[AllAnthropicPassThroughMessageValues], @@ -113,6 +167,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: user image -> message(role=user, input_image) user tool_result -> function_call_output assistant text -> message(role=assistant, output_text) + assistant thinking -> reasoning assistant tool_use -> function_call """ input_items: Final[list[dict[str, Any]]] = [] @@ -233,27 +288,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter: } ) elif isinstance(content, list): - asst_parts: list[dict[str, Any]] = [] - for block in content: - if not isinstance(block, dict): - continue - btype = block.get("type") - if btype == "text": - asst_parts.append({"type": "output_text", "text": block.get("text", "")}) - elif btype == "tool_use": - # tool_use becomes a top-level function_call item - input_items.append( - { - "type": "function_call", - "call_id": block.get("id", ""), - "name": block.get("name", ""), - "arguments": json.dumps(block.get("input", {})), - } - ) - elif btype == "thinking": - thinking_text = block.get("thinking", "") - if thinking_text: - asst_parts.append({"type": "output_text", "text": thinking_text}) + blocks = tuple(block for block in content if isinstance(block, dict)) + input_items.extend( + item + for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key) + if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None + ) + asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload + {"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload + for block in blocks + if block.get("type") == "text" + ] if asst_parts: input_items.append( { @@ -514,16 +559,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: for item in response.output: if isinstance(item, ResponseReasoningItem): - for summary in item.summary: - text = getattr(summary, "text", "") - if text: - content.append( - AnthropicResponseContentBlockThinking( - type="thinking", - thinking=text, - signature=None, - ).model_dump() - ) + content.extend(self._thinking_blocks_from_reasoning_item(item.id, item.summary)) elif isinstance(item, ResponseOutputMessage): for part in item.content: @@ -555,6 +591,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content.append( AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump() ) + elif item_type == "reasoning": + content.extend( + self._thinking_blocks_from_reasoning_item( + item.get("id"), + cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json + ) + ) elif item_type == "function_call": try: input_data = json.loads(item.get("arguments", "{}")) diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 4ff92aaf87d..124dc67b4fd 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3762,3 +3762,90 @@ def test_response_incomplete_stream_event_without_details_defaults_to_length(): result = iterator.chunk_parser(chunk) assert result.choices[0].finish_reason == "length" + + +def test_assistant_message_with_tool_calls_keeps_its_content(): + """Regression for https://github.com/BerriAI/litellm/issues/24985. + + An assistant turn that both answered and called a tool used to lose its whole message: + the branch handling tool_calls emitted the calls and dropped the text. + """ + handler = LiteLLMResponsesTransformationHandler() + messages = [ + {"role": "user", "content": "What is the weather in Denver?"}, + { + "role": "assistant", + "content": "Let me look that up.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Denver"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "88F"}, + ] + + input_items, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + assistant_message = next( + item for item in input_items if item.get("type") == "message" and item.get("role") == "assistant" + ) + assert assistant_message["content"] == [{"type": "output_text", "text": "Let me look that up."}] + assert [item.get("type") for item in input_items] == [ + "message", + "message", + "function_call", + "function_call_output", + ] + + +def test_assistant_thinking_blocks_become_a_reasoning_input_item(): + """Thinking blocks are how an Anthropic-shaped turn carries reasoning into this bridge.""" + handler = LiteLLMResponsesTransformationHandler() + messages = [ + {"role": "user", "content": "What is the weather in Denver?"}, + { + "role": "assistant", + "content": "Denver is sunny.", + "thinking_blocks": [ + {"type": "thinking", "thinking": "August in Denver is dry.", "signature": "sig1"}, + {"type": "redacted_thinking", "data": "REDACTED"}, + ], + }, + {"role": "user", "content": "Why?"}, + ] + + input_items, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + reasoning_item = next(item for item in input_items if item.get("type") == "reasoning") + assert reasoning_item["summary"] == [{"type": "summary_text", "text": "August in Denver is dry."}] + assert "id" not in reasoning_item + + +def test_stored_reasoning_items_win_over_thinking_blocks(): + """A minted reasoning id beats a re-derived one, so the two must not both be sent.""" + handler = LiteLLMResponsesTransformationHandler() + messages = [ + { + "role": "assistant", + "content": "Denver is sunny.", + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_real", + "summary": [{"type": "summary_text", "text": "August in Denver is dry."}], + } + ], + "thinking_blocks": [ + {"type": "thinking", "thinking": "August in Denver is dry.", "signature": "rs_real"} + ], + }, + ] + + input_items, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + reasoning_items = [item for item in input_items if item.get("type") == "reasoning"] + assert len(reasoning_items) == 1 + assert reasoning_items[0]["id"] == "rs_real" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index e4dacc308dc..d0169963962 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -359,6 +359,48 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks(): assert result[1]["tool_calls"][0]["id"] == "toolu_01234" +def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): + """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. + + Without it Moonshot and DeepSeek fill in a single-space placeholder and the model gets + a blank where its own prior reasoning belongs. + """ + + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "text", "text": "Which city is best for a picnic?"}], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "Denver is dry in August.", "signature": "sig1"}, + {"type": "thinking", "thinking": "San Francisco is foggy.", "signature": "sig2"}, + {"type": "redacted_thinking", "data": "REDACTED"}, + {"type": "text", "text": "Denver."}, + ], + ), + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages) + + assert result[1]["reasoning_content"] == "Denver is dry in August.\nSan Francisco is foggy." + assert result[1]["content"] == "Denver." + + +def test_translate_anthropic_messages_to_openai_sets_no_reasoning_content_without_thinking(): + anthropic_messages = [ + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[{"type": "text", "text": "Denver."}], + ), + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages) + + assert "reasoning_content" not in result[0] + + def test_translate_anthropic_messages_to_openai_tool_message_placement(): """Test that tool result messages are placed before user messages in the conversation order.""" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 8b591fcd7da..12a6ab4324f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -139,14 +139,26 @@ class TestReasoningItemWithoutSummaryText: ("content_block_start", 0), ("content_block_delta", 0), ("content_block_delta", 0), + ("content_block_delta", 0), ("content_block_stop", 0), ("content_block_start", 1), ("content_block_delta", 1), ("content_block_stop", 1), ] - assert chunks[1]["content_block"] == {"type": "thinking", "thinking": ""} + assert chunks[1]["content_block"] == {"type": "thinking", "thinking": "", "signature": ""} assert "".join(c["delta"]["thinking"] for c in chunks[2:4]) == "Weighing options" + def test_reasoning_item_id_is_streamed_as_the_thinking_signature(self): + chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=["Weighing options"])) + + signature_deltas = [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] + assert [(c["index"], c["delta"]["signature"]) for c in signature_deltas] == [(0, "rs_1")] + + def test_no_signature_delta_without_a_thinking_block(self): + chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=[])) + + assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] + class TestToolUseBlockClosedExactlyOnce: """Regression for https://github.com/BerriAI/litellm/issues/37273. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 964f4b9f68b..6479c43ee7a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -486,8 +486,8 @@ class TestTranslateMessagesToResponsesInput: } ] - def test_assistant_thinking_block_becomes_output_text(self): - """Assistant thinking block text is included as output_text.""" + def test_assistant_thinking_block_becomes_reasoning_item(self): + """Assistant thinking block becomes a reasoning item, never visible assistant prose.""" messages = [ { "role": "assistant", @@ -495,7 +495,65 @@ class TestTranslateMessagesToResponsesInput: } ] result = _translate_messages(messages) - assert result[0]["content"] == [{"type": "output_text", "text": "Let me reason step by step."}] + assert result == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Let me reason step by step."}], + } + ] + + def test_reasoning_item_carries_no_id(self): + """A fabricated reasoning id 404s upstream, so the item must go out without one.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "Private reasoning.", "signature": "rs_abc123"}], + } + ] + result = _translate_messages(messages) + assert "id" not in result[0] + + def test_thinking_blocks_sharing_a_signature_become_one_reasoning_item(self): + """Summary parts of one upstream reasoning item are regrouped by their signature.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "First part.", "signature": "rs_abc123"}, + {"type": "thinking", "thinking": "Second part.", "signature": "rs_abc123"}, + {"type": "thinking", "thinking": "A later item.", "signature": "rs_def456"}, + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "reasoning", + "summary": [ + {"type": "summary_text", "text": "First part."}, + {"type": "summary_text", "text": "Second part."}, + ], + }, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "A later item."}], + }, + ] + + def test_thinking_and_text_stay_separate(self): + """The visible answer stays the only thing in the assistant message.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "The user wants Denver."}, + {"type": "text", "text": "Denver is the best pick."}, + ], + } + ] + result = _translate_messages(messages) + assert [item["type"] for item in result] == ["reasoning", "message"] + assert result[1]["content"] == [{"type": "output_text", "text": "Denver is the best pick."}] def test_assistant_empty_thinking_block_skipped(self): """Assistant thinking block with empty thinking text is skipped.""" @@ -1094,7 +1152,7 @@ def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMo return item -def _make_reasoning_item(summaries: List[str]) -> MagicMock: +def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> MagicMock: """Build a mock ResponseReasoningItem.""" from openai.types.responses import ResponseReasoningItem # type: ignore[import] @@ -1105,6 +1163,7 @@ def _make_reasoning_item(summaries: List[str]) -> MagicMock: summary_mocks.append(s) item = MagicMock(spec=ResponseReasoningItem) + item.id = item_id item.summary = summary_mocks return item @@ -1178,6 +1237,29 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["content"] == [] + def test_reasoning_item_id_becomes_thinking_signature(self): + """The reasoning item id rides back as the signature so the next turn can regroup it.""" + reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert [block["signature"] for block in result["content"]] == ["rs_abc123", "rs_abc123"] + + def test_dict_reasoning_item_becomes_thinking_block(self): + """A reasoning item arriving as a plain dict is kept, not dropped.""" + response = _make_mock_response( + output=[ + { + "type": "reasoning", + "id": "rs_dict_1", + "summary": [{"type": "summary_text", "text": "Weighing the options."}], + } + ] + ) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + {"type": "thinking", "thinking": "Weighing the options.", "signature": "rs_dict_1"} + ] + def test_usage_mapped_correctly(self): """Input/output tokens from ResponseAPIUsage are mapped to AnthropicUsage.""" response = _make_mock_response( From 94caab7302ba471647bfba51bb1e5f8ad4cc5222 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:46:18 -0700 Subject: [PATCH 16/54] fix(interactions): release the reservation for creates nothing will poll, and let OTEL see the settled cost Two review findings, both in the handoff between the create's success callback and the background poll task. The callback deferred its budget reservation release for any interactions response with no usage, but the scheduler only starts a poll task when the status is in_progress and an id is present. A create that came back terminal without usage therefore matched the callback's test, got no poll task, and left its reservation open forever: the pre-call estimate stayed added to the key, user, team and org spend counters, and the key began refusing traffic against budget it had never spent. The two conditions now come from one shared gate so they cannot drift apart again. Settling a background interaction re-runs the success handlers for a second result on the same request, and OTEL dedupes span emission on a marker held in that request's metadata. The in-progress create claimed the marker, so the completion, the only event carrying usage and cost, was dropped as a duplicate by OTEL and by every integration deriving from it. Clearing the success-scoped markers alongside the existing dedup flag lets the cost span through, leaving failure and guardrail markers untouched. --- .../interactions/background_cost_polling.py | 13 ++- litellm/litellm_core_utils/litellm_logging.py | 29 +++++- .../proxy/hooks/proxy_track_cost_callback.py | 5 +- .../test_litellm_logging.py | 38 +++++++ .../hooks/test_proxy_track_cost_callback.py | 99 +++++++++++++++++++ 5 files changed, 180 insertions(+), 4 deletions(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index ccf4c2dd853..262d8e3260d 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -162,6 +162,17 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction") +def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool: + """ + The single gate deciding whether a create's response gets a poll task. + The proxy's success callback defers releasing the budget reservation for + exactly these responses, on the promise that a poll task will settle them, + so a response one site accepts and the other refuses strands its + reservation on the spend counters with nothing left to reconcile it. + """ + return response.status == "in_progress" and bool(response.id) + + @dataclass(frozen=True, slots=True) class _ActiveBackgroundPoll: task: "asyncio.Task[None]" @@ -188,7 +199,7 @@ def maybe_schedule_background_interaction_cost_polling( return None if not isinstance(response, InteractionsAPIResponse): return None - if response.status != "in_progress" or not response.id: + if not is_pollable_background_interaction(response): return None logging_obj = create_kwargs.get("litellm_logging_obj") if not isinstance(logging_obj, Logging): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e7d7213b47c..275803c5aed 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2200,12 +2200,37 @@ class Logging(LiteLLMLoggingBaseClass): Log the terminal result of a background interaction as a fresh success event. The create request already ran success logging for its ``in_progress`` response (no usage, so no cost was tracked); clearing - the dedup flag lets the completed result flow through cost calculation + the dedup flags lets the completed result flow through cost calculation and spend tracking exactly once, spanning create to completion. """ - self.model_call_details.pop("has_logged_async_success", None) + self._reset_success_emission_dedupe() await self.async_success_handler(result=result) + def _reset_success_emission_dedupe(self) -> None: + """ + Success callbacks dedupe per request, because the sync and async + handlers both fire on some paths and would otherwise report one call + twice. A settled background interaction is a genuinely second success + event on the same request, so every such marker has to be cleared or + the completion, the only event that carries usage and cost, is + discarded as a duplicate of the in-progress create. + """ + self.model_call_details.pop("has_logged_async_success", None) + litellm_params = self.model_call_details.get("litellm_params") + if not isinstance(litellm_params, dict): + return + metadata = litellm_params.get("metadata") + if not isinstance(metadata, dict): + return + otel_internal = metadata.get("_otel_internal") + if not isinstance(otel_internal, dict): + return + spans_logged = otel_internal.get("spans_logged") + if not isinstance(spans_logged, dict): + return + for scope in [key for key in spans_logged if isinstance(key, tuple) and key[-1:] == ("success",)]: + del spans_logged[scope] + def _flush_passthrough_collected_chunks_helper( self, raw_bytes: list[bytes], diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 55d3c6c6ed5..46cf62ede1c 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -478,9 +478,12 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: def _is_unbilled_in_progress_interaction(completion_response: object) -> bool: + from litellm.interactions.background_cost_polling import is_pollable_background_interaction from litellm.types.interactions import InteractionsAPIResponse - return isinstance(completion_response, InteractionsAPIResponse) and completion_response.usage is None + if not isinstance(completion_response, InteractionsAPIResponse): + return False + return completion_response.usage is None and is_pollable_background_interaction(completion_response) def _should_track_cost_callback( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 8a682517f64..7c90d31261b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4644,6 +4644,44 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 +@pytest.mark.asyncio +async def test_background_interaction_completion_lets_otel_emit_the_cost_span(): + """ + OTEL, and every integration that derives from it, dedupes span emission on + a marker kept in the request's own metadata. The in-progress create claims + that marker, so without clearing it the settled completion, the only event + carrying usage and cost, is discarded as a duplicate and every + OTEL-family backend shows the interaction as a span with no cost at all. + """ + import datetime as dt + + from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig + from litellm.types.interactions import InteractionsAPIResponse + + otel = OpenTelemetry(config=OpenTelemetryConfig(exporter="console")) + logging_obj = _interactions_logging_obj(stream=False) + in_progress = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress") + await logging_obj.async_success_handler( + result=in_progress, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + ) + + assert otel._emit_once(logging_obj.model_call_details, "success") is True + assert otel._emit_once(logging_obj.model_call_details, "success") is False + + completed = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + await logging_obj.async_log_background_interaction_completion(result=completed) + + assert otel._emit_once(logging_obj.model_call_details, "success") is True + + @pytest.mark.parametrize( "call_type", ["aget", "get", "aget_interaction", "adelete_interaction", "acancel_interaction"], diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 642db7e2d5d..93e0cbec596 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -859,6 +859,105 @@ async def test_track_cost_callback_releases_reservation_for_in_progress_interact mock_proxy_logging.failed_tracking_alert.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status", + ["completed", "failed", "cancelled", "incomplete", "requires_action", "budget_exceeded"], +) +async def test_track_cost_callback_releases_reservation_for_unpollable_interaction(status): + """ + Only an in-progress create gets a poll task, so a create that comes back + terminal with no usage has nobody left to reconcile its reservation. The + callback must release it there and then, or the pre-call estimate stays + added to the key, user, team and org spend counters and starts refusing + traffic against budget that was never actually spent. + """ + from litellm.types.interactions import InteractionsAPIResponse + + logger = _ProxyDBLogger() + reservation = {"reserved_cost": 0.05, "entries": [], "finalized": False} + terminal_response = InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-3-flash-preview", + status=status, + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=_in_progress_interaction_kwargs(reservation), + completion_response=terminal_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_track_cost_callback_releases_reservation_for_interaction_without_an_id(): + """ + The scheduler also refuses a response with no id, since it has nothing to + poll for, so the callback must not defer to a poll task that will never + exist. + """ + from litellm.types.interactions import InteractionsAPIResponse + + logger = _ProxyDBLogger() + reservation = {"reserved_cost": 0.05, "entries": [], "finalized": False} + idless_response = InteractionsAPIResponse( + id="", + model="gemini-3-flash-preview", + status="in_progress", + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=_in_progress_interaction_kwargs(reservation), + completion_response=idless_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert reservation["finalized"] is True + + +@pytest.mark.parametrize( + "status", + ["in_progress", "completed", "failed", "cancelled", "incomplete", "requires_action"], +) +@pytest.mark.parametrize("interaction_id", ["interactions/bg-abc", ""]) +def test_callback_defers_exactly_the_interactions_the_scheduler_polls(status, interaction_id): + """ + Pins the invariant the two modules share: the callback may only hold a + budget reservation open for a response the scheduler will actually poll. + Any drift between the two gates leaks reservations onto live spend + counters, so assert they agree rather than restating either condition. + """ + from litellm.interactions.background_cost_polling import is_pollable_background_interaction + from litellm.proxy.hooks.proxy_track_cost_callback import _is_unbilled_in_progress_interaction + from litellm.types.interactions import InteractionsAPIResponse + + response = InteractionsAPIResponse( + id=interaction_id, + model="gemini-3-flash-preview", + status=status, + ) + + assert _is_unbilled_in_progress_interaction(response) is is_pollable_background_interaction(response) + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_propagates_trace_id_from_logging_obj(): """ From 32bf1aba2958a8601d78ba7f86eb97d58a1adf85 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:27:20 -0700 Subject: [PATCH 17/54] fix(anthropic): stop signing replayed thinking blocks and strip reasoning_content A reasoning item id is not an Anthropic signature. Passing it off as one got the block replayed to Anthropic and Bedrock as if it were real, and every backend that verifies signatures rejected the turn. Thinking blocks now come back unsigned, and the streaming path no longer emits a signature_delta for them. Azure AI Foundry, Fireworks, and vLLM reject unknown message fields, so they now strip reasoning_content alongside thinking_blocks the way Mistral already did. The thinking-block helpers take ChatCompletionThinkingBlock and ChatCompletionRedactedThinkingBlock instead of loose mappings. --- .../transformation.py | 8 +-- .../prompt_templates/common_utils.py | 34 ++++++++----- .../responses_adapters/streaming_iterator.py | 11 ---- .../responses_adapters/transformation.py | 25 ++++++---- litellm/llms/azure_ai/chat/transformation.py | 10 +++- .../llms/fireworks_ai/chat/transformation.py | 1 + .../llms/hosted_vllm/chat/transformation.py | 5 +- ...t_responses_adapters_streaming_iterator.py | 10 +--- .../test_responses_adapters_transformation.py | 50 +++++++++++++------ .../chat/test_azure_ai_transformation.py | 2 + .../test_fireworks_ai_chat_transformation.py | 2 + .../test_hosted_vllm_chat_transformation.py | 2 + 12 files changed, 96 insertions(+), 64 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index ea54bd83e12..3e55c3c637e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -97,10 +97,10 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: stored: Final = [_reasoning_item_to_response_input(r_item) for r_item in _get_reasoning_items(msg)] if stored: return stored - from_thinking: Final = responses_reasoning_item_from_thinking_blocks( - cast(Iterable[Mapping[str, Any]], msg.get("thinking_blocks") or ()) # cast-ok: untyped client thinking blocks - ) - return [from_thinking] if from_thinking is not None else [] + raw_blocks: Final = msg.get("thinking_blocks") or () + blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json + from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks) + return [] if from_thinking is None else [dict(from_thinking)] def _build_reasoning_item( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 4211fbe610a..e0ba2e13c6e 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -28,8 +28,12 @@ from litellm.types.llms.openai import ( ChatCompletionAssistantMessage, ChatCompletionFileObject, ChatCompletionImageObject, + ChatCompletionReasoningItem, + ChatCompletionReasoningSummaryTextBlock, + ChatCompletionRedactedThinkingBlock, ChatCompletionResponseMessage, ChatCompletionTextObject, + ChatCompletionThinkingBlock, ChatCompletionToolParam, ChatCompletionUserMessage, ) @@ -1549,36 +1553,42 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]: return None, message_content +def _readable_thinking_text( + block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, +) -> str: + """The text a chat model can read back, empty for redacted blocks and malformed ones.""" + if block.get("type") != "thinking": + return "" + thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag + return str(thinking or "") + + def reasoning_content_from_thinking_blocks( - thinking_blocks: Iterable[Mapping[str, Any]], + thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], ) -> str: """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect. Redacted blocks carry no readable text, so they contribute nothing. """ - return "\n".join( - text - for block in thinking_blocks - if block.get("type") == "thinking" and (text := str(block.get("thinking") or "")) - ) + return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) def responses_reasoning_item_from_thinking_blocks( - thinking_blocks: Iterable[Mapping[str, Any]], -) -> dict[str, Any] | None: # mutable-ok: API message payload + thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], +) -> ChatCompletionReasoningItem | None: """Build a Responses API `reasoning` input item from Anthropic thinking blocks. The item carries no `id`: the Responses API rejects an empty one and 404s on any id it did not mint itself, while an item without an id is always accepted. """ - summary: Final[list[dict[str, Any]]] = [ # mutable-ok: API message payload - {"type": "summary_text", "text": text} # mutable-ok: API message payload + summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload + ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) for block in thinking_blocks - if block.get("type") == "thinking" and (text := str(block.get("thinking") or "")) + if (text := _readable_thinking_text(block)) ] if not summary: return None - return {"type": "reasoning", "summary": summary} # mutable-ok: API message payload + return ChatCompletionReasoningItem(type="reasoning", summary=summary) def _parse_content_for_reasoning( diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 286edb24b9b..5577d4a9c2d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -189,17 +189,6 @@ class AnthropicResponsesStreamWrapper: block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: return - done_item_type: Final = ( - getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) if item else None - ) - if done_item_type == "reasoning" and item_id: - self._chunk_queue.append( - { - "type": "content_block_delta", - "index": block_idx, - "delta": {"type": "signature_delta", "signature": item_id}, - } - ) self._chunk_queue.append( { "type": "content_block_stop", diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index b1061f86d0b..1a6b0498a52 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -38,7 +38,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, AnthropicUsage, ) -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.llms.openai import ( + ChatCompletionThinkingBlock, + ResponseAPIUsage, + ResponsesAPIResponse, +) class LiteLLMAnthropicToResponsesAPIAdapter: @@ -112,19 +116,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @classmethod def _thinking_blocks_from_reasoning_item( cls, - item_id: str | None, summary: Iterable[object], ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload """Anthropic thinking blocks for one Responses reasoning item. - The reasoning item id rides along as the signature so that a follow-up turn can - regroup the summary parts into the single reasoning item they came from. + The signature stays empty: only Anthropic can sign a thinking block, and a stand-in + value would be replayed as a real one and rejected by every backend that verifies it. """ return tuple( AnthropicResponseContentBlockThinking( type="thinking", thinking=text, - signature=item_id or None, + signature=None, ).model_dump() for part in summary if (text := cls._summary_part_text(part)) @@ -132,10 +135,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str: - """Group consecutive thinking blocks sharing a signature; keep every other block alone.""" + """Group a run of consecutive thinking blocks together; keep every other block alone.""" index, block = indexed_block - signature: Final = block.get("signature") or "" - return f"thinking:{signature}" if block.get("type") == "thinking" else f"block:{index}" + return "thinking" if block.get("type") == "thinking" else f"block:{index}" @classmethod def _assistant_group_to_input_item( @@ -144,7 +146,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: first: Final = group[0] btype: Final = first.get("type") if btype == "thinking": - return responses_reasoning_item_from_thinking_blocks(group) + blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload + reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks) + return None if reasoning_item is None else dict(reasoning_item) if btype == "tool_use": return { # mutable-ok: API message payload "type": "function_call", @@ -559,7 +563,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: for item in response.output: if isinstance(item, ResponseReasoningItem): - content.extend(self._thinking_blocks_from_reasoning_item(item.id, item.summary)) + content.extend(self._thinking_blocks_from_reasoning_item(item.summary)) elif isinstance(item, ResponseOutputMessage): for part in item.content: @@ -594,7 +598,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif item_type == "reasoning": content.extend( self._thinking_blocks_from_reasoning_item( - item.get("id"), cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json ) ) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index bc8ea31ea8c..9e7161120cc 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -30,7 +30,12 @@ class AzureFoundryErrorStrings(str, enum.Enum): SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'" -NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ("thinking_blocks", "provider_specific_fields", "cache_control") +NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ( + "thinking_blocks", + "reasoning_content", + "provider_specific_fields", + "cache_control", +) class AzureAIStudioConfig(OpenAIConfig): @@ -173,7 +178,8 @@ class AzureAIStudioConfig(OpenAIConfig): """ - Azure AI Studio doesn't support content as a list. This handles: 1. Strips message fields that are not part of the OpenAI chat-completions - schema (thinking_blocks, provider_specific_fields, cache_control). + schema (thinking_blocks, reasoning_content, provider_specific_fields, + cache_control). Azure AI Foundry backends set additionalProperties=false and reject these with "Extra inputs are not permitted", which breaks multi-turn Anthropic-format clients that echo thinking blocks back as history. diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index e64237da978..4e9731ef485 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -504,6 +504,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): m = cast(dict, message) m.pop("provider_specific_fields", None) m.pop("thinking_blocks", None) + m.pop("reasoning_content", None) return messages diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 46a2320b655..29dc485732f 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -164,12 +164,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): """ Support translating: - video files from file_id or file_data to video_url - - thinking_blocks on assistant messages are removed, and content lists - are converted to strings for vLLM compatibility + - thinking_blocks and reasoning_content on assistant messages are removed, + and content lists are converted to strings for vLLM compatibility """ for message in messages: if message["role"] == "assistant": message.pop("thinking_blocks", None) + message.pop("reasoning_content", None) existing_content = message.get("content") if isinstance(existing_content, list): text_parts = [] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 12a6ab4324f..aebbed88c70 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -139,7 +139,6 @@ class TestReasoningItemWithoutSummaryText: ("content_block_start", 0), ("content_block_delta", 0), ("content_block_delta", 0), - ("content_block_delta", 0), ("content_block_stop", 0), ("content_block_start", 1), ("content_block_delta", 1), @@ -148,15 +147,10 @@ class TestReasoningItemWithoutSummaryText: assert chunks[1]["content_block"] == {"type": "thinking", "thinking": "", "signature": ""} assert "".join(c["delta"]["thinking"] for c in chunks[2:4]) == "Weighing options" - def test_reasoning_item_id_is_streamed_as_the_thinking_signature(self): + def test_the_reasoning_item_id_is_never_streamed_as_a_signature(self): + """A stand-in signature would be replayed as a real one, so none is ever sent.""" chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=["Weighing options"])) - signature_deltas = [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] - assert [(c["index"], c["delta"]["signature"]) for c in signature_deltas] == [(0, "rs_1")] - - def test_no_signature_delta_without_a_thinking_block(self): - chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=[])) - assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 6479c43ee7a..790bcd269e0 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -513,15 +513,14 @@ class TestTranslateMessagesToResponsesInput: result = _translate_messages(messages) assert "id" not in result[0] - def test_thinking_blocks_sharing_a_signature_become_one_reasoning_item(self): - """Summary parts of one upstream reasoning item are regrouped by their signature.""" + def test_consecutive_thinking_blocks_become_one_reasoning_item(self): + """Summary parts of one upstream reasoning item are regrouped into that item.""" messages = [ { "role": "assistant", "content": [ - {"type": "thinking", "thinking": "First part.", "signature": "rs_abc123"}, - {"type": "thinking", "thinking": "Second part.", "signature": "rs_abc123"}, - {"type": "thinking", "thinking": "A later item.", "signature": "rs_def456"}, + {"type": "thinking", "thinking": "First part."}, + {"type": "thinking", "thinking": "Second part."}, ], } ] @@ -533,13 +532,26 @@ class TestTranslateMessagesToResponsesInput: {"type": "summary_text", "text": "First part."}, {"type": "summary_text", "text": "Second part."}, ], - }, - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "A later item."}], - }, + } ] + def test_a_tool_call_splits_the_reasoning_items_around_it(self): + """Thinking on either side of a tool call belongs to two different reasoning items.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Before the call."}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Denver"}}, + {"type": "thinking", "thinking": "After the call."}, + ], + } + ] + result = _translate_messages(messages) + assert [item["type"] for item in result] == ["reasoning", "function_call", "reasoning"] + assert result[0]["summary"] == [{"type": "summary_text", "text": "Before the call."}] + assert result[2]["summary"] == [{"type": "summary_text", "text": "After the call."}] + def test_thinking_and_text_stay_separate(self): """The visible answer stays the only thing in the assistant message.""" messages = [ @@ -1237,12 +1249,12 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["content"] == [] - def test_reasoning_item_id_becomes_thinking_signature(self): - """The reasoning item id rides back as the signature so the next turn can regroup it.""" + def test_reasoning_item_id_never_becomes_a_thinking_signature(self): + """Only Anthropic can sign a thinking block, so a stand-in signature is never invented.""" reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") response = _make_mock_response(output=[reasoning]) result: Any = _ADAPTER.translate_response(response) - assert [block["signature"] for block in result["content"]] == ["rs_abc123", "rs_abc123"] + assert [block["signature"] for block in result["content"]] == [None, None] def test_dict_reasoning_item_becomes_thinking_block(self): """A reasoning item arriving as a plain dict is kept, not dropped.""" @@ -1257,9 +1269,19 @@ class TestTranslateResponse: ) result: Any = _ADAPTER.translate_response(response) assert result["content"] == [ - {"type": "thinking", "thinking": "Weighing the options.", "signature": "rs_dict_1"} + {"type": "thinking", "thinking": "Weighing the options.", "signature": None} ] + def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): + """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _drop_unsignable_thinking_blocks, + ) + + response = _make_mock_response(output=[_make_reasoning_item(["Part one."], item_id="rs_abc123")]) + result: Any = _ADAPTER.translate_response(response) + assert _drop_unsignable_thinking_blocks(result["content"]) == [] + def test_usage_mapped_correctly(self): """Input/output tokens from ResponseAPIUsage are mapped to AnthropicUsage.""" response = _make_mock_response( diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index 0fd9a381a5a..d4fcbc823a6 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -300,6 +300,7 @@ def test_azure_ai_strips_non_openai_spec_message_fields(): "cache_control": {"type": "ephemeral"}, } ], + "reasoning_content": "The user wants me to read a file.", "provider_specific_fields": {"thought_signature": "sig-top"}, "tool_calls": [ { @@ -327,6 +328,7 @@ def test_azure_ai_strips_non_openai_spec_message_fields(): transformed_messages = request["messages"] assert not _find_key_anywhere(transformed_messages, "thinking_blocks") + assert not _find_key_anywhere(transformed_messages, "reasoning_content") assert not _find_key_anywhere(transformed_messages, "provider_specific_fields") assert not _find_key_anywhere(transformed_messages, "cache_control") diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index e728fc4bc40..63a749dab84 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -473,12 +473,14 @@ def test_transform_messages_helper_strips_thinking_blocks(): "thinking_blocks": [ {"type": "thinking", "thinking": "internal", "signature": ""} ], + "reasoning_content": "internal", }, ] out = config._transform_messages_helper( messages, model="accounts/fireworks/models/glm-5p1", litellm_params={} ) assert "thinking_blocks" not in out[1] + assert "reasoning_content" not in out[1] assert out[1]["content"] == "I can help." diff --git a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index e316cd14dd4..82b05601a85 100644 --- a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -200,6 +200,7 @@ def test_hosted_vllm_thinking_blocks_prepended_to_assistant_content(): "signature": "abc123", } ], + "reasoning_content": "Let me reason about this...", }, { "role": "user", @@ -218,6 +219,7 @@ def test_hosted_vllm_thinking_blocks_prepended_to_assistant_content(): assert isinstance(assistant_msg["content"], str) assert assistant_msg["content"] == "Here is my answer." assert "thinking_blocks" not in assistant_msg + assert "reasoning_content" not in assistant_msg def test_hosted_vllm_thinking_blocks_with_list_content(): From 6befeb8a17f93bd65b028fdd0b8c4c3087c92a1e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:48:32 -0700 Subject: [PATCH 18/54] fix(interactions): stop the cost poll loop instead of spinning on a non-positive interval --- litellm/interactions/background_cost_polling.py | 2 +- .../interactions/test_background_cost_polling.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 262d8e3260d..f2957d08840 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -74,7 +74,7 @@ async def _fetch_interaction(context: BackgroundInteractionPollContext) -> Inter def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]: elapsed = 0.0 interval = initial - while elapsed + interval <= timeout: + while interval > 0 and elapsed + interval <= timeout: yield interval elapsed += interval interval = min(interval * 2, maximum) diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py index 64436ebe4a7..7908e0e8f17 100644 --- a/tests/test_litellm/interactions/test_background_cost_polling.py +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -1,11 +1,13 @@ import asyncio import time +from itertools import islice from typing import Optional import pytest from litellm.interactions.background_cost_polling import ( _SETTLED_KEY, + _poll_intervals, BackgroundInteractionPollContext, maybe_schedule_background_interaction_cost_polling, maybe_settle_background_interaction_before_delete, @@ -92,6 +94,17 @@ def _fetch_sequence(*responses): return fetch, calls +@pytest.mark.parametrize( + "initial, maximum", + [(0.0, 0.002), (0.001, 0.0), (-1.0, 0.002), (0.0, 0.0)], +) +def test_poll_intervals_stops_instead_of_looping_on_a_non_positive_interval(initial, maximum): + intervals = list(islice(_poll_intervals(initial=initial, maximum=maximum, timeout=3600.0), 10)) + + assert len(intervals) < 10 + assert all(interval > 0 for interval in intervals) + + @pytest.mark.asyncio async def test_poller_bills_once_when_interaction_completes(): logging_obj = _logging_obj() From 5317a5ab5019c1d40c2ae4994876e2ed6a81b581 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:53:18 -0700 Subject: [PATCH 19/54] test: cover the reverse bridge on an assistant message that precedes its function_call --- ..._tool_output_order_preserved_for_gemini.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py index 3a1c77d1dab..3dec1d571c4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py @@ -114,3 +114,46 @@ def test_assistant_message_after_tool_call_is_folded_into_it(): tool_call_idx = next(i for i, m in enumerate(msgs) if isinstance(m, dict) and m.get("tool_calls")) assert msgs[tool_call_idx].get("role") == "assistant" assert msgs[tool_call_idx + 1].get("role") == "tool" + + +def test_assistant_message_before_function_call_keeps_one_assistant_turn(): + """The chat->responses bridge emits an assistant message ahead of its function_call. + + Round-tripping that order back to chat must fold both into a single assistant + turn, so the tool result still follows the message that made the call. + """ + msgs = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=[ + { + "role": "user", + "type": "message", + "content": [{"type": "input_text", "text": "What is the weather?"}], + }, + { + "role": "assistant", + "type": "message", + "content": [{"type": "output_text", "text": "Let me check."}], + }, + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_1", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": "sunny", + }, + ] + ) + + assistant_msgs = [m for m in msgs if isinstance(m, dict) and m.get("role") == "assistant"] + assert len(assistant_msgs) == 1 + assistant = assistant_msgs[0] + assert assistant["content"] == [{"type": "text", "text": "Let me check."}] + assert [tc["function"]["name"] for tc in assistant["tool_calls"]] == ["get_weather"] + + assistant_idx = msgs.index(assistant) + assert msgs[assistant_idx + 1].get("role") == "tool" + assert msgs[assistant_idx + 1].get("tool_call_id") == "call_1" From 8b566a7f0aa3e1c935c0d84e646379a505d6904d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:28:12 -0700 Subject: [PATCH 20/54] fix(interactions): stop two settlement paths from pinning the budget reservation Both leave a background interaction's pre-call reservation open, so the serving process keeps refusing traffic on the key at the estimated cost while its recorded spend stays near zero. A raise from the completion event propagated out with the settlement gate already claimed, and nothing retries a claim that is set, so the reservation was never released. Billing now releases it on the way out. `requires_action` was missing from the terminal set. It is terminal for the interaction it names: the API has no operation that resumes one, and a caller answers a tool request by creating a new interaction whose `previous_interaction_id` points at it. A function-calling background create that stopped there was polled until the 3600s timeout, losing the tokens it had already spent producing the tool request and holding its reservation open for that whole window. --- .../interactions/background_cost_polling.py | 28 ++++++- .../test_background_cost_polling.py | 78 +++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index f2957d08840..5d7dca80b73 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -10,6 +10,14 @@ billing: it schedules a poll task that fetches the interaction until it reaches a terminal status and logs the final usage as a single success event attributed to the original request. +``requires_action`` is terminal for the interaction it names. The API has no +operation that resumes one: a caller answers a tool request by creating a new +interaction whose ``previous_interaction_id`` points at it, and that new +interaction bills itself. The paused interaction keeps the tokens it already +spent producing the tool request, so it is billed and settled where it stops +rather than polled until the timeout, which would both lose that usage and +hold its budget reservation open for the whole timeout window. + Deleting an interaction makes every subsequent poll fail, which would let a caller retrieve the completed output themselves and then delete it before the poll task settles, leaving the work unbilled and the budget reservation @@ -39,7 +47,7 @@ from litellm.types.interactions import InteractionsAPIResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -_TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete", "budget_exceeded"}) +_TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}) @dataclass(frozen=True, slots=True) @@ -125,7 +133,7 @@ async def poll_and_log_background_interaction_cost( if not _claim_settlement(context.logging_obj): return if response.usage is not None: - await context.logging_obj.async_log_background_interaction_completion(result=response) + await _bill_settled_interaction(logging_obj=context.logging_obj, response=response) else: await _release_open_budget_reservation(logging_obj=context.logging_obj) return @@ -162,6 +170,20 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction") +async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None: + """ + Claiming the settlement makes the claimer solely responsible for the + reservation, and no one retries a claim that is already set. A billing + failure here must therefore release the reservation on its way out, or it + stays pinned at the estimated cost until the whole poll times out. + """ + try: + await logging_obj.async_log_background_interaction_completion(result=response) + except Exception: + await _release_open_budget_reservation(logging_obj=logging_obj) + raise + + def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool: """ The single gate deciding whether a create's response gets a poll task. @@ -247,6 +269,6 @@ async def maybe_settle_background_interaction_before_delete( if not _claim_settlement(context.logging_obj): return if response.status in _TERMINAL_STATUSES and response.usage is not None: - await context.logging_obj.async_log_background_interaction_completion(result=response) + await _bill_settled_interaction(logging_obj=context.logging_obj, response=response) return await _release_open_budget_reservation(logging_obj=context.logging_obj) diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py index 7908e0e8f17..759bdc71dfc 100644 --- a/tests/test_litellm/interactions/test_background_cost_polling.py +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -59,6 +59,10 @@ def _logging_obj_with_reservation(reservation: dict) -> LitellmLogging: return _logging_obj(litellm_params={"metadata": {"user_api_key_budget_reservation": reservation}}) +async def _raise_on_billing(result: InteractionsAPIResponse) -> None: + raise RuntimeError("cost calculation failed for a settled background interaction") + + def _context(logging_obj: LitellmLogging, timeout_seconds: float = 1.0) -> BackgroundInteractionPollContext: return BackgroundInteractionPollContext( interaction_id="interactions/bg-abc", @@ -120,6 +124,33 @@ async def test_poller_bills_once_when_interaction_completes(): assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 +@pytest.mark.asyncio +async def test_poller_bills_an_interaction_paused_for_a_tool_result(): + logging_obj = _logging_obj() + fetch, calls = _fetch_sequence( + _response("in_progress", with_usage=False), + _response("requires_action", with_usage=True), + ) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert len(calls) == 2 + assert logging_obj.model_call_details["response_cost"] > 0 + assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + + +@pytest.mark.asyncio +async def test_poller_does_not_pin_the_budget_for_an_interaction_paused_for_a_tool_result(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + fetch, _ = _fetch_sequence(_response("requires_action", with_usage=True)) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert logging_obj.model_call_details["response_cost"] > 0 + assert reservation["finalized"] is False + + @pytest.mark.asyncio async def test_poller_stops_without_billing_on_terminal_status_without_usage(): logging_obj = _logging_obj() @@ -170,6 +201,19 @@ async def test_poller_releases_budget_reservation_on_timeout_give_up(): assert reservation["finalized"] is True +@pytest.mark.asyncio +async def test_poller_releases_budget_reservation_when_billing_raises(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + fetch, _ = _fetch_sequence(_response("completed", with_usage=True)) + logging_obj.async_log_background_interaction_completion = _raise_on_billing + + with pytest.raises(RuntimeError): + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert reservation["finalized"] is True + + @pytest.mark.asyncio async def test_poller_leaves_reservation_reconciliation_to_the_completion_event(): reservation = _reservation() @@ -248,6 +292,22 @@ def _register_poll(logging_obj: LitellmLogging, poll_fetch=None) -> asyncio.Task return task +@pytest.mark.asyncio +async def test_delete_settlement_bills_an_interaction_paused_for_a_tool_result(): + logging_obj = _logging_obj() + task = _register_poll(logging_obj) + fetch, calls = _fetch_sequence(_response("requires_action", with_usage=True)) + + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=fetch, + ) + + assert len(calls) == 1 + assert logging_obj.model_call_details["response_cost"] > 0 + await asyncio.wait_for(task, timeout=5) + + @pytest.mark.asyncio async def test_delete_settlement_bills_pending_background_interaction(): logging_obj = _logging_obj() @@ -299,6 +359,24 @@ async def test_delete_settlement_releases_reservation_when_prefetch_fails(): await asyncio.wait_for(task, timeout=5) +@pytest.mark.asyncio +async def test_delete_settlement_releases_reservation_when_billing_raises(): + reservation = _reservation() + logging_obj = _logging_obj_with_reservation(reservation) + task = _register_poll(logging_obj) + fetch, _ = _fetch_sequence(_response("completed", with_usage=True)) + logging_obj.async_log_background_interaction_completion = _raise_on_billing + + with pytest.raises(RuntimeError): + await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", + fetch_interaction=fetch, + ) + + assert reservation["finalized"] is True + await asyncio.wait_for(task, timeout=5) + + @pytest.mark.asyncio async def test_delete_settlement_ignores_interactions_without_pending_poll(): fetch, calls = _fetch_sequence(_response("completed", with_usage=True)) From 68bda9499533545065299c4995f74214d0c92e05 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:53:12 -0700 Subject: [PATCH 21/54] chore: mark the new API payload literals for the type-discipline budget --- .../litellm_responses_transformation/transformation.py | 7 ++++--- .../responses_adapters/streaming_iterator.py | 5 ++++- .../responses_adapters/transformation.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 3e55c3c637e..cf227cfa7d1 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -94,13 +94,14 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: Stored reasoning items win because they carry an id the Responses API minted; thinking blocks are the fallback for turns that arrived over another API surface. """ - stored: Final = [_reasoning_item_to_response_input(r_item) for r_item in _get_reasoning_items(msg)] + items: Final = _get_reasoning_items(msg) + stored: Final = [_reasoning_item_to_response_input(item) for item in items] # mutable-ok: API message payload if stored: return stored raw_blocks: Final = msg.get("thinking_blocks") or () blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return [] if from_thinking is None else [dict(from_thinking)] + return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload def _build_reasoning_item( @@ -393,7 +394,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_items.extend(_reasoning_input_items(msg)) if content: input_items.append( - { + { # mutable-ok: API message payload "type": "message", "role": "assistant", "content": self._convert_content_to_responses_format(content, "assistant"), diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 5577d4a9c2d..292d2622c7f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -152,7 +152,10 @@ class AnthropicResponsesStreamWrapper: if block_idx < 0: if not delta: return - block_idx = self._open_block(item_id, {"type": "thinking", "thinking": "", "signature": ""}) + block_idx = self._open_block( + item_id, + {"type": "thinking", "thinking": "", "signature": ""}, # mutable-ok: API message payload + ) self._chunk_queue.append( { "type": "content_block_delta", diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 1a6b0498a52..23a9a60d810 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -148,13 +148,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if btype == "thinking": blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return None if reasoning_item is None else dict(reasoning_item) + return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload if btype == "tool_use": return { # mutable-ok: API message payload "type": "function_call", "call_id": first.get("id", ""), "name": first.get("name", ""), - "arguments": json.dumps(first.get("input", {})), + "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload } return None From 0ab8ef60bf86f4fbc61359488c0d967f75b01a9b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 22 Aug 2026 22:54:08 +0000 Subject: [PATCH 22/54] fix(interactions): stop the unpollable-create path from firing a false cost-tracking alert The tightened gate correctly stopped deferring the reservation release for InteractionsAPIResponses the scheduler will not poll (terminal status, or in_progress without an id), but the response then fell through into the generic 'Cost tracking failed' raise and the failed_tracking_alert path. A create returning failed, cancelled, requires_action, incomplete, budget_exceeded, or an id-less in_progress without usage therefore released its reservation as intended and, in the same breath, alerted operators for a legitimate no-usage response, both creating noise and masking real cost-tracking failures. The two gates are now nested under a single 'unbilled interaction response' outer check, so any InteractionsAPIResponse with no usage takes either the defer path (pollable, polling on) or the release-and-return path, and none of them fall through to the generic failure raise. The two regression tests also now assert failed_tracking_alert is not called, closing the observation gap the report flagged. --- litellm/proxy/hooks/proxy_track_cost_callback.py | 16 ++++++++++++---- .../hooks/test_proxy_track_cost_callback.py | 9 ++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 46cf62ede1c..70677521d6d 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -319,8 +319,10 @@ class _ProxyDBLogger(CustomLogger): elif budget_reservation is not None: await _release_budget_reservation(budget_reservation=budget_reservation) else: - if _is_unbilled_in_progress_interaction(completion_response): - if BACKGROUND_INTERACTION_COST_POLLING_ENABLED: + if _is_unbilled_interaction_response(completion_response): + if BACKGROUND_INTERACTION_COST_POLLING_ENABLED and _is_unbilled_in_progress_interaction( + completion_response + ): verbose_proxy_logger.debug( "Cost tracking deferred for in-progress background interaction; " "the budget reservation stays open until the poll task logs the final usage" @@ -328,8 +330,8 @@ class _ProxyDBLogger(CustomLogger): return await _release_budget_reservation(budget_reservation=budget_reservation) verbose_proxy_logger.debug( - "Background interaction cost polling is disabled; released the budget " - "reservation for an in-progress interaction that will not be billed" + "Released the budget reservation for an interaction create with no usage " + "that no poll task will settle" ) return await _release_budget_reservation(budget_reservation=budget_reservation) @@ -477,6 +479,12 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: bucket[key] = value +def _is_unbilled_interaction_response(completion_response: object) -> bool: + from litellm.types.interactions import InteractionsAPIResponse + + return isinstance(completion_response, InteractionsAPIResponse) and completion_response.usage is None + + def _is_unbilled_in_progress_interaction(completion_response: object) -> bool: from litellm.interactions.background_cost_polling import is_pollable_background_interaction from litellm.types.interactions import InteractionsAPIResponse diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 93e0cbec596..fd79f9ea3b3 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -871,6 +871,10 @@ async def test_track_cost_callback_releases_reservation_for_unpollable_interacti callback must release it there and then, or the pre-call estimate stays added to the key, user, team and org spend counters and starts refusing traffic against budget that was never actually spent. + + A no-usage terminal create is also not a cost-tracking failure, so the + callback must not fire ``failed_tracking_alert``: doing so would flood + operators with false alerts and mask real cost-tracking failures. """ from litellm.types.interactions import InteractionsAPIResponse @@ -897,6 +901,7 @@ async def test_track_cost_callback_releases_reservation_for_unpollable_interacti ) assert reservation["finalized"] is True + mock_proxy_logging.failed_tracking_alert.assert_not_called() @pytest.mark.asyncio @@ -904,7 +909,8 @@ async def test_track_cost_callback_releases_reservation_for_interaction_without_ """ The scheduler also refuses a response with no id, since it has nothing to poll for, so the callback must not defer to a poll task that will never - exist. + exist, and it must not fire ``failed_tracking_alert`` for what is a + legitimate no-usage response rather than a cost-tracking failure. """ from litellm.types.interactions import InteractionsAPIResponse @@ -931,6 +937,7 @@ async def test_track_cost_callback_releases_reservation_for_interaction_without_ ) assert reservation["finalized"] is True + mock_proxy_logging.failed_tracking_alert.assert_not_called() @pytest.mark.parametrize( From d46b0bddd7f39628e4782b2839ca4ca26291ef51 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 22 Aug 2026 22:58:04 +0000 Subject: [PATCH 23/54] fix: drop null summary text and emit reasoning-only assistant turns --- .../litellm_responses_transformation/transformation.py | 2 ++ .../responses_adapters/transformation.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index cf227cfa7d1..13dd41f9b86 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -434,6 +434,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format(content, cast(str, role)), } ) + elif role == "assistant": + input_items.extend(_reasoning_input_items(msg)) return input_items, instructions diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 23a9a60d810..6d47d0de19f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -110,8 +110,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def _summary_part_text(part: object) -> str: if isinstance(part, Mapping): mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json - return str(mapping.get("text", "")) - return str(getattr(part, "text", "")) + return str(mapping.get("text") or "") + return str(getattr(part, "text", None) or "") @classmethod def _thinking_blocks_from_reasoning_item( From b460254428f5c0ff87a1d9e9b5d30e87536cd1d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:06:52 -0700 Subject: [PATCH 24/54] fix(interactions): price the settled background body against its own deployment The poll fetches the terminal interaction through its own client call, which stamps a response cost computed by a throwaway logging object holding none of the original request's deployment context: no model_info, no router model_id, no deployment litellm_params. Carrying that cost into the settlement event billed custom-priced deployments at the wrong rate, and it also satisfied the "already calculated" shortcut in _response_cost_calculator's caller, so the settlement never repriced and never built a cost breakdown. The zeros stamped by the usage-less create survived into the spend log row and the OTEL span. Dropping the imported cost before re-emitting makes the settlement price the settled body itself, against the deployment that served the create. --- litellm/litellm_core_utils/litellm_logging.py | 13 ++++++ .../test_litellm_logging.py | 43 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 275803c5aed..853df91cb89 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2202,7 +2202,20 @@ class Logging(LiteLLMLoggingBaseClass): ``in_progress`` response (no usage, so no cost was tracked); clearing the dedup flags lets the completed result flow through cost calculation and spend tracking exactly once, spanning create to completion. + + The poll fetched this body through its own client call, which priced it + against a throwaway logging object holding none of this request's + deployment context: no ``model_info``, no router ``model_id``, no + deployment ``litellm_params``. Keeping that price would bill a + custom-priced deployment at the wrong rate, and it would also satisfy + the "already calculated" shortcut and skip repricing here, leaving the + cost breakdown at the zeros the usage-less create stamped and writing + those zeros to the spend log. Dropping it makes this event price the + settled body itself, against the deployment that served the create. """ + settled_hidden_params: Final = getattr(result, "_hidden_params", None) + if isinstance(settled_hidden_params, dict): + settled_hidden_params.pop("response_cost", None) self._reset_success_emission_dedupe() await self.async_success_handler(result=result) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 7c90d31261b..a7db8ae7a28 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4644,6 +4644,49 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 +@pytest.mark.asyncio +async def test_background_interaction_completion_prices_the_settled_body_itself(): + """ + The poll fetches the settled body through its own client call, which + prices it against a throwaway logging object holding none of this + request's deployment context. Adopting that price would bill a + custom-priced deployment at the wrong rate, and it would also satisfy the + "already calculated" shortcut and skip repricing, leaving the breakdown at + the zeros the usage-less create stamped and writing those to the spend log. + """ + import datetime as dt + + from litellm.types.interactions import InteractionsAPIResponse + + logging_obj = _interactions_logging_obj(stream=False) + in_progress = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress") + await logging_obj.async_success_handler( + result=in_progress, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + ) + + completed = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + completed._hidden_params = {"response_cost": 99.0} + + await logging_obj.async_log_background_interaction_completion(result=completed) + + response_cost = logging_obj.model_call_details["response_cost"] + assert response_cost != 99.0 + assert response_cost > 0 + + cost_breakdown = logging_obj.model_call_details["standard_logging_object"]["cost_breakdown"] + assert cost_breakdown["total_cost"] == response_cost + assert cost_breakdown["input_cost"] > 0 + assert cost_breakdown["output_cost"] > 0 + + @pytest.mark.asyncio async def test_background_interaction_completion_lets_otel_emit_the_cost_span(): """ From 521dc973c302e7672ed8cbf1d8607a7cb1ba72a5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:15:14 -0700 Subject: [PATCH 25/54] fix(interactions): keep alerting when an interaction that produced output has no usage Silencing the cost-tracking alert for every usage-less interaction response went one status too far. An interaction that stopped at failed, cancelled, incomplete or budget_exceeded genuinely has nothing to charge for, so alerting on it is noise. completed and requires_action are different: both mean the model produced output, so a usage block is always expected, and one arriving without it means the charge for real work was lost. That is precisely the case failed_tracking_alert exists to surface, and swallowing it would let an operator's interactions bill nothing with no signal that anything was wrong. The status knowledge lives next to the other status predicates rather than in the proxy callback. The reservation is still released on both paths, since suppressing the alert was never what freed it. --- .../interactions/background_cost_polling.py | 17 +++++++ .../proxy/hooks/proxy_track_cost_callback.py | 5 +- .../hooks/test_proxy_track_cost_callback.py | 50 +++++++++++++++++-- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 5d7dca80b73..279d2d6b32e 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -49,6 +49,8 @@ if TYPE_CHECKING: _TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}) +_STATUSES_THAT_PRODUCED_OUTPUT = frozenset({"completed", "requires_action"}) + @dataclass(frozen=True, slots=True) class BackgroundInteractionPollContext: @@ -195,6 +197,21 @@ def is_pollable_background_interaction(response: InteractionsAPIResponse) -> boo return response.status == "in_progress" and bool(response.id) +def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool: + """ + Whether a response arriving with no usage block is a normal outcome rather + than lost billing data. An interaction that is still running, or that + stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``, + has nothing to charge for and should not raise a cost-tracking alarm. + + ``completed`` and ``requires_action`` both mean the model produced output, + so a usage block is always expected with them. If one arrives without it + the charge for real work has been lost, which is precisely what the + proxy's cost-tracking alert exists to surface. + """ + return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT + + @dataclass(frozen=True, slots=True) class _ActiveBackgroundPoll: task: "asyncio.Task[None]" diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 70677521d6d..6abfca1d3a0 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -480,9 +480,12 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: def _is_unbilled_interaction_response(completion_response: object) -> bool: + from litellm.interactions.background_cost_polling import missing_usage_is_expected from litellm.types.interactions import InteractionsAPIResponse - return isinstance(completion_response, InteractionsAPIResponse) and completion_response.usage is None + if not isinstance(completion_response, InteractionsAPIResponse): + return False + return completion_response.usage is None and missing_usage_is_expected(completion_response) def _is_unbilled_in_progress_interaction(completion_response: object) -> bool: diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index fd79f9ea3b3..d840611fc7b 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -862,7 +862,7 @@ async def test_track_cost_callback_releases_reservation_for_in_progress_interact @pytest.mark.asyncio @pytest.mark.parametrize( "status", - ["completed", "failed", "cancelled", "incomplete", "requires_action", "budget_exceeded"], + ["failed", "cancelled", "incomplete", "budget_exceeded"], ) async def test_track_cost_callback_releases_reservation_for_unpollable_interaction(status): """ @@ -872,9 +872,10 @@ async def test_track_cost_callback_releases_reservation_for_unpollable_interacti added to the key, user, team and org spend counters and starts refusing traffic against budget that was never actually spent. - A no-usage terminal create is also not a cost-tracking failure, so the - callback must not fire ``failed_tracking_alert``: doing so would flood - operators with false alerts and mask real cost-tracking failures. + None of these statuses produced output, so their missing usage is a normal + outcome rather than a cost-tracking failure, and the callback must not fire + ``failed_tracking_alert``: doing so would flood operators with false alerts + and mask real cost-tracking failures. """ from litellm.types.interactions import InteractionsAPIResponse @@ -904,6 +905,47 @@ async def test_track_cost_callback_releases_reservation_for_unpollable_interacti mock_proxy_logging.failed_tracking_alert.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["completed", "requires_action"]) +async def test_track_cost_callback_alerts_when_an_interaction_that_produced_output_has_no_usage(status): + """ + ``completed`` and ``requires_action`` both mean the model produced output, + so a usage block is always expected with them. One arriving without it + means the charge for real work was lost, which is exactly what the + cost-tracking alert is for: silencing it here would let an operator's + interactions bill nothing with no signal that anything went wrong. + + The reservation still has to be released, since suppressing the alert was + never what freed it. + """ + from litellm.types.interactions import InteractionsAPIResponse + + logger = _ProxyDBLogger() + reservation = {"reserved_cost": 0.05, "entries": [], "finalized": False} + usageless_response = InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-3-flash-preview", + status=status, + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=_in_progress_interaction_kwargs(reservation), + completion_response=usageless_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert reservation["finalized"] is True + mock_proxy_logging.failed_tracking_alert.assert_called_once() + + @pytest.mark.asyncio async def test_track_cost_callback_releases_reservation_for_interaction_without_an_id(): """ From 8a1fe281fdbc0d63c272febd634741cc14005295 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:29:35 -0700 Subject: [PATCH 26/54] fix(responses-adapter): skip null reasoning summary text and keep thinking-only assistant turns --- .../transformation.py | 4 +++- .../responses_adapters/transformation.py | 4 ++-- ...responses_transformation_transformation.py | 22 +++++++++++++++++++ .../test_responses_adapters_transformation.py | 14 ++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index cf227cfa7d1..b94e91b3034 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -428,12 +428,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if role == "assistant": input_items.extend(_reasoning_input_items(msg)) input_items.append( - { + { # mutable-ok: API message payload "type": "message", "role": role, "content": self._convert_content_to_responses_format(content, cast(str, role)), } ) + elif role == "assistant": + input_items.extend(_reasoning_input_items(msg)) return input_items, instructions diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 23a9a60d810..0b38123e787 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -110,8 +110,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def _summary_part_text(part: object) -> str: if isinstance(part, Mapping): mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json - return str(mapping.get("text", "")) - return str(getattr(part, "text", "")) + return str(mapping.get("text") or "") + return str(getattr(part, "text", "") or "") @classmethod def _thinking_blocks_from_reasoning_item( diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 124dc67b4fd..1fb74b2b7bf 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3824,6 +3824,28 @@ def test_assistant_thinking_blocks_become_a_reasoning_input_item(): assert "id" not in reasoning_item +def test_thinking_only_assistant_turn_still_sends_its_reasoning(): + """An assistant turn can be pure reasoning, with no visible text and no tool call.""" + handler = LiteLLMResponsesTransformationHandler() + messages = [ + {"role": "user", "content": "What is the weather in Denver?"}, + { + "role": "assistant", + "content": None, + "thinking_blocks": [ + {"type": "thinking", "thinking": "August in Denver is dry.", "signature": "sig1"} + ], + }, + {"role": "user", "content": "Why?"}, + ] + + input_items, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + reasoning_items = [item for item in input_items if item.get("type") == "reasoning"] + assert len(reasoning_items) == 1 + assert reasoning_items[0]["summary"] == [{"type": "summary_text", "text": "August in Denver is dry."}] + + def test_stored_reasoning_items_win_over_thinking_blocks(): """A minted reasoning id beats a re-derived one, so the two must not both be sent.""" handler = LiteLLMResponsesTransformationHandler() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 790bcd269e0..8225e7cff39 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1249,6 +1249,20 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["content"] == [] + def test_null_summary_text_skipped_rather_than_stringified(self): + """A summary part whose text is null must not reach the client as the word "None".""" + response = _make_mock_response( + output=[ + { + "type": "reasoning", + "id": "rs_null_1", + "summary": [{"type": "summary_text", "text": None}], + } + ] + ) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + def test_reasoning_item_id_never_becomes_a_thinking_signature(self): """Only Anthropic can sign a thinking block, so a stand-in signature is never invented.""" reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") From a41ac5c1392171e59302926d88c70346fb1e1ea3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 17:35:19 -0700 Subject: [PATCH 27/54] fix(interactions): poll queued background creates and drop the poll's deployment identity `queued` is the Interactions API's not-started-yet status. It was in neither the pollable set nor the terminal one, so a create returning it got no poll task, counted as a response with nothing to charge for, and released its budget reservation: billed nowhere, alerting nobody. Poll it alongside `in_progress`, and pin the union of the pollable and terminal sets against the generated spec enum so a status Google adds later fails CI rather than shipping another unbilled path. A give-up on a status in neither set now names the status and logs at error, instead of the warning that reads as an interaction merely still running. Also drop `model_id` and `litellm_model_name` from the settled body next to the foreign `response_cost` already dropped there. All three come from the poll's own throwaway client call, and left in place the two identity fields overwrite the create's real deployment in the payload every logging integration reads. Rewrites the callback's per-status test to assert the observable outcome (reservation held vs released) across all eight statuses rather than comparing the gate to the function it delegates to, and pins the shipped 5-10-20-40-60 poll backoff and its timeout cutoff. --- .../interactions/background_cost_polling.py | 40 ++++++-- litellm/litellm_core_utils/litellm_logging.py | 8 +- .../test_background_cost_polling.py | 95 +++++++++++++++++++ .../hooks/test_proxy_track_cost_callback.py | 76 +++++++++++---- 4 files changed, 189 insertions(+), 30 deletions(-) diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 279d2d6b32e..51325354e7d 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -32,7 +32,7 @@ the interaction is billed exactly once no matter who settles first. import asyncio from collections.abc import Awaitable, Callable, Iterator, Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias from litellm._logging import verbose_logger from litellm.constants import ( @@ -47,9 +47,13 @@ from litellm.types.interactions import InteractionsAPIResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -_TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}) +_TERMINAL_STATUSES: Final = frozenset( + {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"} +) -_STATUSES_THAT_PRODUCED_OUTPUT = frozenset({"completed", "requires_action"}) +_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"}) + +_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"}) @dataclass(frozen=True, slots=True) @@ -113,6 +117,7 @@ async def poll_and_log_background_interaction_cost( context: BackgroundInteractionPollContext, fetch_interaction: FetchInteraction = _fetch_interaction, ) -> None: + last_seen_status: str | None = None for interval in _poll_intervals( initial=context.initial_interval_seconds, maximum=context.max_interval_seconds, @@ -130,6 +135,7 @@ async def poll_and_log_background_interaction_cost( e, ) continue + last_seen_status = response.status if response.status not in _TERMINAL_STATUSES: continue if not _claim_settlement(context.logging_obj): @@ -141,11 +147,21 @@ async def poll_and_log_background_interaction_cost( return if not _claim_settlement(context.logging_obj): return - verbose_logger.warning( - "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked", - context.interaction_id, - context.timeout_seconds, - ) + if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES: + verbose_logger.error( + "Gave up cost polling for background interaction %s after %ss: its last status %r is in neither " + "the pollable nor the terminal set, so this proxy never learned how to settle it and its usage " + "will not be tracked", + context.interaction_id, + context.timeout_seconds, + last_seen_status, + ) + else: + verbose_logger.warning( + "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked", + context.interaction_id, + context.timeout_seconds, + ) await _release_open_budget_reservation(logging_obj=context.logging_obj) @@ -193,8 +209,14 @@ def is_pollable_background_interaction(response: InteractionsAPIResponse) -> boo exactly these responses, on the promise that a poll task will settle them, so a response one site accepts and the other refuses strands its reservation on the spend counters with nothing left to reconcile it. + + ``queued`` belongs here alongside ``in_progress``. It is the API's + not-started-yet state, so it reaches a terminal status the same way and + needs polling for the same reason: nothing else in the proxy ever bills a + create that came back without usage, so a status missing from both this + set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody. """ - return response.status == "in_progress" and bool(response.id) + return response.status in _POLLABLE_STATUSES and bool(response.id) def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 853df91cb89..cdc12896678 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2212,10 +2212,16 @@ class Logging(LiteLLMLoggingBaseClass): cost breakdown at the zeros the usage-less create stamped and writing those zeros to the spend log. Dropping it makes this event price the settled body itself, against the deployment that served the create. + + The same throwaway call stamped the deployment identity that travels + with the price, so ``model_id`` and ``litellm_model_name`` go with it. + Left in place they overwrite the create's real deployment with the + poll's empty one in the payload every logging integration reads. """ settled_hidden_params: Final = getattr(result, "_hidden_params", None) if isinstance(settled_hidden_params, dict): - settled_hidden_params.pop("response_cost", None) + for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"): + settled_hidden_params.pop(poll_scoped_key, None) self._reset_success_emission_dedupe() await self.async_success_handler(result=result) diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py index 759bdc71dfc..97f09de1b52 100644 --- a/tests/test_litellm/interactions/test_background_cost_polling.py +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -448,3 +448,98 @@ async def test_schedule_respects_kill_switch(monkeypatch): ) assert task is None + + +def test_every_status_the_api_can_return_is_either_pollable_or_terminal(): + """ + The proxy bills a usage-less create in exactly two ways: it polls the + interaction until it settles, or it recognises the status as terminal and + settles immediately. A status in neither set is billed by nobody, alerts + nobody, and releases its budget reservation, which is the zero-spend bug + this whole module exists to fix. + + Pinned against the generated spec enum rather than a hand-written list, so + a status Google adds later breaks this test instead of silently shipping + another unbilled path. + """ + from litellm.interactions.background_cost_polling import _POLLABLE_STATUSES, _TERMINAL_STATUSES + from litellm.types.interactions.generated import Status1 + + spec_statuses = {member.value for member in Status1} + handled = _POLLABLE_STATUSES | _TERMINAL_STATUSES + + assert spec_statuses - handled == set() + assert handled - spec_statuses == set() + + +@pytest.mark.asyncio +async def test_schedule_creates_poll_task_for_queued_create(): + """ + ``queued`` is the API's not-started-yet state. It carries no usage, so the + create cannot bill it, and it is not terminal, so nothing settles it: + without a poll task it is never charged at all. + """ + logging_obj = _logging_obj() + task = maybe_schedule_background_interaction_cost_polling( + response=_response("queued", with_usage=False), + create_kwargs={"litellm_logging_obj": logging_obj}, + custom_llm_provider="gemini", + ) + + assert isinstance(task, asyncio.Task) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +@pytest.mark.asyncio +async def test_poller_bills_an_interaction_that_started_out_queued(): + logging_obj = _logging_obj() + fetch, calls = _fetch_sequence( + _response("queued", with_usage=False), + _response("in_progress", with_usage=False), + _response("completed", with_usage=True), + ) + + await poll_and_log_background_interaction_cost(_context(logging_obj), fetch_interaction=fetch) + + assert len(calls) == 3 + assert logging_obj.model_call_details["response_cost"] > 0 + assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + + +def test_poll_intervals_double_up_to_the_cap_and_stay_inside_the_timeout(): + """ + The degenerate cases are covered above; this pins the shape the proxy + actually ships, so an off-by-one in the doubling or in the remaining-budget + check cannot pass green. + """ + intervals = list(_poll_intervals(initial=5.0, maximum=60.0, timeout=3600.0)) + + assert intervals[:6] == [5.0, 10.0, 20.0, 40.0, 60.0, 60.0] + assert max(intervals) == 60.0 + assert sum(intervals) <= 3600.0 + assert sum(intervals) + 60.0 > 3600.0 + + +@pytest.mark.asyncio +async def test_giving_up_on_an_unrecognized_status_says_which_status_it_was(monkeypatch): + """ + A status outside both sets polls for the full timeout and then gives up. + The give-up line is the only trace it leaves, so it has to name the status + rather than reporting it as an interaction that was merely still running. + """ + import litellm.interactions.background_cost_polling as bg + + errors = [] + monkeypatch.setattr(bg.verbose_logger, "error", lambda *args, **kwargs: errors.append(args)) + + logging_obj = _logging_obj() + fetch, _ = _fetch_sequence(_response("halted_for_review", with_usage=False)) + + await poll_and_log_background_interaction_cost( + _context(logging_obj, timeout_seconds=0.01), fetch_interaction=fetch + ) + + assert len(errors) == 1 + assert "halted_for_review" in errors[0] diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index d840611fc7b..ca517474a5c 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -785,11 +785,17 @@ def _in_progress_interaction_kwargs(reservation: dict) -> dict: @pytest.mark.asyncio -async def test_track_cost_callback_keeps_reservation_open_for_in_progress_background_interaction(): +@pytest.mark.parametrize("status", ["in_progress", "queued"]) +async def test_track_cost_callback_keeps_reservation_open_for_in_progress_background_interaction(status): """ The pre-call budget reservation must stay open while a background interaction is in flight, so concurrent creates cannot stack past the budget; the poll task's completion event reconciles it to the actual cost. + + ``queued`` is in flight for the same reason ``in_progress`` is: it has not + reached a terminal status, so releasing its reservation here would drop the + estimate off the spend counters while the interaction is still going to run + and still going to cost money. """ from litellm.types.interactions import InteractionsAPIResponse @@ -798,7 +804,7 @@ async def test_track_cost_callback_keeps_reservation_open_for_in_progress_backgr in_progress_response = InteractionsAPIResponse( id="interactions/bg-abc", model="gemini-3-flash-preview", - status="in_progress", + status=status, ) with patch( @@ -982,29 +988,59 @@ async def test_track_cost_callback_releases_reservation_for_interaction_without_ mock_proxy_logging.failed_tracking_alert.assert_not_called() -@pytest.mark.parametrize( - "status", - ["in_progress", "completed", "failed", "cancelled", "incomplete", "requires_action"], -) -@pytest.mark.parametrize("interaction_id", ["interactions/bg-abc", ""]) -def test_callback_defers_exactly_the_interactions_the_scheduler_polls(status, interaction_id): +@pytest.mark.asyncio +async def test_callback_handles_every_status_the_interactions_api_can_return(): """ - Pins the invariant the two modules share: the callback may only hold a - budget reservation open for a response the scheduler will actually poll. - Any drift between the two gates leaks reservations onto live spend - counters, so assert they agree rather than restating either condition. + Whatever status a usage-less create comes back with, exactly one of two + things has to happen to its budget reservation: the callback holds it open + for a poll task that will settle it, or it releases it on the spot. A + status that falls through both leaves the pre-call estimate pinned to the + key, user, team and org spend counters forever, refusing traffic against + budget nobody spent. + + Driven off the generated spec enum so a status Google adds later fails here + instead of quietly leaking reservations in production. """ - from litellm.interactions.background_cost_polling import is_pollable_background_interaction - from litellm.proxy.hooks.proxy_track_cost_callback import _is_unbilled_in_progress_interaction from litellm.types.interactions import InteractionsAPIResponse + from litellm.types.interactions.generated import Status1 - response = InteractionsAPIResponse( - id=interaction_id, - model="gemini-3-flash-preview", - status=status, - ) + deferred = set() + released = set() - assert _is_unbilled_in_progress_interaction(response) is is_pollable_background_interaction(response) + for status in sorted(member.value for member in Status1): + logger = _ProxyDBLogger() + reservation = {"reserved_cost": 0.05, "entries": [], "finalized": False} + response = InteractionsAPIResponse( + id="interactions/bg-abc", + model="gemini-3-flash-preview", + status=status, + ) + + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging: + mock_proxy_logging.failed_tracking_alert = AsyncMock() + mock_proxy_logging.db_spend_update_writer = MagicMock() + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=_in_progress_interaction_kwargs(reservation), + completion_response=response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + (deferred if reservation["finalized"] is False else released).add(status) + + assert deferred == {"in_progress", "queued"} + assert released == { + "completed", + "requires_action", + "failed", + "cancelled", + "incomplete", + "budget_exceeded", + } @pytest.mark.asyncio From 0697188be40c0f5528b2ca18ec110fd95b55d9f8 Mon Sep 17 00:00:00 2001 From: milan Date: Sun, 23 Aug 2026 03:14:19 +0000 Subject: [PATCH 28/54] refactor(a2a): use direct typed access in protocol binding normalization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/a2a_protocol/card_resolver.py | 8 +++----- tests/test_litellm/a2a_protocol/test_main.py | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/a2a_protocol/card_resolver.py b/litellm/a2a_protocol/card_resolver.py index 8070d88b761..47d6346b9a6 100644 --- a/litellm/a2a_protocol/card_resolver.py +++ b/litellm/a2a_protocol/card_resolver.py @@ -67,11 +67,9 @@ def normalize_agent_card_protocol_bindings(agent_card: "AgentCard") -> "AgentCar case-sensitively against its uppercase TransportProtocol constants and fails with "no compatible transports found." for spec-adjacent casings. """ - interfaces: Final = getattr(agent_card, "supported_interfaces", None) or () - for interface in interfaces: - binding: str = getattr(interface, "protocol_binding", "") or "" - canonical = _CANONICAL_PROTOCOL_BINDINGS.get(binding.lower()) - if canonical is not None and binding != canonical: + for interface in agent_card.supported_interfaces: + canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower()) + if canonical is not None and canonical != interface.protocol_binding: interface.protocol_binding = canonical return agent_card diff --git a/tests/test_litellm/a2a_protocol/test_main.py b/tests/test_litellm/a2a_protocol/test_main.py index 29aa3aaabd7..aa988bc5b99 100644 --- a/tests/test_litellm/a2a_protocol/test_main.py +++ b/tests/test_litellm/a2a_protocol/test_main.py @@ -344,6 +344,7 @@ async def test_lowercase_protocol_binding_in_agent_card_still_gets_a_client(isol response = await _send_message(a2a_client, _send_request("lc")) assert type(response.root.result).__name__ == "Message" + assert a2a_client._litellm_agent_card.supported_interfaces[0].protocol_binding == "JSONRPC" @pytest.mark.asyncio From de0d8ceb25ea4cc3aa1df79b494ca95646a709f8 Mon Sep 17 00:00:00 2001 From: milan Date: Sun, 23 Aug 2026 03:20:53 +0000 Subject: [PATCH 29/54] refactor(a2a): return a normalized card copy instead of mutating the resolved card Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/a2a_protocol/card_resolver.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/a2a_protocol/card_resolver.py b/litellm/a2a_protocol/card_resolver.py index 47d6346b9a6..9b3e4ec13b9 100644 --- a/litellm/a2a_protocol/card_resolver.py +++ b/litellm/a2a_protocol/card_resolver.py @@ -67,11 +67,13 @@ def normalize_agent_card_protocol_bindings(agent_card: "AgentCard") -> "AgentCar case-sensitively against its uppercase TransportProtocol constants and fails with "no compatible transports found." for spec-adjacent casings. """ - for interface in agent_card.supported_interfaces: + normalized: Final = type(agent_card)() + normalized.CopyFrom(agent_card) + for interface in normalized.supported_interfaces: canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower()) - if canonical is not None and canonical != interface.protocol_binding: + if canonical is not None: interface.protocol_binding = canonical - return agent_card + return normalized def get_agent_card_url(agent_card: "AgentCard") -> str | None: From ee935cec230207a549f2939a6a9ca69023c609d6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 24 Aug 2026 09:46:35 -0700 Subject: [PATCH 30/54] refactor(proxy): trim the multi_items comment to the non-obvious clause --- litellm/proxy/common_utils/http_parsing_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 4cb55f6966e..96621b08ba1 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -275,9 +275,7 @@ async def get_form_data(request: Request) -> dict[str, Any]: """ form: Final = await request.form() parsed_form_data: Final[dict[str, Any]] = {} - # multi_items(), not dict(form): a dict drops every value but the last of a repeated key, - # which is the whole array this function exists to rebuild - for key, value in form.multi_items(): + for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat if key.endswith("[]"): clean_key = key[:-2] parsed_form_data.setdefault(clean_key, []).append(value) From 91b2a9c360dab5bedec2076a2735b66d042ea595 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:40:28 -0700 Subject: [PATCH 31/54] fix(proxy): keep video reference normalization within lint budgets --- litellm/proxy/video_endpoints/utils.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 5f508cd02ca..a3406a21573 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -13,6 +13,18 @@ def extract_model_from_target_model_names(target_model_names: Any) -> str | None return target_model_names[0] if target_model_names else None +def _video_reference_to_id(video_ref: object) -> str: + if isinstance(video_ref, dict): + return video_ref.get("id", "") + if not isinstance(video_ref, str): + return "" + try: + parsed_ref: Final = orjson.loads(video_ref) + except orjson.JSONDecodeError: + return video_ref + return parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref + + def pop_video_reference_to_video_id(data: dict[str, Any]) -> None: """ Normalize OpenAI video edit/extension payloads into ``video_id``. @@ -20,18 +32,7 @@ def pop_video_reference_to_video_id(data: dict[str, Any]) -> None: JSON bodies use ``video: {"id": ...}``. Multipart and form-urlencoded bodies may send a bare id string or a JSON-encoded reference object as a string field. """ - video_ref: Final = data.pop("video", {}) - if isinstance(video_ref, dict): - video_id: Final = video_ref.get("id", "") - elif isinstance(video_ref, str): - try: - parsed_ref: Final = orjson.loads(video_ref) - except orjson.JSONDecodeError: - parsed_ref = None - video_id = parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref - else: - video_id = "" - data["video_id"] = video_id + data["video_id"] = _video_reference_to_id(data.pop("video", {})) def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: From fe567bd8467f7e9702f26abd61bc866acf659d10 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:51:14 -0700 Subject: [PATCH 32/54] fix(a2a): speak the 0.3 dialect to servers with mis-cased protocol bindings --- litellm/a2a_protocol/card_resolver.py | 18 +++++-- litellm/a2a_protocol/main.py | 4 +- .../a2a_protocol/test_card_resolver.py | 24 +++++++++ tests/test_litellm/a2a_protocol/test_main.py | 53 +++++++++++++++---- 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/litellm/a2a_protocol/card_resolver.py b/litellm/a2a_protocol/card_resolver.py index 9b3e4ec13b9..25f2e1a9a0d 100644 --- a/litellm/a2a_protocol/card_resolver.py +++ b/litellm/a2a_protocol/card_resolver.py @@ -57,22 +57,32 @@ _CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType( } ) +_LEGACY_PROTOCOL_VERSION: Final = "0.3" -def normalize_agent_card_protocol_bindings(agent_card: "AgentCard") -> "AgentCard": + +def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard": """ - Canonicalize protocolBinding casing on the card's supported interfaces. + Canonicalize the supported interfaces of spec-adjacent agent cards. Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings case-sensitively against its uppercase TransportProtocol constants and fails with "no compatible transports found." for spec-adjacent casings. + + The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated + payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1 + proto parsing rejects. A mis-cased binding fingerprints such a server, so its + declared version is downgraded to 0.3 to route the SDK's ClientFactory onto + its v0.3 compat transport, which speaks that dialect. """ normalized: Final = type(agent_card)() normalized.CopyFrom(agent_card) for interface in normalized.supported_interfaces: canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower()) - if canonical is not None: - interface.protocol_binding = canonical + if canonical is None or canonical == interface.protocol_binding: + continue + interface.protocol_binding = canonical + interface.protocol_version = _LEGACY_PROTOCOL_VERSION return normalized diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 2d4267cc0db..56b8089b0af 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -73,7 +73,7 @@ except ImportError: from litellm.a2a_protocol.card_resolver import ( LiteLLMA2ACardResolver, get_agent_card_url, - normalize_agent_card_protocol_bindings, + normalize_agent_card_interfaces, ) from litellm.a2a_protocol.exception_mapping_utils import ( handle_a2a_localhost_retry, @@ -784,7 +784,7 @@ async def create_a2a_client( verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys())) resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url) - agent_card: Final = normalize_agent_card_protocol_bindings( + agent_card: Final = normalize_agent_card_interfaces( await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None) ) diff --git a/tests/test_litellm/a2a_protocol/test_card_resolver.py b/tests/test_litellm/a2a_protocol/test_card_resolver.py index 053f28c940f..5cbfa51fa08 100644 --- a/tests/test_litellm/a2a_protocol/test_card_resolver.py +++ b/tests/test_litellm/a2a_protocol/test_card_resolver.py @@ -13,6 +13,7 @@ from litellm.a2a_protocol.card_resolver import ( LiteLLMA2ACardResolver, fix_agent_card_url, is_localhost_or_internal_url, + normalize_agent_card_interfaces, set_agent_card_url, ) @@ -114,3 +115,26 @@ def test_fix_agent_card_url_updates_interface_when_top_level_is_localhost(): assert result.url == "https://my-public-agent.example.com/" assert result.supported_interfaces[0].url == "https://my-public-agent.example.com/" + + +def test_normalize_agent_card_interfaces_downgrades_miscased_interfaces_to_the_0_3_dialect(): + pb2 = pytest.importorskip("a2a.types.a2a_pb2") + + card = pb2.AgentCard( + name="langgraph", + supported_interfaces=[ + pb2.AgentInterface(url="http://a/", protocol_binding="jsonrpc", protocol_version="1.0"), + pb2.AgentInterface(url="http://b/", protocol_binding="JSONRPC", protocol_version="1.0"), + pb2.AgentInterface(url="http://c/", protocol_binding="websocket", protocol_version="1.0"), + ], + ) + + normalized = normalize_agent_card_interfaces(card) + + assert [(i.protocol_binding, i.protocol_version) for i in normalized.supported_interfaces] == [ + ("JSONRPC", "0.3"), + ("JSONRPC", "1.0"), + ("websocket", "1.0"), + ] + assert card.supported_interfaces[0].protocol_binding == "jsonrpc" + assert card.supported_interfaces[0].protocol_version == "1.0" diff --git a/tests/test_litellm/a2a_protocol/test_main.py b/tests/test_litellm/a2a_protocol/test_main.py index aa988bc5b99..8850a2eca6c 100644 --- a/tests/test_litellm/a2a_protocol/test_main.py +++ b/tests/test_litellm/a2a_protocol/test_main.py @@ -176,10 +176,40 @@ _AGENT_A_HEADERS = {"x-agent-token": "token-for-a", "x-tenant": "tenant-a"} _AGENT_B_HEADERS = {"x-agent-token": "token-for-b", "x-tenant": "tenant-b"} -_V1_RPC_REPLY = { +_LANGGRAPH_TASK_REPLY = { "jsonrpc": "2.0", "id": "reply", - "result": {"message": {"messageId": "reply-1", "role": "ROLE_AGENT", "parts": [{"text": "pong"}]}}, + "result": { + "kind": "task", + "id": "run-1:task-1", + "contextId": "thread-1", + "history": [ + { + "kind": "message", + "role": "user", + "parts": [{"kind": "text", "text": "hi"}], + "messageId": "m-user", + "taskId": "run-1:task-1", + "contextId": "thread-1", + }, + { + "kind": "message", + "role": "agent", + "parts": [{"kind": "text", "text": "langgraph echo: hi"}], + "messageId": "m-agent", + "taskId": "run-1:task-1", + "contextId": "thread-1", + }, + ], + "status": {"state": "completed", "timestamp": "2026-08-24T00:00:00+00:00"}, + "artifacts": [ + { + "artifactId": "art-1", + "name": "Assistant Response", + "parts": [{"kind": "text", "text": "langgraph echo: hi"}], + } + ], + }, } @@ -334,17 +364,22 @@ async def test_streaming_send_carries_only_its_own_caller_headers(isolated_clien @pytest.mark.asyncio -async def test_lowercase_protocol_binding_in_agent_card_still_gets_a_client(isolated_client_cache): - """LangGraph Platform serves cards with protocolBinding "jsonrpc"; a2a-sdk matches - bindings case-sensitively, so without normalization client creation raises - ValueError("no compatible transports found.").""" - await _seed_shared_a2a_client(card=_LOWERCASE_BINDING_CARD, rpc_reply=_V1_RPC_REPLY) +async def test_lowercase_protocol_binding_card_round_trips_the_langgraph_dialect(isolated_client_cache): + """LangGraph Platform serves cards with protocolBinding "jsonrpc" and answers in the + A2A 0.3 JSON dialect ("kind"-discriminated) while declaring protocolVersion "1.0". + Without binding normalization client creation raises ValueError("no compatible + transports found."); without the version downgrade the SDK's strict v1 transport + rejects the reply with 'Message type "lf.a2a.v1.Task" has no field named "kind"'.""" + await _seed_shared_a2a_client(card=_LOWERCASE_BINDING_CARD, rpc_reply=_LANGGRAPH_TASK_REPLY) a2a_client = await create_a2a_client(base_url="http://127.0.0.1:9") response = await _send_message(a2a_client, _send_request("lc")) - assert type(response.root.result).__name__ == "Message" - assert a2a_client._litellm_agent_card.supported_interfaces[0].protocol_binding == "JSONRPC" + assert type(response.root.result).__name__ == "Task" + assert response.root.result.artifacts[0].parts[0].root.text == "langgraph echo: hi" + interface = a2a_client._litellm_agent_card.supported_interfaces[0] + assert interface.protocol_binding == "JSONRPC" + assert interface.protocol_version == "0.3" @pytest.mark.asyncio From d0dd24ed6dafeb88032f6f852ef38f349f71986c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:01:34 -0700 Subject: [PATCH 33/54] fix(health): apply model_info.health_check_params to health check probes --- litellm/proxy/health_check.py | 13 ++ .../health_endpoints/_health_endpoints.py | 9 +- .../proxy/test_health_check_max_tokens.py | 142 ++++++++++++++++++ 3 files changed, 160 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index f9d408fb7de..ae95f35333a 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -445,6 +445,9 @@ def _update_litellm_params_for_health_check(model_info: dict, litellm_params: di """ Update the litellm params for health check. + - merges `model_info.health_check_params` into the probe request, so a deployment whose provider + requires a payload field litellm does not synthesize (e.g. `mediaSource` for Bedrock TwelveLabs + Pegasus) can supply it. The dedicated knobs below are applied afterwards and win on conflict. - gets a short `messages` param for health check - adds a bounded `max_tokens` when the deployment is a chat-style mode (`chat`, `completion`, `responses`) or the operator explicitly opts in @@ -459,6 +462,16 @@ def _update_litellm_params_for_health_check(model_info: dict, litellm_params: di model_info, litellm_params, # any-ok: untyped router config dict ) + _health_check_params: Final = model_info.get("health_check_params", None) + if isinstance(_health_check_params, dict): + litellm_params.update(_health_check_params) + elif _health_check_params is not None: + logger.warning( + "health_check_params for model %s is a %s, expected a dict. Ignoring it.", + litellm_params.get("model"), + type(_health_check_params).__name__, + ) + litellm_params["messages"] = _get_random_llm_message() if _should_inject_health_check_max_tokens( model_info, diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 33894777bc3..1feddca9328 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1888,6 +1888,8 @@ async def test_model_connection( # already resolved before reaching this endpoint; any remaining # reference must have come from the request body. _reject_os_environ_references(request_litellm_params) + if model_info: + _reject_os_environ_references(model_info) model_name: Final = request_litellm_params.get("model") # Look up model configuration from router if model name is provided @@ -1951,20 +1953,19 @@ async def test_model_connection( } ## Auth check - auth_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info + resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info await ModelManagementAuthChecks.can_user_make_model_call( model_params=Deployment( model_name="test_model", litellm_params=LiteLLM_Params(**litellm_params), - model_info=auth_model_info, + model_info=resolved_model_info, ), user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, premium_user=premium_user, ) - # Include health_check_params if provided litellm_params = _update_litellm_params_for_health_check( - model_info={}, + model_info=resolved_model_info or {}, litellm_params=litellm_params, ) mode = mode or litellm_params.pop("mode", None) diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index 5a606d5f74e..e20b18c8813 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -1,7 +1,11 @@ +import json +import logging from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx +import litellm from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers from litellm.proxy import health_check as hc_module from litellm.proxy.health_check import ( @@ -543,3 +547,141 @@ async def test_run_model_health_check_skips_auto_router_deployment(): fake_ahealth_check.assert_not_called() assert result == {} + + +# --------------------------------------------------------------------------- +# model_info.health_check_params +# +# Some providers require a payload field litellm does not synthesize for a +# probe. Bedrock TwelveLabs Pegasus rejects any Invoke body without a top-level +# `mediaSource`, so every health check on such a deployment failed with +# "Invalid JSON: $: required property 'mediaSource' not found". The config key +# was accepted and then never read, so operators had no way to supply it. +# --------------------------------------------------------------------------- + + +def test_health_check_params_merge_into_probe_params(): + """health_check_params reach the probe request for the deployment that declares them.""" + media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} + + updated = _update_litellm_params_for_health_check( + {"mode": "chat", "health_check_params": {"mediaSource": media_source}}, + {"model": "bedrock/us.twelvelabs.pegasus-1-2-v1:0"}, + ) + + assert updated["mediaSource"] == media_source + assert updated["model"] == "us.twelvelabs.pegasus-1-2-v1:0" + assert updated["custom_llm_provider"] == "bedrock" + + +def test_health_check_params_lose_to_dedicated_health_check_knobs(): + """The dedicated knobs are applied after the merge, so they win on conflict.""" + model_info = { + "mode": "chat", + "health_check_params": { + "max_tokens": 4096, + "model": "openai/expensive-model", + "messages": [{"role": "user", "content": "from health_check_params"}], + "reasoning_effort": "high", + }, + "health_check_max_tokens": 5, + "health_check_model": "openai/cheap-model", + "health_check_reasoning_effort": "none", + } + + updated = _update_litellm_params_for_health_check(model_info, {"model": "openai/dummy"}) + + assert updated["max_tokens"] == 5 + assert updated["model"] == "openai/cheap-model" + assert updated["reasoning_effort"] == "none" + assert updated["messages"] != model_info["health_check_params"]["messages"] + + +def test_health_check_params_lose_to_the_audio_speech_voice_knob(): + """health_check_voice still wins for audio_speech deployments.""" + updated = _update_litellm_params_for_health_check( + { + "mode": "audio_speech", + "health_check_params": {"voice": "sage", "response_format": "wav"}, + "health_check_voice": "shimmer", + }, + {"model": "openai/tts-1"}, + ) + + assert updated["voice"] == "shimmer" + assert updated["response_format"] == "wav" + + +@pytest.mark.parametrize( + "bad_value", + ["mediaSource", ["mediaSource"], 5, True], +) +def test_health_check_params_ignored_when_not_a_dict(bad_value, caplog): + """A misconfigured health_check_params is skipped with a warning instead of breaking the probe.""" + with caplog.at_level(logging.WARNING, logger="litellm.proxy.health_check"): + updated = _update_litellm_params_for_health_check( + {"mode": "chat", "health_check_params": bad_value}, + {"model": "openai/dummy"}, + ) + + assert updated["model"] == "openai/dummy" + assert updated["max_tokens"] == 16 + assert "health_check_params" in caplog.text + + +def test_health_check_params_apply_to_non_chat_modes(): + """Non-chat probes get health_check_params too, and still no max_tokens.""" + updated = _update_litellm_params_for_health_check( + {"mode": "embedding", "health_check_params": {"dimensions": 8}}, + {"model": "bedrock/amazon.titan-embed-text-v2:0"}, + ) + + assert updated["dimensions"] == 8 + assert "max_tokens" not in updated + + +async def _pegasus_health_check_request_body(model_info: dict, monkeypatch) -> dict: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + litellm_params = _update_litellm_params_for_health_check( + model_info, + { + "model": "bedrock/us.twelvelabs.pegasus-1-2-v1:0", + "aws_access_key_id": "fake-access-key", + "aws_secret_access_key": "fake-secret-key", + "aws_region_name": "us-east-1", + }, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + invoke_route = respx_mock.post( + host="bedrock-runtime.us-east-1.amazonaws.com", + path__regex=r"/model/.+/invoke", + ).respond(json={"message": "a person walks a dog", "finishReason": "stop"}) + result = await litellm.ahealth_check(litellm_params, mode="chat") + + assert "error" not in result, result + return json.loads(invoke_route.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_health_check_params_reach_the_bedrock_invoke_body(monkeypatch): + """The probe Bedrock actually receives carries mediaSource, which is what unblocks Pegasus.""" + media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} + + body = await _pegasus_health_check_request_body( + {"mode": "chat", "health_check_params": {"mediaSource": media_source}}, monkeypatch + ) + + assert body["mediaSource"] == media_source + assert body["maxOutputTokens"] == 16 + assert body["inputPrompt"] + + +@pytest.mark.asyncio +async def test_bedrock_invoke_body_has_no_media_source_without_health_check_params(monkeypatch): + """Negative control: the field only appears because the deployment asked for it.""" + body = await _pegasus_health_check_request_body({"mode": "chat"}, monkeypatch) + + assert "mediaSource" not in body From 3337a0a01fa3f5169ffd0c78dd17515873da9813 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:08:21 -0700 Subject: [PATCH 34/54] fix: match OpenAI SDK wire format on image/video routes (#36493) POST /v1/videos without an input_reference file now goes out as multipart/form-data the way the OpenAI SDK always sends it, instead of a JSON body that OpenAI-compatible backends (SGLang Diffusion, vLLM-Omni) reject; gemini, vertex, and runwayml keep their JSON bodies /v1/images/edits on the openai/azure/openai-compatible path now forwards unknown provider params (e.g. seed) and honors extra_body, matching /v1/images/generations, and aimage_edit forwards extra_headers/extra_query/extra_body instead of dropping them Generic pass-through no longer downgrades a file-less multipart form to application/x-www-form-urlencoded --- litellm/images/main.py | 12 ++ .../litellm_core_utils/llm_request_utils.py | 39 +++++++ .../llms/base_llm/videos/transformation.py | 8 ++ litellm/llms/custom_httpx/llm_http_handler.py | 28 +++-- litellm/llms/openai/videos/transformation.py | 3 + .../pass_through_endpoints.py | 18 ++- .../images/test_image_edit_extra_params.py | 102 ++++++++++++++++ .../test_llm_request_utils.py | 36 ++++++ .../custom_httpx/test_llm_http_handler.py | 109 ++++++++++++++++++ .../test_pass_through_endpoints.py | 37 ++++++ 10 files changed, 378 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/images/test_image_edit_extra_params.py create mode 100644 tests/test_litellm/litellm_core_utils/test_llm_request_utils.py diff --git a/litellm/images/main.py b/litellm/images/main.py index ae4818b1967..fd18edc66fb 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -846,6 +846,15 @@ def image_edit( additional_drop_params=kwargs.get("additional_drop_params"), ) + if ( + custom_llm_provider == "openai" + or custom_llm_provider == "azure" + or custom_llm_provider in litellm.openai_compatible_providers + ): + image_edit_request_params.update(non_default_params) + if isinstance(extra_body, dict): + image_edit_request_params.update(extra_body) + # Pre Call logging litellm_logging_obj.update_from_kwargs( kwargs=kwargs, @@ -995,6 +1004,9 @@ async def aimage_edit( response_format=response_format, size=size, user=user, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, **kwargs, diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index b4e27b129fe..33b402789b3 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,8 +1,47 @@ +from collections.abc import Mapping from typing import Final import litellm +def _form_field_value(value: object) -> str: + if value is True: + return "true" + if value is False: + return "false" + return str(value) + + +def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]: + if isinstance(value, Mapping): + return tuple( + item for subkey, subvalue in value.items() for item in _flatten_form_field(f"{key}[{subkey}]", subvalue) + ) + if isinstance(value, (list, tuple)): + return tuple(item for entry in value for item in _flatten_form_field(f"{key}[]", entry)) + if value is None: + return () + serialized: Final = _form_field_value(value) + if not serialized: + return () + return ((key, serialized),) + + +def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]: + """ + Encode a JSON-shaped body as httpx file-tuples so a request with no file + parts is still sent as multipart/form-data (httpx downgrades a file-less + ``data=`` payload to application/x-www-form-urlencoded). Nested values are + flattened the way the OpenAI SDK serializes multipart bodies: dicts as + ``key[subkey]``, lists as ``key[]``, booleans lowercased, None dropped. + """ + return tuple( + (key, (None, serialized)) + for top_key, top_value in data.items() + for key, serialized in _flatten_form_field(top_key, top_value) + ) + + def _ensure_extra_body_is_safe(extra_body: dict | None) -> dict | None: """ Ensure that the extra_body sent in the request is safe, otherwise users will see this error diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 1aea3cafe33..dcecdc646ff 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -91,6 +91,14 @@ class BaseVideoConfig(ABC): raise ValueError("api_base is required") return api_base + def use_multipart_form_data(self) -> bool: + """ + Whether video create requests without files must still be sent as + multipart/form-data (the encoding the OpenAI SDK always uses for + /videos), instead of falling back to JSON. + """ + return False + @abstractmethod def transform_video_create_request( self, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ed079197513..cd215f9daa8 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -7050,9 +7051,7 @@ class BaseLLMHTTPHandler: ) try: - # Use JSON when no files, otherwise use form data with files if files and len(files) > 0: - # Use multipart/form-data when files are present response = sync_httpx_client.post( url=api_base, headers=headers, @@ -7060,9 +7059,14 @@ class BaseLLMHTTPHandler: files=files, timeout=timeout, ) - + elif video_generation_provider_config.use_multipart_form_data(): + response = sync_httpx_client.post( + url=api_base, + headers=headers, + files=serialize_multipart_form_fields(data), + timeout=timeout, + ) else: - # Use JSON content type for POST requests without files response = sync_httpx_client.post( url=api_base, headers=headers, @@ -7154,20 +7158,26 @@ class BaseLLMHTTPHandler: ) try: - # Use JSON when no files, otherwise use form data with files - if files is None or len(files) == 0: + if files and len(files) > 0: response = await async_httpx_client.post( url=api_base, headers=headers, - json=data, + data=data, + files=files, + timeout=timeout, + ) + elif video_generation_provider_config.use_multipart_form_data(): + response = await async_httpx_client.post( + url=api_base, + headers=headers, + files=serialize_multipart_form_fields(data), timeout=timeout, ) else: response = await async_httpx_client.post( url=api_base, headers=headers, - data=data, - files=files, + json=data, timeout=timeout, ) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 50b466ae996..4fd0429c182 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -101,6 +101,9 @@ class OpenAIVideoConfig(BaseVideoConfig): return f"{api_base.rstrip('/')}/videos" + def use_multipart_form_data(self) -> bool: + return True + def transform_video_create_request( self, model: str, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1915a853983..3d721dead4d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -470,7 +470,10 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): ``items()`` collapses duplicate keys to the last value. Files go out as a list of ``(field_name, (filename, content, content_type))`` tuples and repeated non-file fields are grouped into list values, both of which httpx - encodes as separate multipart parts. + encodes as separate multipart parts. A form with no file parts is sent + entirely through ``files`` as ``(field_name, (None, value))`` tuples, + because httpx downgrades a file-less ``data=`` payload to + application/x-www-form-urlencoded. """ form_items: Final = (await request.form()).multi_items() @@ -500,6 +503,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): ) } + multipart_files: Final = ( + files if files else tuple((field_name, (None, field_value)) for field_name, field_value in non_file_items) + ) + multipart_data: Final = form_data_dict if files else None + # Remove content-type header - httpx will set it correctly with the new boundary # when it creates the multipart body from files/data parameters headers_copy: Final = headers.copy() @@ -512,8 +520,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): url, headers=headers_copy, params=requested_query_params, - files=files, - data=form_data_dict, + files=multipart_files, + data=multipart_data, ) return await async_client.send(req, stream=True) @@ -522,8 +530,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): url=url, headers=headers_copy, params=requested_query_params, - files=files, - data=form_data_dict, + files=multipart_files, + data=multipart_data, ) @staticmethod diff --git a/tests/test_litellm/images/test_image_edit_extra_params.py b/tests/test_litellm/images/test_image_edit_extra_params.py new file mode 100644 index 00000000000..46a5feb08a0 --- /dev/null +++ b/tests/test_litellm/images/test_image_edit_extra_params.py @@ -0,0 +1,102 @@ +""" +Regression tests for https://github.com/BerriAI/litellm/issues/36493 + +/v1/images/edits on the openai path silently dropped unknown provider params +(e.g. seed) and the extra_body escape hatch, unlike /v1/images/generations. +""" + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + +PNG_BYTES = b"\x89PNG\r\n\x1a\nfakepng" + + +def _capture_image_edit_request(captured): + def respond(request): + captured["content_type"] = request.headers.get("content-type") + captured["body"] = request.content + return httpx.Response(200, json={"created": 1712697600, "data": [{"b64_json": "aW1n"}]}) + + return respond + + +def _multipart_text_fields(content_type: str, body: bytes) -> dict: + boundary = content_type.split("boundary=")[1].encode() + return { + part.split(b'name="')[1].split(b'"')[0].decode(): part.partition(b"\r\n\r\n")[2].rstrip(b"\r\n-").decode() + for part in body.split(b"--" + boundary) + if b'name="' in part and b"filename=" not in part + } + + +def test_image_edit_forwards_provider_params_and_extra_body(): + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_image_edit_request(captured)))) + + response = litellm.image_edit( + model="openai/gpt-image-1", + image=PNG_BYTES, + prompt="add a hat", + api_key="sk-test", + api_base="https://edit.example/v1", + client=client, + seed=42, + extra_body={"quality_level": "high"}, + ) + + assert captured["content_type"].startswith("multipart/form-data") + fields = _multipart_text_fields(captured["content_type"], captured["body"]) + assert fields["seed"] == "42" + assert fields["quality_level"] == "high" + assert "extra_body" not in fields + assert fields["model"] == "gpt-image-1" + assert fields["prompt"] == "add a hat" + assert b'name="image[]"' in captured["body"] + assert response.data + + +def test_image_edit_extra_body_takes_precedence_over_kwargs(): + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_image_edit_request(captured)))) + + litellm.image_edit( + model="openai/gpt-image-1", + image=PNG_BYTES, + prompt="add a hat", + api_key="sk-test", + api_base="https://edit.example/v1", + client=client, + seed=42, + extra_body={"seed": 7}, + ) + + assert _multipart_text_fields(captured["content_type"], captured["body"])["seed"] == "7" + + +@pytest.mark.asyncio +async def test_aimage_edit_forwards_extra_body(): + """aimage_edit used to drop extra_headers/extra_query/extra_body when + building its partial, so they never reached image_edit.""" + captured = {} + client = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(_capture_image_edit_request(captured))) + + response = await litellm.aimage_edit( + model="openai/gpt-image-1", + image=PNG_BYTES, + prompt="add a hat", + api_key="sk-test", + api_base="https://edit.example/v1", + client=client, + seed=42, + extra_body={"quality_level": "high"}, + ) + + fields = _multipart_text_fields(captured["content_type"], captured["body"]) + assert fields["seed"] == "42" + assert fields["quality_level"] == "high" + assert "extra_body" not in fields + assert response.data diff --git a/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py b/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py new file mode 100644 index 00000000000..bd4f8943b47 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py @@ -0,0 +1,36 @@ +from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields + + +def test_serialize_multipart_form_fields_flattens_like_the_openai_sdk(): + fields = serialize_multipart_form_fields( + { + "model": "sora-2", + "prompt": "a cat surfing", + "hd": True, + "watermark": False, + "seconds": 4, + "size": None, + "metadata": {"trace": {"id": "t1"}}, + "characters": [{"id": "char_1", "name": "Mia"}, "solo"], + } + ) + + assert fields == ( + ("model", (None, "sora-2")), + ("prompt", (None, "a cat surfing")), + ("hd", (None, "true")), + ("watermark", (None, "false")), + ("seconds", (None, "4")), + ("metadata[trace][id]", (None, "t1")), + ("characters[][id]", (None, "char_1")), + ("characters[][name]", (None, "Mia")), + ("characters[]", (None, "solo")), + ) + + +def test_serialize_multipart_form_fields_drops_empty_strings(): + assert serialize_multipart_form_fields({"prompt": "", "model": "sora-2"}) == (("model", (None, "sora-2")),) + + +def test_serialize_multipart_form_fields_empty_body(): + assert serialize_multipart_form_fields({}) == () diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 9faa77d6dce..b78829e2e11 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -26,6 +26,7 @@ from litellm.llms.custom_httpx.llm_http_handler import ( _has_pre_call_deployment_hook, _rust_responses_websocket_enabled, ) +from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import TranscriptionResponse @@ -2524,3 +2525,111 @@ def test_only_callbacks_that_can_charge_a_frame_are_collected_for_ws_quota(monke monkeypatch.setattr(litellm, "callbacks", [plain, quota, decoy]) assert _collect_ws_project_quota_callbacks() == (quota,) + + +class _JSONBodyVideoConfig(OpenAIVideoConfig): + def use_multipart_form_data(self) -> bool: + return False + + +def _video_create_call_kwargs(config, **optional_params): + return { + "model": "sora-2", + "prompt": "a cat surfing", + "video_generation_provider_config": config, + "video_generation_optional_request_params": {"seconds": "4", **optional_params}, + "custom_llm_provider": "openai", + "litellm_params": GenericLiteLLMParams(api_key="sk-test", api_base="https://video.example/v1"), + "logging_obj": Mock(), + "timeout": 10.0, + } + + +def _capture_video_create_request(captured): + def respond(request): + captured["content_type"] = request.headers.get("content-type") + captured["body"] = request.content + return httpx.Response( + 200, + json={"id": "video_123", "object": "video", "status": "queued", "created_at": 1712697600, "model": "sora-2"}, + ) + + return respond + + +def _multipart_text_fields(content_type: str, body: bytes) -> dict: + boundary = content_type.split("boundary=")[1].encode() + return { + part.split(b'name="')[1].split(b'"')[0].decode(): part.partition(b"\r\n\r\n")[2].rstrip(b"\r\n-").decode() + for part in body.split(b"--" + boundary) + if b'name="' in part and b"filename=" not in part + } + + +def test_video_generation_without_file_sends_multipart_form_data(): + """Regression for #36493: the OpenAI SDK always sends /videos requests as + multipart/form-data, so OpenAI-compatible backends (SGLang Diffusion, + vLLM-Omni) reject the JSON body LiteLLM used to send when no + input_reference file was attached.""" + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) + + result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(OpenAIVideoConfig())) + + assert captured["content_type"].startswith("multipart/form-data") + assert _multipart_text_fields(captured["content_type"], captured["body"]) == { + "model": "sora-2", + "prompt": "a cat surfing", + "seconds": "4", + } + assert result.status == "queued" + + +@pytest.mark.asyncio +async def test_async_video_generation_without_file_sends_multipart_form_data(): + captured = {} + client = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(_capture_video_create_request(captured))) + + result = await BaseLLMHTTPHandler().async_video_generation_handler( + client=client, **_video_create_call_kwargs(OpenAIVideoConfig()) + ) + + assert captured["content_type"].startswith("multipart/form-data") + assert _multipart_text_fields(captured["content_type"], captured["body"]) == { + "model": "sora-2", + "prompt": "a cat surfing", + "seconds": "4", + } + assert result.status == "queued" + + +def test_video_generation_json_provider_keeps_json_body(): + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) + + result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(_JSONBodyVideoConfig())) + + assert captured["content_type"] == "application/json" + assert json.loads(captured["body"]) == {"model": "sora-2", "prompt": "a cat surfing", "seconds": "4"} + assert result.status == "queued" + + +def test_video_generation_with_input_reference_keeps_file_multipart(): + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) + + result = BaseLLMHTTPHandler().video_generation_handler( + client=client, + **_video_create_call_kwargs(OpenAIVideoConfig(), input_reference=b"\x89PNG\r\n\x1a\nfakepng"), + ) + + assert captured["content_type"].startswith("multipart/form-data") + assert b'name="input_reference"' in captured["body"] + assert b'filename="input_reference.png"' in captured["body"] + assert _multipart_text_fields(captured["content_type"], captured["body"]) == { + "model": "sora-2", + "prompt": "a cat surfing", + "seconds": "4", + } + assert result.status == "queued" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 25d176e48bb..99a84d43c9b 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -186,6 +186,43 @@ async def test_make_multipart_http_request_forwards_repeated_fields(): assert call_args["data"] == {"other_parameter": ["xxx", "yyy"]} +@pytest.mark.asyncio +async def test_make_multipart_http_request_fileless_form_stays_multipart(): + """ + Regression for #36493: a multipart form with no file parts was forwarded + through httpx's ``data=`` alone, which downgrades the request to + application/x-www-form-urlencoded. Every field must go through ``files`` + as a ``(field_name, (None, value))`` tuple so httpx keeps the + multipart/form-data encoding the client sent. + """ + request = MagicMock(spec=Request) + request.method = "POST" + form_data = FormData([("prompt", "a cat surfing"), ("model", "sora-2"), ("seconds", "4")]) + request.form = AsyncMock(return_value=form_data) + + mock_response = MagicMock() + mock_response.status_code = 200 + async_client = MagicMock() + async_client.request = AsyncMock(return_value=mock_response) + + await HttpPassThroughEndpointHelpers.make_multipart_http_request( + request=request, + async_client=async_client, + url=httpx.URL("http://test.com"), + headers={}, + requested_query_params=None, + ) + + call_args = async_client.request.call_args[1] + + assert call_args["files"] == ( + ("prompt", (None, "a cat surfing")), + ("model", (None, "sora-2")), + ("seconds", (None, "4")), + ) + assert call_args["data"] is None + + @pytest.mark.asyncio async def test_make_multipart_http_request_removes_content_type_header(): """ From fef5f41985c12b5b5278b99bdbadbe5ee0381ff0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:12:54 -0700 Subject: [PATCH 35/54] fix(anthropic): keep legacy thinking budget_tokens on Claude 4.6 models --- litellm/llms/anthropic/common_utils.py | 10 ++ .../messages/transformation.py | 10 +- ...odel_prices_and_context_window_backup.json | 36 +++++++ litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 36 +++++++ .../test_reasoning_effort_translation.py | 93 +++++++++++++++---- 7 files changed, 168 insertions(+), 19 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 3297aa95715..9461e40cf2e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -440,6 +440,16 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider) + @staticmethod + def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool: + """Whether ``model`` is an adaptive-thinking model that still accepts legacy + ``thinking.type=enabled`` with ``budget_tokens`` (the Claude 4.6 family). + The model cost map is authoritative: an explicit ``supports_legacy_thinking`` + entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations`` + rule for unmapped 4.6 ids. Absent flag means the model rejects the legacy shape. + """ + return AnthropicModelInfo._supports_model_capability(model, "supports_legacy_thinking", custom_llm_provider) + @staticmethod def maybe_drop_disabled_thinking( model: str, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index adabfa2d62d..032bf0130ce 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -379,13 +379,19 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _translate_legacy_thinking_for_adaptive_model( model: str, optional_params: dict, custom_llm_provider: str ) -> None: - """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7. - Caller-provided ``output_config.effort`` is never overridden. + """Translate legacy ``thinking.type=enabled`` to adaptive for the + adaptive-thinking models that reject it (4.7+ and the 5 families). + Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the + legacy shape natively, so it is forwarded verbatim and the caller's + ``budget_tokens`` cap keeps applying. Caller-provided + ``output_config.effort`` is never overridden. """ from litellm.llms.anthropic.chat.transformation import AnthropicConfig if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): return + if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): + return thinking: Final = optional_params.get("thinking") if not isinstance(thinking, dict) or thinking.get("type") != "enabled": return diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3af7d9e5019..dda327bf9f6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1019,6 +1019,7 @@ }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1053,6 +1054,7 @@ }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1087,6 +1089,7 @@ }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1121,6 +1124,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1155,6 +1159,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -2233,6 +2238,7 @@ }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2266,6 +2272,7 @@ }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2299,6 +2306,7 @@ }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2332,6 +2340,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2365,6 +2374,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2398,6 +2408,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2950,6 +2961,7 @@ "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -3181,6 +3193,7 @@ "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12489,6 +12502,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -12698,6 +12712,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12735,6 +12750,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -14677,6 +14693,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -14753,6 +14770,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -23180,6 +23198,7 @@ }, "github_copilot/claude-opus-4.6-fast": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, @@ -33563,6 +33582,7 @@ }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost": 3e-07, @@ -33607,6 +33627,7 @@ }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -35681,6 +35702,7 @@ }, "perplexity/anthropic/claude-opus-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, @@ -39056,6 +39078,7 @@ }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -40315,6 +40338,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40347,6 +40371,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40712,6 +40737,7 @@ "vertex_ai/claude-sonnet-4-6": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -48481,6 +48507,7 @@ "vertex_ai/claude-sonnet-4-6@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -49265,6 +49292,7 @@ }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, @@ -50255,6 +50283,14 @@ "supports_adaptive_thinking": true } }, + { + "name": "claude-legacy-thinking", + "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", + "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", + "model_info": { + "supports_legacy_thinking": true + } + }, { "name": "claude-always-on-thinking", "pattern": "claude-(?:fable|mythos)-", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 94526de0757..e8e8edb85b3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -154,6 +154,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_web_search: bool | None supports_reasoning: bool | None supports_adaptive_thinking: bool | None + supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] supports_tool_search: bool | None supports_mid_conversation_system: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..c0e26ef19d4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5753,6 +5753,7 @@ def _get_model_info_helper( supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), + supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3af7d9e5019..dda327bf9f6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1019,6 +1019,7 @@ }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1053,6 +1054,7 @@ }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1087,6 +1089,7 @@ }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1121,6 +1124,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1155,6 +1159,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -2233,6 +2238,7 @@ }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2266,6 +2272,7 @@ }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2299,6 +2306,7 @@ }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2332,6 +2340,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2365,6 +2374,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2398,6 +2408,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -2950,6 +2961,7 @@ "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -3181,6 +3193,7 @@ "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12489,6 +12502,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -12698,6 +12712,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12735,6 +12750,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -14677,6 +14693,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -14753,6 +14770,7 @@ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_legacy_thinking": true, "supports_reasoning": true, "supports_tool_choice": true }, @@ -23180,6 +23198,7 @@ }, "github_copilot/claude-opus-4.6-fast": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, @@ -33563,6 +33582,7 @@ }, "openrouter/anthropic/claude-sonnet-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost": 3e-07, @@ -33607,6 +33627,7 @@ }, "openrouter/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -35681,6 +35702,7 @@ }, "perplexity/anthropic/claude-opus-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "litellm_provider": "perplexity", "mode": "responses", "supports_web_search": true, @@ -39056,6 +39078,7 @@ }, "vercel_ai_gateway/anthropic/claude-opus-4.6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -40315,6 +40338,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40347,6 +40371,7 @@ "deprecation_date": "2027-02-05", "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -40712,6 +40737,7 @@ "vertex_ai/claude-sonnet-4-6": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -48481,6 +48507,7 @@ "vertex_ai/claude-sonnet-4-6@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -49265,6 +49292,7 @@ }, "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, + "supports_legacy_thinking": true, "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, @@ -50255,6 +50283,14 @@ "supports_adaptive_thinking": true } }, + { + "name": "claude-legacy-thinking", + "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", + "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", + "model_info": { + "supports_legacy_thinking": true + } + }, { "name": "claude-always-on-thinking", "pattern": "claude-(?:fable|mythos)-", diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py index f393a7b50b1..12ab536ed45 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py @@ -2,7 +2,6 @@ import pytest -import litellm from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, @@ -17,7 +16,6 @@ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_tran ) - @pytest.mark.parametrize( "reasoning_effort,expected_effort", [ @@ -258,19 +256,22 @@ def test_reasoning_effort_in_supported_params(): "model", [ "claude-sonnet-4-6", - "bedrock/invoke/us.anthropic.claude-sonnet-4-6", - "vertex_ai/claude-sonnet-4-6", "claude-opus-4-6", + "claude-sonnet-4-6-20260219", + "bedrock/invoke/us.anthropic.claude-sonnet-4-6", "bedrock/invoke/us.anthropic.claude-opus-4-6-v1:0", + "vertex_ai/claude-sonnet-4-6", "vertex_ai/claude-opus-4-6", + "azure_ai/claude-sonnet-4-6", ], ) -def test_legacy_thinking_high_budget_clamps_to_high_when_xhigh_unsupported( - local_model_cost_map, model -): - """Claude Code sends ``thinking.budget_tokens=31999``; Sonnet 4.6 and Opus 4.6 - have no ``xhigh`` tier, so the translator must emit ``high`` rather than the - provider-invalid ``xhigh`` (regression for issue #29282).""" +def test_legacy_thinking_budget_preserved_verbatim_on_46(local_model_cost_map, model): + """Regression for the passthrough silently dropping a caller's hard thinking + budget: the 4.6 family accepts ``thinking.type=enabled`` with ``budget_tokens`` + natively, so rewriting it to ``thinking.type=adaptive`` + ``output_config.effort`` + (which carries no ceiling) let reasoning run past the requested cap. The legacy + shape must be forwarded verbatim, in every 4.6 id shape including unmapped dated + releases resolved by the ``claude-legacy-thinking`` fallback rule.""" config = AnthropicMessagesConfig() optional_params = { "max_tokens": 1024, @@ -285,8 +286,8 @@ def test_legacy_thinking_high_budget_clamps_to_high_when_xhigh_unsupported( headers={}, ) - assert result.get("thinking") == {"type": "adaptive"} - assert result.get("output_config") == {"effort": "high"} + assert result.get("thinking") == {"type": "enabled", "budget_tokens": 31999} + assert "output_config" not in result def test_legacy_thinking_high_budget_keeps_xhigh_when_supported(): @@ -343,11 +344,44 @@ def test_legacy_thinking_translates_to_adaptive_for_opus_48( assert result.get("output_config") == {"effort": "xhigh"} +@pytest.mark.parametrize( + "model,expected_effort", + [ + ("claude-sonnet-5", "xhigh"), + ("claude-opus-5", "xhigh"), + ("claude-newfamily-6", "high"), + ], +) +def test_legacy_thinking_translates_to_adaptive_for_5_and_future_models( + local_model_cost_map, model, expected_effort +): + """The 5 families reject ``thinking.type=enabled``, so the adaptive translation + stays the safe default for every adaptive model not flagged + ``supports_legacy_thinking``, unmapped future ids included. An unmapped id + cannot prove ``xhigh`` support, so its high-budget bucket clamps to ``high``.""" + config = AnthropicMessagesConfig() + optional_params = { + "max_tokens": 1024, + "thinking": {"type": "enabled", "budget_tokens": 31999}, + } + + result = config.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": expected_effort} + + @pytest.mark.parametrize( "budget_tokens,expected_effort", [ - (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET * 2, "high"), - (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, "high"), + (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET * 2, "xhigh"), + (DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, "xhigh"), (DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, "high"), (DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET - 1, "medium"), (DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, "medium"), @@ -355,7 +389,9 @@ def test_legacy_thinking_translates_to_adaptive_for_opus_48( (1, "low"), ], ) -def test_legacy_thinking_budget_buckets_on_sonnet_46(budget_tokens, expected_effort): +def test_legacy_thinking_budget_buckets_on_opus_48( + local_model_cost_map, budget_tokens, expected_effort +): config = AnthropicMessagesConfig() optional_params = { "max_tokens": 1024, @@ -363,7 +399,7 @@ def test_legacy_thinking_budget_buckets_on_sonnet_46(budget_tokens, expected_eff } result = config.transform_anthropic_messages_request( - model="claude-sonnet-4-6", + model="claude-opus-4-8", messages=[{"role": "user", "content": "Hello"}], anthropic_messages_optional_request_params=optional_params, litellm_params={}, @@ -373,7 +409,29 @@ def test_legacy_thinking_budget_buckets_on_sonnet_46(budget_tokens, expected_eff assert result.get("output_config") == {"effort": expected_effort} -def test_legacy_thinking_does_not_override_explicit_output_config(): +def test_legacy_thinking_does_not_override_explicit_output_config(local_model_cost_map): + config = AnthropicMessagesConfig() + optional_params = { + "max_tokens": 1024, + "thinking": {"type": "enabled", "budget_tokens": 31999}, + "output_config": {"effort": "low"}, + } + + result = config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": "low"} + + +def test_legacy_thinking_with_explicit_output_config_untouched_on_46( + local_model_cost_map, +): config = AnthropicMessagesConfig() optional_params = { "max_tokens": 1024, @@ -389,6 +447,7 @@ def test_legacy_thinking_does_not_override_explicit_output_config(): headers={}, ) + assert result.get("thinking") == {"type": "enabled", "budget_tokens": 31999} assert result.get("output_config") == {"effort": "low"} From d23069e907fa179fde00a6769ca8acc9f772d0e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:16:57 -0700 Subject: [PATCH 36/54] fix(passthrough): attribute spend and release budget reservation on router-model /vllm and /azure routes The /vllm and /azure router-model passthrough branches called llm_router.allm_passthrough_route directly with no request metadata, so the cost callback saw no user_api_key and no user_api_key_budget_reservation. Spend for a budgeted virtual key hit neither the key's spend nor the spend logs, and the reservation minted at auth into the shared Redis counter was never released, drifting the counter up until the key falsely tripped BudgetExceededError. Thread the authenticated key's attribution metadata into both calls via the same builder add_litellm_data_to_request uses, so the cost callback attributes spend and reconciles the reservation. Regression tests cover both branches. --- .../llm_passthrough_endpoints.py | 26 +++++- .../test_llm_pass_through_endpoints.py | 84 +++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 7ce41c1d5b6..fc891978ad5 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -9,7 +9,7 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. import json import os import re -from collections.abc import Callable +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, cast @@ -106,6 +106,28 @@ def is_passthrough_request_streaming(request_body: object) -> bool: return bool(request_body.get("stream", False)) +def get_passthrough_router_request_metadata(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, Any]: + """ + Build the request metadata carrying key-level spend attribution and the + pre-call budget reservation for a router-model passthrough request. + + Router-model passthrough branches call ``allm_passthrough_route`` directly, + bypassing ``add_litellm_data_to_request``. Without this metadata the cost + callback cannot attribute spend to the calling key and never releases the + budget reservation minted at auth time, so the shared spend counter drifts + up until the key falsely trips ``BudgetExceededError``. + """ + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + request_data: Final = {"metadata": {}} # mutable-ok: attribution builder + litellm mutate this dict in place + LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( + data=request_data, + user_api_key_dict=user_api_key_dict, + _metadata_variable_name="metadata", + ) + return request_data["metadata"] + + async def llm_passthrough_factory_proxy_route( custom_llm_provider: str, endpoint: str, @@ -346,6 +368,7 @@ async def vllm_proxy_route( params=None, headers=None, cookies=None, + metadata=get_passthrough_router_request_metadata(user_api_key_dict), ), ) @@ -1475,6 +1498,7 @@ async def azure_proxy_route( params=None, headers=None, cookies=None, + metadata=get_passthrough_router_request_metadata(user_api_key_dict), ) if is_streaming_request: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index ac140abe31f..032cb360d4c 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -4055,3 +4055,87 @@ class TestVertexAILiveWebsocketPassthrough: assert "use_in_pass_through" in close_kwargs["reason"] assert "default_vertex_config" in close_kwargs["reason"] assert len(close_kwargs["reason"].encode("utf-8")) <= 123 + + +class TestPassthroughRouterModelBudgetReservation: + """ + Router-model passthrough on /vllm and /azure must thread the calling key's + metadata into ``allm_passthrough_route``. Without ``user_api_key`` the spend + is attributed to nobody, and without ``user_api_key_budget_reservation`` the + pre-call reservation is never released, so the shared spend counter drifts up + until the key falsely trips a 429 BudgetExceededError (LIT-5470). + """ + + def _key_with_reservation(self) -> UserAPIKeyAuth: + reservation = { + "reserved_cost": 0.5, + "entries": [{"counter_key": "spend:key:hashed-token", "reserved_cost": 0.5}], + } + return UserAPIKeyAuth( + api_key="hashed-token", + user_id="u1", + team_id="t1", + budget_reservation=reservation, + ) + + def _request(self) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": "application/json"} + request.query_params = {} + return request + + def _install_recording_router(self, monkeypatch, body: dict) -> list[dict]: + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + + captured: list[dict] = [] + + class RecordingRouter: + async def allm_passthrough_route(self, **kwargs): + captured.append(kwargs) + return httpx.Response(200, json={"ok": True}) + + async def fake_get_request_body(_request): + return body + + monkeypatch.setattr(proxy_server, "llm_router", RecordingRouter()) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + return captured + + def _assert_metadata_carries_attribution(self, captured: list[dict], user_api_key_dict: UserAPIKeyAuth) -> None: + assert len(captured) == 1, "the router-model branch must dispatch exactly once" + metadata = captured[0]["metadata"] + assert metadata["user_api_key"] == user_api_key_dict.api_key + assert metadata["user_api_key_budget_reservation"] is user_api_key_dict.budget_reservation + assert metadata["user_api_key_user_id"] == user_api_key_dict.user_id + assert metadata["user_api_key_team_id"] == user_api_key_dict.team_id + + @pytest.mark.asyncio + async def test_vllm_router_model_threads_key_metadata(self, monkeypatch): + user_api_key_dict = self._key_with_reservation() + captured = self._install_recording_router(monkeypatch, {"model": "router-model", "stream": False}) + + await vllm_proxy_route( + endpoint="/chat/completions", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=user_api_key_dict, + ) + + self._assert_metadata_carries_attribution(captured, user_api_key_dict) + + @pytest.mark.asyncio + async def test_azure_router_model_threads_key_metadata(self, monkeypatch): + user_api_key_dict = self._key_with_reservation() + captured = self._install_recording_router(monkeypatch, {"model": "gpt-5", "stream": False}) + + await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=user_api_key_dict, + ) + + self._assert_metadata_carries_attribution(captured, user_api_key_dict) From 6407a6637510126177806fe503376d5d457f1ae4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:23:03 -0700 Subject: [PATCH 37/54] fix(runwayml): route every generation endpoint and fix video cost tracking Six defects in the RunwayML video provider: - transform_video_create_request hardcoded /image_to_video, so text-to-video 400'd and video-to-video was unreachable; the endpoint is now selected from the inputs present (promptVideo/videoUri, promptImage, or text only) - get_error_class raised instead of returning, turning a provider 4xx into a proxy 500 APIConnectionError; it now returns a RunwayMLError - VideoObject.progress was typed int while Runway sends a 0..1 float, 500'ing status polls while RUNNING; it is now scaled to a 0..100 percent - custom per-deployment pricing stored under litellm_metadata was ignored for video; the deployment model_info lookup now checks both metadata keys - stale cost-map entries (gen3a_turbo, gen4_aleph) were removed and current models added, with output_cost_per_second_480p/_4k tier keys plumbed through the model-info and router types - video cost now falls back to Runway's estimatedCost from the create response when no custom pricing is configured, and custom pricing always wins over it Fixes #36483 --- litellm/cost_calculator.py | 18 ++- .../exception_mapping_utils.py | 1 + litellm/llms/openai/cost_calculation.py | 9 +- .../llms/runwayml/videos/transformation.py | 115 +++++++++---- ...odel_prices_and_context_window_backup.json | 151 +++++++++++++++++- litellm/types/router.py | 4 +- litellm/types/utils.py | 4 + litellm/utils.py | 2 + model_prices_and_context_window.json | 151 +++++++++++++++++- .../test_exception_mapping_utils.py | 3 + .../test_runway_video_transformation.py | 140 +++++++++++++++- tests/test_litellm/test_utils.py | 5 + tests/test_litellm/test_video_generation.py | 111 +++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 + 14 files changed, 667 insertions(+), 55 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8f7cd09d364..58d29d99e3e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -150,6 +150,7 @@ _VIDEO_CALL_TYPES: Final = frozenset( } ) + _SPEECH_CALL_TYPES: Final = frozenset( { CallTypes.speech.value, @@ -1372,23 +1373,36 @@ def completion_cost( if custom_pricing and litellm_logging_obj is not None: _litellm_params = getattr(litellm_logging_obj, "litellm_params", None) if _litellm_params is not None: - _metadata = _litellm_params.get("metadata", {}) or {} - _video_model_info = _metadata.get("model_info", None) + _video_model_info = next( + ( + model_info + for _metadata_key in ("metadata", "litellm_metadata") + if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info")) + is not None + ), + None, + ) usage_obj = getattr(completion_response, "usage", None) duration_seconds: float | None = None video_resolution: str | None = None + provider_reported_cost: float | None = None if completion_response is not None and usage_obj: # Handle both dict and Pydantic Usage object if isinstance(usage_obj, dict): duration_seconds = usage_obj.get("duration_seconds", None) _vr = usage_obj.get("video_resolution", None) + provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None) else: duration_seconds = getattr(usage_obj, "duration_seconds", None) _vr = getattr(usage_obj, "video_resolution", None) + provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None) if _vr is not None: video_resolution = str(_vr).strip().lower() + if _video_model_info is None and provider_reported_cost is not None: + return float(provider_reported_cost) + if duration_seconds is not None: # Calculate cost based on video duration using video-specific cost calculation from litellm.llms.openai.cost_calculation import ( diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 4a25eb218c0..4ee726b67de 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2301,6 +2301,7 @@ def exception_type( or custom_llm_provider == "custom_openai" or custom_llm_provider in litellm.openai_compatible_providers or custom_llm_provider == "mistral" + or custom_llm_provider == "runwayml" ): _map_openai_exception( model=model, diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 0352d246c09..115b2e27983 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -134,14 +134,7 @@ def cost_per_second(model: str, custom_llm_provider: str | None, duration: float def _video_resolution_to_cost_field_suffix(resolution: str) -> str | None: - """ - Map usage resolution to a safe suffix for ``output_cost_per_second_`` keys. - - Note: Currently only ``output_cost_per_second_1080p`` is explicitly declared in - ModelInfo (types/utils.py). Other resolution tiers (e.g., 720p, 4k) can be added - to model_prices_and_context_window.json but are not exposed via get_model_info() - until added to the ModelInfo TypedDict. - """ + """Map usage resolution to a safe suffix for ``output_cost_per_second_`` keys.""" r: Final = resolution.strip().lower() if not r: return None diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index b8e57fa7cc0..065661a8731 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -1,5 +1,6 @@ from collections.abc import Mapping, Sequence from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -33,6 +34,10 @@ else: LiteLLMLoggingObj = Any +class RunwayMLError(BaseLLMException): + pass + + class _RunwayTaskResponse(TypedDict, total=False): id: ReadOnly[str] status: ReadOnly[str] @@ -41,7 +46,8 @@ class _RunwayTaskResponse(TypedDict, total=False): output: ReadOnly[Sequence[str] | str] failureCode: ReadOnly[str] failure: ReadOnly[str] - progress: ReadOnly[int] + progress: ReadOnly[float] + estimatedCost: ReadOnly[Mapping[str, float]] class _VideoObjectData(TypedDict, extra_items=object): @@ -56,12 +62,54 @@ def _parse_runway_task_response(raw_response: httpx.Response) -> _RunwayTaskResp return response_data +_USD_PER_CREDIT: Final = 0.01 + +_RESOLUTION_AREA_TIERS: Final[tuple[tuple[int, str], ...]] = ( + (600_000, "480p"), + (1_500_000, "720p"), + (4_000_000, "1080p"), +) + + +def _ratio_to_resolution(ratio: object) -> str | None: + if not isinstance(ratio, str) or ":" not in ratio: + return None + width_str, _, height_str = ratio.partition(":") + if not (width_str.isdigit() and height_str.isdigit()): + return None + area: Final = int(width_str) * int(height_str) + return next((label for threshold, label in _RESOLUTION_AREA_TIERS if area < threshold), "4k") + + +def _duration_seconds(seconds: str | None) -> float | None: + if not seconds: + return None + try: + return float(seconds) + except ValueError: + return None + + +def _estimated_cost_usd(response_data: _RunwayTaskResponse) -> float | None: + estimated_cost: Final = response_data.get("estimatedCost") + if not isinstance(estimated_cost, Mapping): + return None + credits: Final = estimated_cost.get("credits") + if not isinstance(credits, (int, float)): + return None + return float(credits) * _USD_PER_CREDIT + + +def _progress_percent(progress: float) -> int: + return min(100, max(0, round(float(progress) * 100))) + + class RunwayMLVideoConfig(BaseVideoConfig): """ Configuration class for RunwayML video generation. RunwayML uses a task-based API where: - 1. POST /v1/image_to_video creates a task + 1. POST /v1/text_to_video, /v1/image_to_video, or /v1/video_to_video creates a task 2. The task returns immediately with a task ID 3. Client must poll or wait for task completion """ @@ -195,31 +243,36 @@ class RunwayMLVideoConfig(BaseVideoConfig): """ Transform the video creation request for RunwayML API. - RunwayML expects: - { - "model": "gen4_turbo", - "promptImage": "https://... or data:image/...", - "promptText": "description", - "ratio": "1280:720", - "duration": 5 - } + RunwayML has three generation endpoints discriminated by which input is + present, and each request body rejects unknown fields: + - /text_to_video: promptText only (rejects promptImage) + - /image_to_video: promptImage (+ optional promptText) + - /video_to_video: promptVideo or videoUri (rejects promptImage) """ - # Build the request data + merged_params: Final = MappingProxyType( + { + "model": model, + "promptText": prompt, + **video_create_optional_request_params, + } + ) + + endpoint: Final = self._select_generation_endpoint(merged_params) + request_data: Final[dict[str, object]] = { - "model": model, - "promptText": prompt, + key: value for key, value in merged_params.items() if endpoint == "image_to_video" or key != "promptImage" } - # Add mapped parameters - request_data.update(video_create_optional_request_params) - - # RunwayML uses JSON body, no files multipart files_list: Final[RequestFiles] = [] - # Append the specific endpoint for video generation - full_api_base: Final = f"{api_base}/image_to_video" + return request_data, files_list, f"{api_base}/{endpoint}" - return request_data, files_list, full_api_base + def _select_generation_endpoint(self, request_data: Mapping[str, object]) -> str: + if request_data.get("promptVideo") is not None or request_data.get("videoUri") is not None: + return "video_to_video" + if request_data.get("promptImage") is not None: + return "image_to_video" + return "text_to_video" def transform_video_create_response( self, @@ -285,13 +338,15 @@ class RunwayMLVideoConfig(BaseVideoConfig): video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) # Add usage data for cost tracking - usage_data: Final = {} - if video_obj and hasattr(video_obj, "seconds") and video_obj.seconds: - try: - usage_data["duration_seconds"] = float(video_obj.seconds) - except (ValueError, TypeError): - pass - video_obj.usage = usage_data + video_obj.usage = { + key: value + for key, value in ( + ("duration_seconds", _duration_seconds(video_obj.seconds)), + ("video_resolution", _ratio_to_resolution(request_data.get("ratio") if request_data else None)), + ("provider_reported_cost_usd", _estimated_cost_usd(response_data)), + ) + if value is not None + } return video_obj @@ -582,7 +637,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt")) if "progress" in response_data: - video_data["progress"] = response_data["progress"] + video_data["progress"] = _progress_percent(response_data["progress"]) if "failureCode" in response_data or "failure" in response_data: video_data["error"] = { @@ -646,9 +701,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): raise NotImplementedError("video extension is not supported for RunwayML") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: - from ...base_llm.chat.transformation import BaseLLMException - - raise BaseLLMException( + return RunwayMLError( status_code=status_code, message=error_message, headers=headers, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3af7d9e5019..3c80e645d6b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43795,10 +43795,10 @@ "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" } }, - "runwayml/gen4_aleph": { + "runwayml/gen4.5": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.15, + "output_cost_per_second": 0.12, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43808,13 +43808,136 @@ "video" ], "metadata": { - "comment": "15 credits per second @ $0.01 per credit = $0.15 per second" + "comment": "12 credits per second @ $0.01 per credit = $0.12 per second" } }, - "runwayml/gen3a_turbo": { + "runwayml/aleph2": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.05, + "output_cost_per_second": 0.28, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "28 credits per second @ $0.01 per credit = $0.28 per second; 56 credit minimum per task not modeled" + } + }, + "runwayml/seedance2": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.36, + "output_cost_per_second_1080p": 0.4, + "output_cost_per_second_4k": 1.5, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit" + } + }, + "runwayml/seedance2_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.29, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second" + } + }, + "runwayml/seedance2_mini": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled" + } + }, + "runwayml/seedance2_5": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.3, + "output_cost_per_second_480p": 0.2, + "output_cost_per_second_1080p": 0.68, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" + } + }, + "runwayml/hailuo3": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second at 768P, 15 at 2K (mapped to the 1080p tier) @ $0.01 per credit; 2 credits per reference image not modeled" + } + }, + "runwayml/gemini_omni_flash": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second @ $0.01 per credit = $0.10 per second" + } + }, + "runwayml/veo3.1": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.4, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43824,7 +43947,23 @@ "video" ], "metadata": { - "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" + "comment": "40 credits per second with audio, 20 without @ $0.01 per credit; priced at the with-audio rate" + } + }, + "runwayml/veo3.1_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "15 credits per second with audio, 10 without @ $0.01 per credit; priced at the with-audio rate" } }, "runwayml/gen4_image": { diff --git a/litellm/types/router.py b/litellm/types/router.py index 99a4603ae49..9fd5cfa96ef 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -10,7 +10,7 @@ from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hin import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from typing_extensions import Protocol, Required, TypedDict, runtime_checkable +from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._uuid import uuid @@ -480,7 +480,9 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): output_cost_per_token: float | None input_cost_per_second: float | None output_cost_per_second: float | None + output_cost_per_second_480p: ReadOnly[float | None] output_cost_per_second_1080p: float | None + output_cost_per_second_4k: ReadOnly[float | None] num_retries: int | None ## MOCK RESPONSES ## mock_response: str | ModelResponse | Exception | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 67eae2b4f21..b9e8518a683 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -277,6 +277,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_second_1080p: ( float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) + output_cost_per_second_480p: ReadOnly[float | None] + output_cost_per_second_4k: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_credit: float | None # for OCR models priced by credit annotation_cost_per_page: float | None # for OCR models @@ -3331,6 +3333,8 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_second: float | None = None output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None + output_cost_per_second_480p: float | None = None + output_cost_per_second_4k: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..1d9dec6cfe8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5726,6 +5726,8 @@ def _get_model_info_helper( ), output_cost_per_second=_model_info.get("output_cost_per_second", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), + output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), + output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3af7d9e5019..3c80e645d6b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43795,10 +43795,10 @@ "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" } }, - "runwayml/gen4_aleph": { + "runwayml/gen4.5": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.15, + "output_cost_per_second": 0.12, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43808,13 +43808,136 @@ "video" ], "metadata": { - "comment": "15 credits per second @ $0.01 per credit = $0.15 per second" + "comment": "12 credits per second @ $0.01 per credit = $0.12 per second" } }, - "runwayml/gen3a_turbo": { + "runwayml/aleph2": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_video_per_second": 0.05, + "output_cost_per_second": 0.28, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "28 credits per second @ $0.01 per credit = $0.28 per second; 56 credit minimum per task not modeled" + } + }, + "runwayml/seedance2": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.36, + "output_cost_per_second_1080p": 0.4, + "output_cost_per_second_4k": 1.5, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit" + } + }, + "runwayml/seedance2_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.29, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second" + } + }, + "runwayml/seedance2_mini": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled" + } + }, + "runwayml/seedance2_5": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.3, + "output_cost_per_second_480p": 0.2, + "output_cost_per_second_1080p": 0.68, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" + } + }, + "runwayml/hailuo3": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second at 768P, 15 at 2K (mapped to the 1080p tier) @ $0.01 per credit; 2 credits per reference image not modeled" + } + }, + "runwayml/gemini_omni_flash": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "10 credits per second @ $0.01 per credit = $0.10 per second" + } + }, + "runwayml/veo3.1": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.4, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -43824,7 +43947,23 @@ "video" ], "metadata": { - "comment": "5 credits per second @ $0.01 per credit = $0.05 per second" + "comment": "40 credits per second with audio, 20 without @ $0.01 per credit; priced at the with-audio rate" + } + }, + "runwayml/veo3.1_fast": { + "litellm_provider": "runwayml", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "metadata": { + "comment": "15 credits per second with audio, 10 without @ $0.01 per credit; priced at the with-audio rate" } }, "runwayml/gen4_image": { diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 599ad016827..6f7ea9da640 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -867,6 +867,7 @@ PROVIDERS_WITH_A_HANDLER = ( "openrouter", "perplexity", "replicate", + "runwayml", "sagemaker", "together_ai", "vertex_ai", @@ -956,6 +957,7 @@ PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW = ( "mistral", "openai", "perplexity", + "runwayml", "together_ai", "vertex_ai", "xai", @@ -971,6 +973,7 @@ PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK = ( "mistral", "openai", "perplexity", + "runwayml", "together_ai", "xai", ) diff --git a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py index 24879ce83f9..e52cf9211ec 100644 --- a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py +++ b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py @@ -7,7 +7,12 @@ from unittest.mock import Mock import httpx import pytest -from litellm.llms.runwayml.videos.transformation import RunwayMLVideoConfig +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.runwayml.videos.transformation import ( + RunwayMLError, + RunwayMLVideoConfig, + _ratio_to_resolution, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject @@ -49,6 +54,139 @@ class TestRunwayMLVideoTransformation: # Validate URL has correct endpoint assert url == "https://api.dev.runwayml.com/v1/image_to_video" + def test_transform_video_create_request_text_to_video(self): + """A prompt-only request must hit /text_to_video, not /image_to_video.""" + data, files, url = self.config.transform_video_create_request( + model="veo3.1", + prompt="A serene mountain lake at sunrise", + api_base="https://api.dev.runwayml.com/v1", + video_create_optional_request_params={"duration": 8, "ratio": "1280:720"}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.dev.runwayml.com/v1/text_to_video" + assert "promptImage" not in data + assert data["promptText"] == "A serene mountain lake at sunrise" + + def test_transform_video_create_request_video_to_video(self): + """A promptVideo request must hit /video_to_video with promptImage stripped.""" + data, files, url = self.config.transform_video_create_request( + model="aleph2", + prompt="Make it snow", + api_base="https://api.dev.runwayml.com/v1", + video_create_optional_request_params={ + "promptVideo": "https://example.com/source.mp4", + "promptImage": "https://example.com/reference.png", + "ratio": "1280:720", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.dev.runwayml.com/v1/video_to_video" + assert data["promptVideo"] == "https://example.com/source.mp4" + assert "promptImage" not in data + + def test_transform_video_create_request_video_uri_routes_to_video_to_video(self): + _, _, url = self.config.transform_video_create_request( + model="aleph2", + prompt="Make it snow", + api_base="https://api.dev.runwayml.com/v1", + video_create_optional_request_params={"videoUri": "https://example.com/source.mp4"}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert url == "https://api.dev.runwayml.com/v1/video_to_video" + + def test_status_progress_fraction_scales_to_percent(self): + """Runway reports progress as a 0..1 float; VideoObject.progress is an int percent.""" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "63fd0f13-f29d-4e58-99d3-1cb9efa14a5b", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "RUNNING", + "progress": 0.027, + } + + result = self.config.transform_video_status_retrieve_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + ) + + assert result.status == "in_progress" + assert result.progress == 3 + + def test_get_error_class_returns_exception_instead_of_raising(self): + error = self.config.get_error_class( + error_message="Invalid API key", + status_code=401, + headers={}, + ) + + assert isinstance(error, RunwayMLError) + assert isinstance(error, BaseLLMException) + assert error.status_code == 401 + assert error.message == "Invalid API key" + + def test_create_response_usage_includes_resolution_and_provider_cost(self): + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "test-video-id-123", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "PENDING", + "estimatedCost": {"credits": 25.0}, + } + + video_obj = self.config.transform_video_create_response( + model="gen4_turbo", + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + request_data={"model": "gen4_turbo", "ratio": "1280:720", "duration": 5}, + ) + + assert video_obj.usage == { + "duration_seconds": 5.0, + "video_resolution": "720p", + "provider_reported_cost_usd": 0.25, + } + + def test_create_response_usage_omits_unknown_fields(self): + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "test-video-id-123", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "PENDING", + } + + video_obj = self.config.transform_video_create_response( + model="gen4_turbo", + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + request_data={"model": "gen4_turbo"}, + ) + + assert video_obj.usage == {} + + @pytest.mark.parametrize( + "ratio,expected", + [ + ("848:480", "480p"), + ("1280:720", "720p"), + ("1920:1080", "1080p"), + ("2560:1440", "1080p"), + ("3840:2160", "4k"), + (None, None), + ("banana", None), + ], + ) + def test_ratio_to_resolution_tiers(self, ratio, expected): + assert _ratio_to_resolution(ratio) == expected + def test_transform_video_status_with_timestamp_handling(self): """Test status retrieval handles RunwayML's ISO 8601 timestamps correctly.""" from litellm.types.videos.utils import encode_video_id_with_provider diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d655eb96a02..e817924f5fb 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -730,7 +730,9 @@ def validate_model_cost_values(model_data, exceptions=None): "output_cost_per_pixel", "input_cost_per_second", "output_cost_per_second", + "output_cost_per_second_480p", "output_cost_per_second_1080p", + "output_cost_per_second_4k", "input_cost_per_query", "input_cost_per_request", "input_cost_per_audio_token", @@ -944,7 +946,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_video_token": {"type": "number"}, "output_cost_per_pixel": {"type": "number"}, "output_cost_per_second": {"type": "number"}, + "output_cost_per_second_480p": {"type": "number"}, "output_cost_per_second_1080p": {"type": "number"}, + "output_cost_per_second_4k": {"type": "number"}, "output_cost_per_token": {"type": "number"}, "output_cost_per_token_above_128k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens": {"type": "number"}, @@ -1124,6 +1128,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): exceptions = [ # Add any model IDs that should be exempt from the cost validation # Example: "expensive-model-id", + "runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second ] is_valid, violations = validate_model_cost_values(actual_json, exceptions) diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index fb167a8624e..9125521d7a4 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -421,6 +421,117 @@ class TestVideoGeneration: ) assert cost == 0.5 + def test_completion_cost_video_custom_pricing_under_litellm_metadata(self): + """Video routes store deployment model_info under litellm_metadata, not metadata. + + Regression for https://github.com/BerriAI/litellm/issues/36483: custom video + pricing was silently ignored because completion_cost only read metadata. + """ + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = {"duration_seconds": 10.0} + type(mock_response)._hidden_params = {} + + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "litellm_metadata": { + "model_info": { + "output_cost_per_video_per_second": 0.18, + } + } + } + + cost = completion_cost( + completion_response=mock_response, + model="runwayml/seedance2", + call_type="create_video", + custom_llm_provider="runwayml", + custom_pricing=True, + litellm_logging_obj=mock_logging_obj, + ) + assert abs(cost - 1.8) < 0.001 + + def test_completion_cost_video_uses_provider_reported_cost_without_custom_pricing(self): + """With no custom pricing, the provider's own reported cost wins over a duration estimate.""" + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = { + "duration_seconds": 5.0, + "video_resolution": "720p", + "provider_reported_cost_usd": 0.31, + } + type(mock_response)._hidden_params = {} + + cost = completion_cost( + completion_response=mock_response, + model="runwayml/gen4_turbo", + call_type="create_video", + custom_llm_provider="runwayml", + ) + assert cost == 0.31 + + def test_completion_cost_video_custom_pricing_beats_provider_reported_cost(self): + """Deployment-level custom pricing overrides the provider's reported cost.""" + from litellm.cost_calculator import completion_cost + + mock_response = MagicMock() + mock_response.usage = { + "duration_seconds": 10.0, + "provider_reported_cost_usd": 0.31, + } + type(mock_response)._hidden_params = {} + + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "metadata": { + "model_info": { + "output_cost_per_video_per_second": 0.18, + } + } + } + + cost = completion_cost( + completion_response=mock_response, + model="runwayml/seedance2", + call_type="create_video", + custom_llm_provider="runwayml", + custom_pricing=True, + litellm_logging_obj=mock_logging_obj, + ) + assert abs(cost - 1.8) < 0.001 + + def test_completion_cost_video_resolution_tiers_from_cost_map(self, monkeypatch): + """The 480p/1080p/4k tier keys resolve from the shipped runwayml cost map entries.""" + from litellm.cost_calculator import completion_cost + + local_map_path = os.path.join( + os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" + ) + with open(local_map_path, "r") as f: + monkeypatch.setattr(litellm, "model_cost", json.load(f)) + + def cost_for(model: str, resolution: str | None, duration: float) -> float: + mock_response = MagicMock() + mock_response.usage = { + "duration_seconds": duration, + **({"video_resolution": resolution} if resolution else {}), + } + type(mock_response)._hidden_params = {} + return completion_cost( + completion_response=mock_response, + model=model, + call_type="create_video", + custom_llm_provider="runwayml", + ) + + assert abs(cost_for("runwayml/seedance2", "4k", 8.0) - 12.0) < 0.001 + assert abs(cost_for("runwayml/seedance2", "1080p", 8.0) - 3.2) < 0.001 + assert abs(cost_for("runwayml/seedance2", "720p", 8.0) - 2.88) < 0.001 + assert abs(cost_for("runwayml/seedance2_5", "480p", 8.0) - 1.6) < 0.001 + assert abs(cost_for("runwayml/gen4.5", None, 8.0) - 0.96) < 0.001 + def test_video_generation_with_files(self): """Test video generation with file uploads.""" config = OpenAIVideoConfig() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d311bfa3cbc..4b285a6efee 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27624,6 +27624,10 @@ export interface components { output_cost_per_second?: number | null; /** Output Cost Per Second 1080P */ output_cost_per_second_1080p?: number | null; + /** Output Cost Per Second 480P */ + output_cost_per_second_480p?: number | null; + /** Output Cost Per Second 4K */ + output_cost_per_second_4k?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ @@ -36833,6 +36837,10 @@ export interface components { output_cost_per_second?: number | null; /** Output Cost Per Second 1080P */ output_cost_per_second_1080p?: number | null; + /** Output Cost Per Second 480P */ + output_cost_per_second_480p?: number | null; + /** Output Cost Per Second 4K */ + output_cost_per_second_4k?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ From bf4069a80c46a3da804e1d078f9052e9b8ce449c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:24:21 -0700 Subject: [PATCH 38/54] fix(proxy): authorize health test-connection on final probe params Move _update_litellm_params_for_health_check before can_user_make_model_call so health_check_params cannot retarget the probe after the auth check. Type the Pegasus test helper signature and drop the redundant test narrative. --- .../proxy/health_endpoints/_health_endpoints.py | 11 ++++++----- .../proxy/test_health_check_max_tokens.py | 15 +++------------ 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 1feddca9328..c23cde6052e 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1952,8 +1952,13 @@ async def test_model_connection( **request_litellm_params, } - ## Auth check resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info + litellm_params = _update_litellm_params_for_health_check( + model_info=resolved_model_info or {}, + litellm_params=litellm_params, + ) + + ## Auth check, on the final probe params so health_check_params cannot retarget it afterwards await ModelManagementAuthChecks.can_user_make_model_call( model_params=Deployment( model_name="test_model", @@ -1964,10 +1969,6 @@ async def test_model_connection( prisma_client=prisma_client, premium_user=premium_user, ) - litellm_params = _update_litellm_params_for_health_check( - model_info=resolved_model_info or {}, - litellm_params=litellm_params, - ) mode = mode or litellm_params.pop("mode", None) result: Final = await run_with_timeout( diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index e20b18c8813..162360328d5 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -549,17 +549,6 @@ async def test_run_model_health_check_skips_auto_router_deployment(): assert result == {} -# --------------------------------------------------------------------------- -# model_info.health_check_params -# -# Some providers require a payload field litellm does not synthesize for a -# probe. Bedrock TwelveLabs Pegasus rejects any Invoke body without a top-level -# `mediaSource`, so every health check on such a deployment failed with -# "Invalid JSON: $: required property 'mediaSource' not found". The config key -# was accepted and then never read, so operators had no way to supply it. -# --------------------------------------------------------------------------- - - def test_health_check_params_merge_into_probe_params(): """health_check_params reach the probe request for the deployment that declares them.""" media_source = {"s3Location": {"uri": "s3://my-bucket/clip.mp4"}} @@ -640,7 +629,9 @@ def test_health_check_params_apply_to_non_chat_modes(): assert "max_tokens" not in updated -async def _pegasus_health_check_request_body(model_info: dict, monkeypatch) -> dict: +async def _pegasus_health_check_request_body( + model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> dict[str, object]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() From b117190b0ba8ac3d9fe288318fc062f0d316f6e2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:43:40 -0700 Subject: [PATCH 39/54] chore(pricing): regenerate model prices schema for new video cost tier fields --- model_prices_and_context_window.schema.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f5560a20ab2..60a016ecf37 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -428,6 +428,14 @@ "type": "number", "minimum": 0 }, + "output_cost_per_second_480p": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_second_4k": { + "type": "number", + "minimum": 0 + }, "output_cost_per_token": { "type": "number", "minimum": 0, From b36f34813a5e718bc3de77297daddfa89233baad Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Mon, 24 Aug 2026 14:43:47 -0400 Subject: [PATCH 40/54] fix(anthropic): reconcile enum with declared type in output_format schema (#37882) * fix(anthropic): reconcile enum with declared type in output_format schema Anthropic cross-validates `enum` against `type` in structured outputs: every enum value must match a single declared type. A union `type` array, or an enum value whose JSON type differs from a scalar `type`, is rejected with "Invalid schema: Enum value 'low' does not match declared type '['string','null']'" filter_anthropic_output_schema had no enum/type reconciliation, so both keys reached Anthropic untouched. Drop the conflicting `type` -- `enum` is the tighter constraint, and an enum with no `type` is accepted The drop is conditional: `type` is only removed when it is a union array, or when some enum value does not match the scalar type. A matching enum plus scalar `type` is left exactly as-is, so existing behaviour is unchanged Pydantic emits the failing shape for Optional[SomeEnum], so this affects any caller with a nullable enum field on the native output_format path. vertex_ai is unaffected because it is forced onto the permissive tool-use path Fixes #37881 * refactor(anthropic): make enum/type reconciliation immutable and precisely typed Address review: the predicate registry was a mutable `dict[str, Any]`, and the reconciliation removed `type` by mutating the built result with `pop` - registry is now `Final[Mapping[str, Callable[[Any], bool]]]` wrapped in `MappingProxyType`, so predicate signatures are statically checked and the table cannot be mutated - the conflict decision moves into a pure helper evaluated once against the input schema, and the conflicting `type` key is skipped at build time in the existing loop instead of being popped afterwards, so nothing is mutated Behaviour is unchanged; all 27 tests in the schema-filter suite still pass --- litellm/llms/anthropic/chat/transformation.py | 33 ++++++- .../anthropic/test_anthropic_schema_filter.py | 91 +++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ef278c8f723..23abca7d5f2 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1,7 +1,8 @@ import json import re import time -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, cast import httpx @@ -121,6 +122,32 @@ else: # response side. _ANTHROPIC_TOOL_NAME_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]") _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128 + +_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[Any], bool]]] = MappingProxyType( + { + "null": lambda v: v is None, + "boolean": lambda v: isinstance(v, bool), + "integer": lambda v: isinstance(v, int) and not isinstance(v, bool), + "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), + "string": lambda v: isinstance(v, str), + "array": lambda v: isinstance(v, list), + "object": lambda v: isinstance(v, dict), + } +) + + +def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool: + """Whether ``schema``'s ``enum`` cannot match its declared ``type``.""" + enum_values: Final = schema.get("enum") + declared_type: Final = schema.get("type") + if not isinstance(enum_values, list) or declared_type is None: + return False + if isinstance(declared_type, list): + return True + check: Final = _ENUM_TYPE_CHECKS.get(declared_type) + return check is not None and not all(check(value) for value in enum_values) + + # Single, internal-only key on ``litellm_params`` used to thread the per- # request reverse map (sanitized -> original) from request build to response # parsing. ``litellm_params`` is never serialized to a provider; ``optional_ @@ -565,9 +592,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: result["description"] = constraint_note + drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema) + for key, value in schema.items(): if key in unsupported_fields: continue + if key == "type" and drops_conflicting_type: + continue if key == "description" and "description" in result: # Already handled above continue diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_schema_filter.py b/tests/test_litellm/llms/anthropic/test_anthropic_schema_filter.py index 71c9cfe8f41..32423d6679f 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_schema_filter.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_schema_filter.py @@ -417,3 +417,94 @@ class TestFilterAnthropicOutputSchema: result = AnthropicConfig.filter_anthropic_output_schema(schema) assert result["additionalProperties"] is False + + def test_drops_union_type_alongside_enum(self): + """A union ``type`` can never match a single declared type. + + Anthropic rejects it with "Invalid schema: Enum value 'low' does not + match declared type '['string', 'null']'". ``enum`` is the tighter + constraint, so the conflicting ``type`` is dropped. + """ + schema = { + "type": "object", + "properties": { + "confidence": { + "enum": ["low", "medium", "high", None], + "type": ["string", "null"], + } + }, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert "type" not in result["properties"]["confidence"] + assert result["properties"]["confidence"]["enum"] == [ + "low", + "medium", + "high", + None, + ] + + def test_drops_type_when_an_enum_value_does_not_match_it(self): + """``enum: ["x", None]`` with ``type: "string"`` is rejected too.""" + schema = { + "type": "object", + "properties": {"a": {"enum": ["x", None], "type": "string"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert "type" not in result["properties"]["a"] + + def test_preserves_type_when_every_enum_value_matches(self): + """The non-conflicting case must be left exactly as-is.""" + schema = { + "type": "object", + "properties": {"a": {"enum": ["x", "y"], "type": "string"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["a"]["type"] == "string" + assert result["properties"]["a"]["enum"] == ["x", "y"] + + def test_integer_enum_satisfies_number_type(self): + """JSON Schema ``number`` accepts integers, so this is not a conflict.""" + schema = { + "type": "object", + "properties": {"a": {"enum": [1, 2], "type": "number"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["a"]["type"] == "number" + + def test_bool_enum_does_not_satisfy_integer_type(self): + """``bool`` is a Python ``int`` subclass but is not a JSON integer.""" + schema = { + "type": "object", + "properties": {"a": {"enum": [True], "type": "integer"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert "type" not in result["properties"]["a"] + + def test_normalizes_enum_type_inside_array_items(self): + """Normalization applies at every recursion site, not just top level.""" + schema = { + "type": "object", + "properties": { + "rows": { + "type": "array", + "items": { + "type": "object", + "properties": {"c": {"enum": ["a", None], "type": ["string", "null"]}}, + }, + } + }, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert "type" not in result["properties"]["rows"]["items"]["properties"]["c"] From 25b379f3c7ce837ee60a0a5952cb90cfcb7d768a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:47:21 -0700 Subject: [PATCH 41/54] test: register supports_legacy_thinking in model-prices schema Regenerate model_prices_and_context_window.schema.json and add the flag to the inline validator schema in test_utils.py so the new cost-map key passes validate-model-prices-json and the JSON-valid test. --- model_prices_and_context_window.schema.json | 3 +++ tests/test_litellm/test_utils.py | 1 + 2 files changed, 4 insertions(+) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f5560a20ab2..aeb88962305 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -625,6 +625,9 @@ "supports_image_size": { "type": "boolean" }, + "supports_legacy_thinking": { + "type": "boolean" + }, "supports_low_reasoning_effort": { "type": "boolean" }, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d655eb96a02..cb451d02efd 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -993,6 +993,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"}, "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, From 0322107414b30948b4e0eb49d65313f1993b0815 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:51:28 -0700 Subject: [PATCH 42/54] fix(images): flatten nested image-edit params to SDK multipart form The openai/azure/compat image-edit funnel merged non_default_params and extra_body straight into the multipart body, so a nested value (e.g. extra_body={"metadata": {...}}) reached the httpx encoder and 500'd with "Invalid type for value. Expected primitive type". Route the funnel through a shared flattener that serializes nested values as OpenAI-SDK bracket fields (key[subkey], lists as key[], bools lowercased, None/empty dropped), matching the wire format of the rest of this fix. --- litellm/images/main.py | 10 +++++-- .../litellm_core_utils/llm_request_utils.py | 19 ++++++++++++ .../images/test_image_edit_extra_params.py | 24 +++++++++++++++ .../test_llm_request_utils.py | 30 ++++++++++++++++++- 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/litellm/images/main.py b/litellm/images/main.py index fd18edc66fb..e45adda2526 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT from litellm.exceptions import LiteLLMUnknownProvider from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values from litellm.litellm_core_utils.mock_functions import mock_image_generation from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -851,9 +852,12 @@ def image_edit( or custom_llm_provider == "azure" or custom_llm_provider in litellm.openai_compatible_providers ): - image_edit_request_params.update(non_default_params) - if isinstance(extra_body, dict): - image_edit_request_params.update(extra_body) + image_edit_request_params.update( + flatten_form_field_values( + non_default_params, + extra_body if isinstance(extra_body, dict) else None, + ) + ) # Pre Call logging litellm_logging_obj.update_from_kwargs( diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 33b402789b3..5e822971e8f 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -27,6 +27,25 @@ def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]: return ((key, serialized),) +def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str], ...]: + """ + Flatten JSON-shaped bodies into primitive ``(name, value)`` form fields the + way the OpenAI SDK serializes multipart bodies: dicts as ``key[subkey]``, + lists as ``key[]``, booleans lowercased, None and empty values dropped. + Sources are applied in order, so a later source wins on a key collision when + fed to ``dict.update``. Used to funnel provider-specific params into a + multipart request without handing the httpx encoder a nested value it + rejects with ``Invalid type for value``. + """ + return tuple( + pair + for source in sources + if source is not None + for top_key, top_value in source.items() + for pair in _flatten_form_field(top_key, top_value) + ) + + def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]: """ Encode a JSON-shaped body as httpx file-tuples so a request with no file diff --git a/tests/test_litellm/images/test_image_edit_extra_params.py b/tests/test_litellm/images/test_image_edit_extra_params.py index 46a5feb08a0..01490cdd988 100644 --- a/tests/test_litellm/images/test_image_edit_extra_params.py +++ b/tests/test_litellm/images/test_image_edit_extra_params.py @@ -76,6 +76,30 @@ def test_image_edit_extra_body_takes_precedence_over_kwargs(): assert _multipart_text_fields(captured["content_type"], captured["body"])["seed"] == "7" +def test_image_edit_flattens_nested_provider_params(): + """A nested value in extra_body (or a nested unknown kwarg) must be + serialized as OpenAI-SDK bracket form fields (key[subkey]) rather than + handed to the httpx multipart encoder, which raises 'Invalid type for + value. Expected primitive type' on a dict and 500s the request.""" + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_image_edit_request(captured)))) + + litellm.image_edit( + model="openai/gpt-image-1", + image=PNG_BYTES, + prompt="add a hat", + api_key="sk-test", + api_base="https://edit.example/v1", + client=client, + extra_body={"generation_config": {"steps": 30, "guidance": True}}, + ) + + fields = _multipart_text_fields(captured["content_type"], captured["body"]) + assert fields["generation_config[steps]"] == "30" + assert fields["generation_config[guidance]"] == "true" + assert "generation_config" not in fields + + @pytest.mark.asyncio async def test_aimage_edit_forwards_extra_body(): """aimage_edit used to drop extra_headers/extra_query/extra_body when diff --git a/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py b/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py index bd4f8943b47..0140d4ff232 100644 --- a/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py @@ -1,4 +1,7 @@ -from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields +from litellm.litellm_core_utils.llm_request_utils import ( + flatten_form_field_values, + serialize_multipart_form_fields, +) def test_serialize_multipart_form_fields_flattens_like_the_openai_sdk(): @@ -34,3 +37,28 @@ def test_serialize_multipart_form_fields_drops_empty_strings(): def test_serialize_multipart_form_fields_empty_body(): assert serialize_multipart_form_fields({}) == () + + +def test_flatten_form_field_values_flattens_nested_and_drops_empty(): + assert flatten_form_field_values( + { + "seed": 42, + "hd": True, + "size": None, + "prompt": "", + "generation_config": {"steps": 30, "guidance": True}, + } + ) == ( + ("seed", "42"), + ("hd", "true"), + ("generation_config[steps]", "30"), + ("generation_config[guidance]", "true"), + ) + + +def test_flatten_form_field_values_later_source_wins_on_collision(): + assert flatten_form_field_values({"seed": 1}, None, {"seed": 2}) == ( + ("seed", "1"), + ("seed", "2"), + ) + assert dict(flatten_form_field_values({"seed": 1}, {"seed": 2}))["seed"] == "2" From 51ab4c74867dc6a862f5ce3db7c5c6fdccf1d41c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:54:29 -0700 Subject: [PATCH 43/54] test(videos): lock AzureVideoConfig inherited file-less multipart behavior AzureVideoConfig subclasses OpenAIVideoConfig and so inherits the new use_multipart_form_data() -> True. Azure's /openai/v1/videos surface is OpenAI-SDK-compatible, so the JSON->multipart flip is intentional; assert it through the real handler so the inherited behavior can't silently regress. --- .../custom_httpx/test_llm_http_handler.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index b78829e2e11..a93b14d45f3 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -26,6 +26,7 @@ from litellm.llms.custom_httpx.llm_http_handler import ( _has_pre_call_deployment_hook, _rust_responses_websocket_enabled, ) +from litellm.llms.azure.videos.transformation import AzureVideoConfig from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams @@ -2604,6 +2605,27 @@ async def test_async_video_generation_without_file_sends_multipart_form_data(): assert result.status == "queued" +def test_azure_video_generation_without_file_sends_multipart_form_data(): + """AzureVideoConfig subclasses OpenAIVideoConfig, so it inherits the + file-less multipart behavior. Azure's /openai/v1/videos surface is + OpenAI-SDK-compatible (the SDK sends multipart there too), so this is + intentional; lock it so the inherited flip can't silently regress to JSON.""" + assert AzureVideoConfig().use_multipart_form_data() is True + + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) + + result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(AzureVideoConfig())) + + assert captured["content_type"].startswith("multipart/form-data") + assert _multipart_text_fields(captured["content_type"], captured["body"]) == { + "model": "sora-2", + "prompt": "a cat surfing", + "seconds": "4", + } + assert result.status == "queued" + + def test_video_generation_json_provider_keeps_json_body(): captured = {} client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) From a1134755cab81e875b3a1294eda34a7b6a7b979f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:57:36 -0700 Subject: [PATCH 44/54] fix(ui): boot the UI image as an arbitrary uid by anchoring nginx writes under /tmp (#37982) * fix(ui): boot the UI image as an arbitrary uid by anchoring nginx writes under /tmp Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): type the arbitrary-uid image test fixture Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/image-scan.yml | 31 ++++ helm/litellm/values.yaml | 8 +- .../test_ui_image_serves_offline.py | 132 ++++++++++++++++++ ui/nginx.conf | 13 ++ 4 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 tests/proxy_migration_tests/test_ui_image_serves_offline.py diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index d798df4c3a4..bb04563c1a8 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -23,6 +23,8 @@ on: - tests/proxy_migration_tests/** - uv.lock - ui/litellm-dashboard/package-lock.json + - ui/Dockerfile + - ui/nginx.conf - .github/workflows/image-scan.yml schedule: - cron: "41 6 * * *" @@ -185,6 +187,35 @@ jobs: python -m pip install "pytest==9.0.3" python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v + ui-image: + name: ui-image + runs-on: ubuntu-latest + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Build UI image + run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} . + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs + env: + LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }} + run: | + python -m pip install "pytest==9.0.3" + python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v + backend-image: name: backend-image runs-on: ubuntu-latest diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 998d225a317..06ba72d84b3 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -428,9 +428,11 @@ ui: maxUnavailable: "" podAnnotations: {} # Same shape as the gateway blocks of the same name. The nginx runtime - # writes its pid, cache, and proxy temp files under the image's root - # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs - # emptyDir volumes mounted over those paths. + # writes its pid, cache, and proxy temp files under /tmp, so it boots as + # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem: + # true` here needs an emptyDir volume mounted over /tmp. Images before + # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to + # run as a non-root uid at all. podLabels: {} podSecurityContext: {} securityContext: {} diff --git a/tests/proxy_migration_tests/test_ui_image_serves_offline.py b/tests/proxy_migration_tests/test_ui_image_serves_offline.py new file mode 100644 index 00000000000..5ff68effd7f --- /dev/null +++ b/tests/proxy_migration_tests/test_ui_image_serves_offline.py @@ -0,0 +1,132 @@ +"""Image-level regression net for arbitrary-uid boot of the UI image. + +OpenShift ``restricted-v2`` ignores the image ``USER`` and assigns an +arbitrary uid in GID 0. The stock nginx base expects to start as root, so +its cache (``/var/cache/nginx``) and pid (``/run``) paths are root-owned +755 and the master process dies at startup with +``mkdir() "/var/cache/nginx/client_temp" failed (13: Permission denied)``. +The fix anchors everything nginx writes under ``/tmp`` in ``ui/nginx.conf``. + +Booting the image the way that deployment does, with a read-only root +filesystem and ``/tmp`` as the only writable mount, is what catches the +whole class: a boot as the default (root) uid passes even on the broken +config. + +Gated on LITELLM_IMAGE so it is skipped in the normal unit-test run and +exercised only where an image has been built (the image-scan workflow). +Requires a working docker CLI. +""" + +import os +import shutil +import subprocess +import time +import uuid +from collections.abc import Iterator + +import pytest + +IMAGE = os.getenv("LITELLM_IMAGE") +CURL_IMAGE = os.getenv("LITELLM_TEST_CURL_IMAGE", "curlimages/curl:8.11.1") +UI_PORT = os.getenv("LITELLM_UI_PORT", "3000") +ARBITRARY_UID = "1001200000:0" +STARTUP_TIMEOUT_SECONDS = int(os.getenv("LITELLM_UI_STARTUP_TIMEOUT", "60")) + +pytestmark = [ + pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"), + pytest.mark.skipif(shutil.which("docker") is None, reason="requires the docker CLI"), +] + + +def _docker(*args: str, check: bool = True) -> "subprocess.CompletedProcess[str]": + return subprocess.run(["docker", *args], capture_output=True, text=True, check=check) + + +@pytest.fixture() +def ui_container() -> Iterator[tuple[str, str]]: + """The UI container as an arbitrary uid in GID 0 on a network with no egress. + + ``--read-only`` with a tmpfs on ``/tmp`` mirrors the strictest supported + deployment: ``readOnlyRootFilesystem: true`` with an emptyDir on ``/tmp``. + A config that writes anywhere else fails here exactly like it does on + OpenShift. + """ + run_id = f"uiserve-{uuid.uuid4().hex[:8]}" + network = f"{run_id}-net" + container = f"{run_id}-ui" + + _docker("pull", "--quiet", CURL_IMAGE) + _docker("network", "create", "--internal", network) + try: + assert IMAGE is not None + _docker( + "run", "-d", "--name", container, "--network", network, + "--user", ARBITRARY_UID, + "--read-only", "--tmpfs", "/tmp", + IMAGE, + ) + yield network, container + finally: + _docker("logs", container, check=False) + _docker("rm", "-f", container, check=False) + _docker("network", "rm", network, check=False) + + +def _container_logs(container: str) -> str: + logs = _docker("logs", container, check=False) + return f"stdout:\n{logs.stdout}\nstderr:\n{logs.stderr}" + + +def _is_running(container: str) -> bool: + return bool( + _docker( + "ps", "--filter", f"name={container}", "--filter", "status=running", + "--format", "{{.Names}}", check=False, + ).stdout.strip() + ) + + +def _probe(network: str, container: str, path: str) -> "subprocess.CompletedProcess[str]": + return _docker( + "run", "--rm", "--network", network, CURL_IMAGE, + "--silent", "--show-error", "--max-time", "10", + "--output", "/dev/null", "--write-out", "%{http_code}", + f"http://{container}:{UI_PORT}{path}", + check=False, + ) + + +def test_ui_serves_as_arbitrary_uid_read_only(ui_container: tuple[str, str]) -> None: + """nginx boots and serves as an arbitrary uid with a read-only root fs. + + On the pre-fix config nginx exits during startup with + ``mkdir() "/var/cache/nginx/client_temp" failed (13: Permission denied)`` + and the running-check below fails; it never reaches the probes. + """ + network, container = ui_container + + deadline = time.time() + STARTUP_TIMEOUT_SECONDS + healthz = None + while time.time() < deadline: + if not _is_running(container): + pytest.fail( + f"the UI container exited during startup as uid {ARBITRARY_UID} with a " + f"read-only root filesystem. nginx writes outside /tmp.\n" + f"{_container_logs(container)}" + ) + healthz = _probe(network, container, "/healthz") + if healthz.returncode == 0 and healthz.stdout.strip() == "200": + break + time.sleep(2) + + assert healthz is not None and healthz.stdout.strip() == "200", ( + f"/healthz never answered 200 within {STARTUP_TIMEOUT_SECONDS}s as uid " + f"{ARBITRARY_UID}.\n{_container_logs(container)}" + ) + + for path in ("/", "/ui", "/ui/login"): + page = _probe(network, container, path) + assert page.stdout.strip() == "200", ( + f"GET {path} returned {page.stdout.strip()!r} as uid {ARBITRARY_UID}.\n" + f"{_container_logs(container)}" + ) diff --git a/ui/nginx.conf b/ui/nginx.conf index a41ee5bd5b4..235cb9c501e 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -1,7 +1,20 @@ worker_processes auto; + +# Anchor everything nginx writes under /tmp so the image boots as an +# arbitrary uid (OpenShift restricted-v2 assigns one in gid 0; the stock +# nginx image's /var/cache/nginx and /run are root-owned 755) and works +# with readOnlyRootFilesystem when /tmp is an emptyDir. +pid /tmp/nginx.pid; + events { worker_connections 1024; } http { + client_body_temp_path /tmp/nginx-client-temp; + proxy_temp_path /tmp/nginx-proxy-temp; + fastcgi_temp_path /tmp/nginx-fastcgi-temp; + uwsgi_temp_path /tmp/nginx-uwsgi-temp; + scgi_temp_path /tmp/nginx-scgi-temp; + include /etc/nginx/mime.types; default_type application/octet-stream; sendfile on; From 458a63935f55c63cf50b847c35c890784eac58b2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:58:33 -0700 Subject: [PATCH 45/54] test(health): guard that test_connection authorizes on post-merge probe params Add an endpoint-level regression test asserting can_user_make_model_call receives the litellm_params after health_check_params are merged in, so the merge-before-auth ordering cannot silently regress and let a request smuggle a field past authorization. --- .../health_endpoints/test_health_endpoints.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 62919200d47..72c6f77a4f7 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -898,6 +898,62 @@ async def test_test_model_connection_uses_loaded_deployment_team_id_via_model_na assert passed_model_params.model_info.team_id == deployment_owner_team_id +@pytest.mark.asyncio +async def test_test_model_connection_authorizes_on_params_after_health_check_params_merge(): + """ + Regression guard for the ordering fix: health_check_params from the request + body are merged into the probe params BEFORE the authorization check, so a + caller cannot smuggle a field past auth via health_check_params. Auth is + stubbed to reject, which halts the endpoint right after it records the + params it was handed, so the outbound probe is never reached. If the merge + is moved back to after can_user_make_model_call, the marker is absent from + those params and this test fails. + """ + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_management_endpoints import ( + ModelManagementAuthChecks, + ) + from litellm.types.router import Deployment + + marker = "sentinel-from-health-check-params" + mock_can_user_make_model_call = AsyncMock( + side_effect=HTTPException(status_code=403, detail="denied") + ) + + with ( + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.llm_router", None + ), + patch.object( # test-quality-ok: capturing the params handed to auth is the assertion + ModelManagementAuthChecks, + "can_user_make_model_call", + mock_can_user_make_model_call, + ), + pytest.raises(HTTPException), + ): + await health_test_model_connection( + request=MagicMock(), + mode="chat", + litellm_params={"model": "openai/gpt-4o"}, + model_info={"health_check_params": {"probe_marker": marker}}, + user_api_key_dict=UserAPIKeyAuth( + token="requester-token", + user_id="admin-user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ), + ) + + assert mock_can_user_make_model_call.called + passed_model_params = mock_can_user_make_model_call.call_args.kwargs["model_params"] + assert isinstance(passed_model_params, Deployment) + authorized_params = passed_model_params.litellm_params.model_dump() + assert authorized_params.get("probe_marker") == marker + + @pytest.mark.asyncio async def test_test_model_connection_authorized_team_admin_passes_real_auth(): """ From 7f0c1c76652139df45844676121e03689d43ebe5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:11:59 -0700 Subject: [PATCH 46/54] chore(videos): mark multi-branch video-create response as rebind-ok The file-less multipart branch added a third mutually-exclusive request-shape branch, so response can no longer be Final. Suppress the type-discipline gate the way the codebase does for other multi-branch locals. --- litellm/llms/custom_httpx/llm_http_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 53c38733ed7..76f109f0eb6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -7061,7 +7061,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) elif video_generation_provider_config.use_multipart_form_data(): - response = sync_httpx_client.post( + response = sync_httpx_client.post( # rebind-ok: one of three mutually-exclusive branches url=api_base, headers=headers, files=serialize_multipart_form_fields(data), @@ -7168,7 +7168,7 @@ class BaseLLMHTTPHandler: timeout=timeout, ) elif video_generation_provider_config.use_multipart_form_data(): - response = await async_httpx_client.post( + response = await async_httpx_client.post( # rebind-ok: one of three mutually-exclusive branches url=api_base, headers=headers, files=serialize_multipart_form_fields(data), From 6975b8ea4b48807844e026186a28f86e352f89ac Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 24 Aug 2026 12:19:03 -0700 Subject: [PATCH 47/54] fix(utils): make prompt_token_calculator count claude models again The claude branch called the anthropic SDK's `Anthropic().count_tokens`, which the SDK removed, so every claude call raised AttributeError. Counting now goes through litellm's own token_counter, which handles anthropic models offline and drops the SDK dependency entirely. Hiding that was a swallowed error: `except Exception: Exception("Anthropic import failed please run `pip install anthropic`")` built the exception without raising it, so an environment missing the SDK fell through to the unguarded `from anthropic import ...` on the next line and got a bare ModuleNotFoundError instead of the install hint. That was the codebase's last PLW0133, so the rule graduates from the ratcheted budget into ruff.toml where it hard-fails, and editors get the diagnostic inline. --- litellm/utils.py | 15 ++------------- ruff-strict-budget.json | 7 ++----- ruff.toml | 6 +++--- tests/test_litellm/test_utils.py | 18 ++++++++++++++++++ 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..467b8602c27 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6526,21 +6526,10 @@ def acreate(*args, **kwargs): ## Thin client to handle the acreate langchain ca def prompt_token_calculator(model, messages): - # use tiktoken or anthropic's tokenizer depending on the model text: Final = " ".join(message["content"] for message in messages) - num_tokens = 0 if "claude" in model: - try: - import anthropic - except Exception: - Exception("Anthropic import failed please run `pip install anthropic`") - from anthropic import AI_PROMPT, HUMAN_PROMPT, Anthropic - - anthropic_obj: Final = Anthropic() - num_tokens = anthropic_obj.count_tokens(text) - else: - num_tokens = len(_get_default_encoding().encode(text)) - return num_tokens + return token_counter(model=model, text=text) + return len(_get_default_encoding().encode(text)) def valid_model(model): diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index a990f7c3830..3b3c3e8ae5b 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2920 + "limit": 2919 }, "C401": { "limit": 8 @@ -108,7 +108,7 @@ "limit": 3 }, "F401": { - "limit": 17 + "limit": 14 }, "LOG015": { "limit": 5 @@ -152,9 +152,6 @@ "PLW0127": { "limit": 57 }, - "PLW0133": { - "limit": 1 - }, "PLW0602": { "limit": 215 }, diff --git a/ruff.toml b/ruff.toml index 9b90910b355..44bdf9d8125 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,9 +5,9 @@ lint.ignore = ["F405", "E402", "F403"] lint.extend-select = [ "T20", "PGH004", "RUF008", "RUF009", "RUF100", "B033", "FURB136", "FURB168", "FURB188", "I001", "PERF402", "PIE790", "PIE800", "PLC0208", - "PLR0402", "PLR1711", "PLR1730", "PLR2044", "PYI030", "PYI041", "PYI064", "RET501", "RUF010", - "RUF022", "RUF023", "RUF051", "SIM114", "SIM118", "TC005", "UP006", "UP007", "UP008", "UP012", - "UP018", "UP024", "UP032", "UP034", "UP035", "UP037", "UP045", + "PLR0402", "PLR1711", "PLR1730", "PLR2044", "PLW0133", "PYI030", "PYI041", "PYI064", "RET501", + "RUF010", "RUF022", "RUF023", "RUF051", "SIM114", "SIM118", "TC005", "UP006", "UP007", "UP008", + "UP012", "UP018", "UP024", "UP032", "UP034", "UP035", "UP037", "UP045", ] # RUF100 (unused-noqa) only knows the rules enabled in THIS config, so it would strip # `# noqa` directives that protect rules enforced elsewhere. List those codes as external diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d655eb96a02..97baf818d7c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,6 +1,7 @@ import json import logging import os +import sys from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -41,6 +42,7 @@ from litellm.utils import ( get_prompt_cache_min_tokens, is_cached_message, is_prompt_caching_valid_prompt, + prompt_token_calculator, ) # Adds the parent directory to the system path @@ -4973,3 +4975,19 @@ def test_completion_does_not_leak_rust_flag_into_provider_request_body(): create_kwargs = mock_client.chat.completions.with_raw_response.create.call_args.kwargs assert "rust" not in create_kwargs assert "rust" not in (create_kwargs.get("extra_body") or {}) + + +def test_prompt_token_calculator_counts_claude_without_the_anthropic_sdk(): + """ + The claude branch used to call the anthropic SDK's `count_tokens`, which the SDK + removed, so every claude call raised AttributeError. Counting must work with + `anthropic` unimportable. + """ + messages: Final = [{"role": "user", "content": "the quick brown fox jumps over the lazy dog"}] + + with patch.dict(sys.modules, {"anthropic": None}): + claude_tokens = prompt_token_calculator("claude-sonnet-4-5", messages) + gpt_tokens = prompt_token_calculator("gpt-4o", messages) + + assert claude_tokens == 9 + assert gpt_tokens == 9 From 3030e974b889e744500bd22e8e70a07326100e2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:20:00 -0700 Subject: [PATCH 48/54] fix(runwayml): skip progress scaling when Runway returns a null progress --- .../llms/runwayml/videos/transformation.py | 5 +++-- .../test_runway_video_transformation.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 065661a8731..2264f52e2a3 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -636,8 +636,9 @@ class RunwayMLVideoConfig(BaseVideoConfig): if "completedAt" in response_data: video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt")) - if "progress" in response_data: - video_data["progress"] = _progress_percent(response_data["progress"]) + progress_value: Final = response_data.get("progress") + if progress_value is not None: + video_data["progress"] = _progress_percent(progress_value) if "failureCode" in response_data or "failure" in response_data: video_data["error"] = { diff --git a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py index e52cf9211ec..afc8e7ec4a2 100644 --- a/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py +++ b/tests/test_litellm/llms/runwayml/videos/test_runway_video_transformation.py @@ -119,6 +119,25 @@ class TestRunwayMLVideoTransformation: assert result.status == "in_progress" assert result.progress == 3 + def test_status_progress_null_leaves_progress_unset(self): + """Runway sends an explicit null progress for pending polls; scaling it must not crash.""" + mock_response = Mock(spec=httpx.Response) + mock_response.json.return_value = { + "id": "63fd0f13-f29d-4e58-99d3-1cb9efa14a5b", + "createdAt": "2025-11-11T21:48:50.448Z", + "status": "PENDING", + "progress": None, + } + + result = self.config.transform_video_status_retrieve_response( + raw_response=mock_response, + logging_obj=self.mock_logging_obj, + custom_llm_provider="runwayml", + ) + + assert result.status == "queued" + assert result.progress is None + def test_get_error_class_returns_exception_instead_of_raising(self): error = self.config.get_error_class( error_message="Invalid API key", From 4eb09ad56e29c998647dc9e494959cfd31dfbebe Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:28:34 -0700 Subject: [PATCH 49/54] refactor: trim multipart form helper docstrings to the non-obvious rationale --- .../litellm_core_utils/llm_request_utils.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 5e822971e8f..0575af3d6f7 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -29,13 +29,11 @@ def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]: def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str], ...]: """ - Flatten JSON-shaped bodies into primitive ``(name, value)`` form fields the - way the OpenAI SDK serializes multipart bodies: dicts as ``key[subkey]``, - lists as ``key[]``, booleans lowercased, None and empty values dropped. - Sources are applied in order, so a later source wins on a key collision when - fed to ``dict.update``. Used to funnel provider-specific params into a - multipart request without handing the httpx encoder a nested value it - rejects with ``Invalid type for value``. + Flatten JSON-shaped bodies into primitive ``(name, value)`` form fields the way the + OpenAI SDK serializes multipart bodies, applying ``sources`` in order so a later source + wins on a key collision under ``dict.update``. Lets provider params reach a multipart + request without handing the httpx encoder a nested value it rejects with + ``Invalid type for value``. """ return tuple( pair @@ -48,11 +46,9 @@ def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tu def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]: """ - Encode a JSON-shaped body as httpx file-tuples so a request with no file - parts is still sent as multipart/form-data (httpx downgrades a file-less - ``data=`` payload to application/x-www-form-urlencoded). Nested values are - flattened the way the OpenAI SDK serializes multipart bodies: dicts as - ``key[subkey]``, lists as ``key[]``, booleans lowercased, None dropped. + Encode a JSON-shaped body as OpenAI-SDK-style multipart file-tuples so a file-less + request is still sent as multipart/form-data, working around httpx downgrading a + file-less ``data=`` payload to application/x-www-form-urlencoded. """ return tuple( (key, (None, serialized)) From b1035368f8983372d05e8158db895a345cc39883 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:35:52 -0700 Subject: [PATCH 50/54] fix(passthrough): thread router-model attribution on the litellm_metadata bucket The router hop _ageneric_api_call_with_fallbacks canonicalises the passthrough call type onto litellm_metadata, and the cost callback reads spend attribution from that bucket while only backfilling user_api_key* keys from metadata. The helper was building on metadata, so agent_id and user_api_end_user_max_budget were silently dropped before the callback ever saw them. Build and pass the attribution under litellm_metadata so every field survives. --- .../llm_passthrough_endpoints.py | 18 +++++++++++++----- .../test_llm_pass_through_endpoints.py | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index fc891978ad5..55d6b88363f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -116,16 +116,24 @@ def get_passthrough_router_request_metadata(user_api_key_dict: UserAPIKeyAuth) - callback cannot attribute spend to the calling key and never releases the budget reservation minted at auth time, so the shared spend counter drifts up until the key falsely trips ``BudgetExceededError``. + + The payload rides the ``litellm_metadata`` bucket, not ``metadata``: the + router hop ``_ageneric_api_call_with_fallbacks`` canonicalises this call + type into ``litellm_metadata``, and the cost callback reads spend + attribution from that bucket while only backfilling ``user_api_key*`` keys + from ``metadata``. Passing ``metadata=`` would silently drop the secondary + attribution fields the helper sets (``agent_id``, + ``user_api_end_user_max_budget``) before the callback ever sees them. """ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - request_data: Final = {"metadata": {}} # mutable-ok: attribution builder + litellm mutate this dict in place + request_data: Final = {"litellm_metadata": {}} # mutable-ok: builder + litellm mutate this in place LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=request_data, user_api_key_dict=user_api_key_dict, - _metadata_variable_name="metadata", + _metadata_variable_name="litellm_metadata", ) - return request_data["metadata"] + return request_data["litellm_metadata"] async def llm_passthrough_factory_proxy_route( @@ -368,7 +376,7 @@ async def vllm_proxy_route( params=None, headers=None, cookies=None, - metadata=get_passthrough_router_request_metadata(user_api_key_dict), + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), ), ) @@ -1498,7 +1506,7 @@ async def azure_proxy_route( params=None, headers=None, cookies=None, - metadata=get_passthrough_router_request_metadata(user_api_key_dict), + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), ) if is_streaming_request: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 032cb360d4c..f789ba10490 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -4076,6 +4076,8 @@ class TestPassthroughRouterModelBudgetReservation: user_id="u1", team_id="t1", budget_reservation=reservation, + agent_id="agent-xyz", + end_user_max_budget=42.0, ) def _request(self) -> MagicMock: @@ -4106,11 +4108,17 @@ class TestPassthroughRouterModelBudgetReservation: def _assert_metadata_carries_attribution(self, captured: list[dict], user_api_key_dict: UserAPIKeyAuth) -> None: assert len(captured) == 1, "the router-model branch must dispatch exactly once" - metadata = captured[0]["metadata"] - assert metadata["user_api_key"] == user_api_key_dict.api_key - assert metadata["user_api_key_budget_reservation"] is user_api_key_dict.budget_reservation - assert metadata["user_api_key_user_id"] == user_api_key_dict.user_id - assert metadata["user_api_key_team_id"] == user_api_key_dict.team_id + assert captured[0].get("metadata") is None, ( + "attribution must ride the litellm_metadata bucket the router canonicalizes on; " + "the plain metadata bucket is dropped for every non-user_api_key field" + ) + litellm_metadata = captured[0]["litellm_metadata"] + assert litellm_metadata["user_api_key"] == user_api_key_dict.api_key + assert litellm_metadata["user_api_key_budget_reservation"] is user_api_key_dict.budget_reservation + assert litellm_metadata["user_api_key_user_id"] == user_api_key_dict.user_id + assert litellm_metadata["user_api_key_team_id"] == user_api_key_dict.team_id + assert litellm_metadata["agent_id"] == user_api_key_dict.agent_id + assert litellm_metadata["user_api_end_user_max_budget"] == user_api_key_dict.end_user_max_budget @pytest.mark.asyncio async def test_vllm_router_model_threads_key_metadata(self, monkeypatch): From a703378915ccedeb49324c0c63fbf6d384cc17c6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:48:44 -0700 Subject: [PATCH 51/54] fix(images): forward scalar-array edit params as repeated multipart fields Flatten dict-backed multipart bodies so a scalar list becomes one field with a tuple value, which httpx emits as a repeated part per element, instead of collapsing to the last element under dict.update. Nested objects still flatten to key[subkey] like the OpenAI SDK, and the file-tuple video path is untouched. --- .../litellm_core_utils/llm_request_utils.py | 40 +++++++++++++++---- .../test_llm_request_utils.py | 35 ++++++++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 0575af3d6f7..c833d57b6a9 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -27,20 +27,46 @@ def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]: return ((key, serialized),) -def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str], ...]: +def _is_form_scalar(value: object) -> bool: + return value is not None and not isinstance(value, (Mapping, list, tuple)) + + +def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]: + if isinstance(value, Mapping): + return tuple( + item + for subkey, subvalue in value.items() + for item in _flatten_form_data_field(f"{key}[{subkey}]", subvalue) + ) + if isinstance(value, (list, tuple)): + if all(_is_form_scalar(entry) for entry in value): + serialized_fields: Final = tuple(field for entry in value if (field := _form_field_value(entry))) + return ((key, serialized_fields),) if serialized_fields else () + return tuple(item for entry in value for item in _flatten_form_data_field(f"{key}[]", entry)) + if value is None: + return () + serialized: Final = _form_field_value(value) + if not serialized: + return () + return ((key, serialized),) + + +def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]: """ - Flatten JSON-shaped bodies into primitive ``(name, value)`` form fields the way the - OpenAI SDK serializes multipart bodies, applying ``sources`` in order so a later source - wins on a key collision under ``dict.update``. Lets provider params reach a multipart - request without handing the httpx encoder a nested value it rejects with - ``Invalid type for value``. + Flatten JSON-shaped bodies into ``(name, value)`` form fields for a ``dict``-backed + multipart body, applying ``sources`` in order so a later source wins on a key collision + under ``dict.update``. Nested objects become ``key[subkey]`` fields the way the OpenAI SDK + serializes them, so provider params reach a multipart request without handing the httpx + encoder a nested value it rejects with ``Invalid type for value``. A scalar list becomes a + single field carrying a tuple value, which httpx emits as one repeated part per element, so + every element survives instead of collapsing to the last under ``dict.update``. """ return tuple( pair for source in sources if source is not None for top_key, top_value in source.items() - for pair in _flatten_form_field(top_key, top_value) + for pair in _flatten_form_data_field(top_key, top_value) ) diff --git a/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py b/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py index 0140d4ff232..3a09702de45 100644 --- a/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_llm_request_utils.py @@ -1,9 +1,24 @@ +import httpx + from litellm.litellm_core_utils.llm_request_utils import ( flatten_form_field_values, serialize_multipart_form_fields, ) +def _multipart_field_names(data: dict) -> list[str]: + request = httpx.Request( + "POST", + "http://backend/v1/images/edits", + data=data, + files=[("image[]", ("in.png", b"stub", "image/png"))], + ) + request.read() + body = request.content.decode("utf-8", "replace") + prefix = 'Content-Disposition: form-data; name="' + return [line[len(prefix) : line.index('"', len(prefix))] for line in body.splitlines() if line.startswith(prefix)] + + def test_serialize_multipart_form_fields_flattens_like_the_openai_sdk(): fields = serialize_multipart_form_fields( { @@ -62,3 +77,23 @@ def test_flatten_form_field_values_later_source_wins_on_collision(): ("seed", "2"), ) assert dict(flatten_form_field_values({"seed": 1}, {"seed": 2}))["seed"] == "2" + + +def test_flatten_form_field_values_keeps_scalar_lists_as_repeated_fields(): + assert flatten_form_field_values( + {"loras": ["a", "b", "c"], "generation_config": {"tags": [1, 2]}, "seed": 42} + ) == ( + ("loras", ("a", "b", "c")), + ("generation_config[tags]", ("1", "2")), + ("seed", "42"), + ) + + +def test_flatten_form_field_values_scalar_list_survives_update_into_multipart(): + request_params: dict = {"model": "my-edit-model"} + request_params.update(flatten_form_field_values({"loras": ["style_a", "style_b"]})) + + names = _multipart_field_names(request_params) + + assert names.count("loras") == 2 + assert names.count("model") == 1 From 3ffec658fe48e49a16de193d03380bdc56f9f14a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:49:54 -0700 Subject: [PATCH 52/54] test(images): pin scalar-array edit params survive as repeated multipart fields --- .../images/test_image_edit_extra_params.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_litellm/images/test_image_edit_extra_params.py b/tests/test_litellm/images/test_image_edit_extra_params.py index 01490cdd988..088faafa9f3 100644 --- a/tests/test_litellm/images/test_image_edit_extra_params.py +++ b/tests/test_litellm/images/test_image_edit_extra_params.py @@ -100,6 +100,27 @@ def test_image_edit_flattens_nested_provider_params(): assert "generation_config" not in fields +def test_image_edit_forwards_scalar_array_as_repeated_fields(): + """A list-valued provider param must reach the backend as one repeated part + per element, not collapse to its last element under dict.update.""" + captured = {} + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_image_edit_request(captured)))) + + litellm.image_edit( + model="openai/gpt-image-1", + image=PNG_BYTES, + prompt="add a hat", + api_key="sk-test", + api_base="https://edit.example/v1", + client=client, + loras=["style_a", "style_b", "style_c"], + ) + + body = captured["body"] + assert body.count(b'name="loras"') == 3 + assert b"style_a" in body and b"style_b" in body and b"style_c" in body + + @pytest.mark.asyncio async def test_aimage_edit_forwards_extra_body(): """aimage_edit used to drop extra_headers/extra_query/extra_body when From f1f6d83d47ec5fc6e2d10048fce036c1f41b1a2d Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 24 Aug 2026 16:26:21 -0400 Subject: [PATCH 53/54] fix(ci): give three unit shards a job deadline that outlasts their pytest budget (#38139) The caching-local, proxy-extras and enterprise-package shards each budget pytest 20m but cap the whole job at 55m. Setup can consume up to 35m, and the runner adds 5m of overhead, so the job deadline can preempt pytest inside its own advertised budget and the shard dies without a test report. check_workflow_startup_safety enforces that invariant and is currently failing on litellm_internal_staging, which reds the code-quality job for every open PR. Raising the three caps to 60m satisfies 20 + 35 + 5. --- .github/workflows/test-unit.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a7c67f2b35d..2dfca3d308f 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -211,7 +211,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: proxy-extras artifact-name: proxy-extras @@ -219,7 +219,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: enterprise-package artifact-name: enterprise-package @@ -227,7 +227,7 @@ jobs: workers: 4 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: responses-caching-types artifact-name: responses-caching-types From 85d5ac2b5ca25c3214fdd50d2455830a40a5a90b Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 24 Aug 2026 16:28:45 -0400 Subject: [PATCH 54/54] feat(ui): add Gemini Family auto-router preset (#38138) Adds the `gemini_family` bundled template to the auto-router tab, a heuristic-classifier preset alongside the existing Anthropic and OpenAI family presets. Tiers ascend in cost across the Gemini lineup: SIMPLE gemini-2.5-flash-lite $0.10 / $0.40 MEDIUM gemini-3.1-flash-lite $0.25 / $1.50 COMPLEX gemini-3.7-flash $0.75 / $3.75 REASONING gemini-3.1-pro-preview $2.00 / $12.00 Uses concrete model ids rather than Google's `gemini-*-latest` aliases. Those aliases hot-swap to the newest release of their variation (stable, preview or experimental) with only a two-week notice, while their rows in model_prices_and_context_window.json are pinned at 2.5-generation rates, so a swap onto a 3.x model would bill at the stale price and silently undercount auto-router spend. A pin test asserts no tier resolves to a `-latest` alias and that all four rungs are distinct. --- .../src/autorouter_presets.json | 16 ++++++++++++++ .../add_model/add_auto_router_tab.test.tsx | 4 ++-- .../src/lib/autorouter_presets.test.ts | 21 +++++++++++++++++-- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/ui/litellm-dashboard/src/autorouter_presets.json index 9a38c6b1a17..aff6f09da04 100644 --- a/ui/litellm-dashboard/src/autorouter_presets.json +++ b/ui/litellm-dashboard/src/autorouter_presets.json @@ -15,6 +15,22 @@ "deployment_affinity": true } }, + "gemini_family": { + "label": "Gemini Family", + "description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["gemini-2.5-flash-lite"], + "MEDIUM": ["gemini-3.1-flash-lite"], + "COMPLEX": ["gemini-3.7-flash"], + "REASONING": ["gemini-3.1-pro-preview"] + }, + "classifier_type": "heuristic", + "escalation_keywords": ["LITELLM ESCALATE"], + "session_affinity": false, + "deployment_affinity": true + } + }, "lite": { "label": "Lite", "description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 for medium, Kimi K3 for complex, Claude Opus 5 for reasoning-heavy requests. An LLM classifier with the agentic rubric assigns tiers.", diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index e45422dee08..1219ac6138b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -385,7 +385,7 @@ describe("AddAutoRouterTab", () => { const labels = visibleOptions().map((option) => option.querySelector(".font-medium")?.textContent); - expect(labels).toEqual(["Anthropic Family", "Lite", "OpenAI Family", "Custom Configuration"]); + expect(labels).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family", "Custom Configuration"]); }); describe("routing test", () => { @@ -788,7 +788,7 @@ describe("AddAutoRouterTab", () => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); const labels = visibleOptions().map((option) => option.querySelector(".font-medium")?.textContent); - expect(labels).toEqual(["Anthropic Family", "Lite", "OpenAI Family", "Custom Configuration"]); + expect(labels).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family", "Custom Configuration"]); }); it.each([ diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 2ff89fffe30..01c61ba6130 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -20,7 +20,7 @@ const groupsOnly = (models: Iterable) => buildModelAvailability(models, describe("autorouter_presets", () => { it("loads exactly the bundled presets", () => { const presets = getAllPresets(); - expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "Lite", "OpenAI Family"]); + expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family"]); // Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here. for (const p of presets) { expect(p).toMatchObject({ key: expect.any(String), label: expect.any(String), description: expect.any(String) }); @@ -51,7 +51,7 @@ describe("autorouter_presets", () => { }); it("keeps the model-family presets on the heuristic classifier", () => { - for (const key of ["anthropic_family", "openai_family"]) { + for (const key of ["anthropic_family", "gemini_family", "openai_family"]) { expect(getPresetByKey(key)!.complexity_router_config.classifier_type).toBe("heuristic"); } }); @@ -75,6 +75,23 @@ describe("autorouter_presets", () => { ); }); + it("pins the gemini preset to concrete model ids, never Google's hot-swapping -latest aliases", () => { + const gemini = getPresetByKey("gemini_family")!; + const config = gemini.complexity_router_config; + expect(config.classifier_type).toBe("heuristic"); + expect(config.classifier_llm_config).toBeUndefined(); + const expectedTiers = { + SIMPLE: ["gemini-2.5-flash-lite"], + MEDIUM: ["gemini-3.1-flash-lite"], + COMPLEX: ["gemini-3.7-flash"], + REASONING: ["gemini-3.1-pro-preview"], + }; + expect(config.tiers).toEqual(expectedTiers); + const required = getRequiredModelsInPreset(gemini); + for (const model of required) expect(model).not.toMatch(/-latest$/); + expect(required.size).toBe(4); + }); + it("collects every tier model as a required model", () => { const preset = getPresetByKey("anthropic_family")!; const required = getRequiredModelsInPreset(preset);