From cab48123d157258e8674e0aea77e563edd3ca57c Mon Sep 17 00:00:00 2001 From: pandalaohe Date: Thu, 15 Oct 2026 18:17:49 +0800 Subject: [PATCH 1/3] fix(images): default null token counts in image usage on generation and edit OpenAI-compatible image relays can return usage with null token counts: null input_tokens / output_tokens / total_tokens, or null image_tokens / text_tokens inside input_tokens_details. ImageUsageInputTokensDetails types those as int, so the generation path (which only defaulted a missing input_tokens_details) and the edit path (which had no guard) both failed pydantic validation and the proxy returned 500 after the upstream billed. Move the guard into normalize_image_response_usage() and use it on both paths. Only those count fields are defaulted; output_tokens_details and every other usage key pass through unchanged, and the edit path keeps constructing ImageResponse through its constructor. Co-Authored-By: Claude Opus 5.5 --- .../convert_dict_to_response.py | 46 +++++--- .../llms/openai/image_edit/transformation.py | 5 +- .../test_convert_dict_to_response.py | 102 +++++++++++++++++- .../test_openai_image_edit_transformation.py | 102 +++++++++++++++++- 4 files changed, 238 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 63bbd6dc904..ba6c41eef45 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -484,6 +484,32 @@ def handle_invalid_parallel_tool_calls( _handle_invalid_parallel_tool_calls = handle_invalid_parallel_tool_calls +def normalize_image_response_usage(response: Mapping[str, object]) -> Mapping[str, object]: + usage: Final = cast(Mapping[str, object] | None, response.get("usage")) + if usage is None: + return {**response} + + input_tokens: Final = cast(int, usage["input_tokens"] if usage.get("input_tokens") is not None else 0) + output_tokens: Final = cast(int, usage["output_tokens"] if usage.get("output_tokens") is not None else 0) + input_details: Final = cast(Mapping[str, object], usage.get("input_tokens_details") or {}) + return { + **response, + "usage": { + **usage, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": usage["total_tokens"] + if usage.get("total_tokens") is not None + else input_tokens + output_tokens, + "input_tokens_details": { + **input_details, + "image_tokens": input_details["image_tokens"] if input_details.get("image_tokens") is not None else 0, + "text_tokens": input_details["text_tokens"] if input_details.get("text_tokens") is not None else 0, + }, + }, + } + + class LiteLLMResponseObjectHandler: @staticmethod def convert_to_image_response( @@ -491,23 +517,15 @@ class LiteLLMResponseObjectHandler: model_response_object: ImageResponse | None = None, hidden_params: dict | None = None, ) -> ImageResponse: - response_object.update({"hidden_params": hidden_params}) + response_object.update( + { + **normalize_image_response_usage(cast(Mapping[str, object], response_object)), + "hidden_params": hidden_params, + } + ) - # Handle gpt-image-1 usage field with None values if "usage" in response_object and response_object["usage"] is not None: usage: Final = response_object["usage"] - # Check if usage fields are None and provide defaults - if usage.get("input_tokens") is None: - usage["input_tokens"] = 0 - if usage.get("output_tokens") is None: - usage["output_tokens"] = 0 - if usage.get("total_tokens") is None: - usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"] - if usage.get("input_tokens_details") is None: - usage["input_tokens_details"] = { - "image_tokens": 0, - "text_tokens": 0, - } # Map Responses API naming to Chat Completions API naming for cost calculator if usage.get("prompt_tokens") is None: diff --git a/litellm/llms/openai/image_edit/transformation.py b/litellm/llms/openai/image_edit/transformation.py index d54522597a0..36917532045 100644 --- a/litellm/llms/openai/image_edit/transformation.py +++ b/litellm/llms/openai/image_edit/transformation.py @@ -6,6 +6,9 @@ from httpx._types import RequestFiles import litellm from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + normalize_image_response_usage, +) from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ( @@ -150,7 +153,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig): raw_response_json: Final = raw_response.json() except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) - return ImageResponse(**raw_response_json) + return ImageResponse(**cast(dict[str, Any], normalize_image_response_usage(raw_response_json))) def validate_environment( self, diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index f918d4cb79e..0f4bc086a94 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,6 +1,7 @@ import asyncio import importlib import json +from copy import deepcopy from datetime import datetime, timedelta from typing import Final @@ -8,7 +9,15 @@ import pytest import litellm from litellm.constants import RESPONSE_FORMAT_TOOL_NAME -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler, convert_to_model_response_object, convert_to_streaming_response, handle_invalid_parallel_tool_calls, safe_convert_created_field, should_convert_tool_call_to_json_mode +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + LiteLLMResponseObjectHandler, + convert_to_model_response_object, + convert_to_streaming_response, + handle_invalid_parallel_tool_calls, + normalize_image_response_usage, + safe_convert_created_field, + should_convert_tool_call_to_json_mode, +) from litellm.types.utils import ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, CompletionTokensDetailsWrapper, Function, ImageObject, ImageResponse, Message, ModelResponse, PromptTokensDetailsWrapper, Usage from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -373,6 +382,97 @@ def test_convert_to_image_response_with_extra_fields_2(): assert result.data[0].url == "http://example.com/image1.jpg" assert result.data[1].url == "http://example.com/image2.jpg" + +@pytest.mark.parametrize("response", [{}, {"usage": None}]) +def test_normalize_image_response_without_usage_returns_copy(response: dict[str, object]) -> None: + normalized: Final = normalize_image_response_usage(response) + + assert normalized == response + assert normalized is not response + + +@pytest.mark.parametrize( + ("usage_fields", "expected_fields"), + [ + pytest.param( + {"input_tokens_details": {"image_tokens": None, "text_tokens": 5}}, + {"input_tokens_details": {"image_tokens": 0, "text_tokens": 5}}, + id="null-input-image-tokens", + ), + pytest.param( + {"input_tokens_details": {"image_tokens": 6}}, + {"input_tokens_details": {"image_tokens": 6, "text_tokens": 0}}, + id="missing-input-text-tokens", + ), + pytest.param( + {"input_tokens_details": {"text_tokens": 5}}, + {"input_tokens_details": {"image_tokens": 0, "text_tokens": 5}}, + id="missing-input-image-tokens", + ), + pytest.param( + {"output_tokens_details": None}, + {"output_tokens_details": None}, + id="null-output-details", + ), + pytest.param( + {"input_tokens_details": {"image_tokens": 6, "text_tokens": 4, "cached_tokens_details": None}}, + {"input_tokens_details": {"image_tokens": 6, "text_tokens": 4, "cached_tokens_details": None}}, + id="null-cached-details", + ), + pytest.param( + {"output_tokens_details": {"image_tokens": 20}}, + {"output_tokens_details": {"image_tokens": 20}}, + id="sparse-output-details", + ), + pytest.param( + {"output_tokens_details": {"image_tokens": None}}, + {"output_tokens_details": {"image_tokens": None}}, + id="null-output-image-tokens", + ), + pytest.param( + {"output_tokens_details": {"image_tokens": 18, "text_tokens": 2}}, + {"output_tokens_details": {"image_tokens": 18, "text_tokens": 2}}, + id="fully-populated-usage", + ), + pytest.param( + {"provider_metadata": None}, + {"provider_metadata": None}, + id="null-provider-metadata", + ), + pytest.param({}, {}, id="absent-output-details"), + ], +) +def test_convert_to_image_response_preserves_usage_details( + usage_fields: dict[str, object], expected_fields: dict[str, object] +) -> None: + usage: Final = { + "input_tokens": 10, + "input_tokens_details": {"image_tokens": 6, "text_tokens": 4}, + "output_tokens": 20, + "total_tokens": 30, + } + response: Final = { + "created": 1234567890, + "data": [{"b64_json": "generated-image"}], + "usage": {**usage, **usage_fields}, + } + original: Final = deepcopy(response) + expected_usage: Final = {**usage, **expected_fields} + + normalized: Final = normalize_image_response_usage(response) + + assert normalized == {**response, "usage": expected_usage} + assert normalized is not response + assert response == original + + result: Final = LiteLLMResponseObjectHandler.convert_to_image_response(response) + + assert result.usage is not None + serialized_usage: Final = result.usage.model_dump() + assert {key: serialized_usage[key] for key in expected_usage} == expected_usage + assert result.data[0].b64_json == "generated-image" + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_none_usage_fields(): """ diff --git a/tests/unit/llms/openai/test_openai_image_edit_transformation.py b/tests/unit/llms/openai/test_openai_image_edit_transformation.py index da3352a4943..b5b03cf825f 100644 --- a/tests/unit/llms/openai/test_openai_image_edit_transformation.py +++ b/tests/unit/llms/openai/test_openai_image_edit_transformation.py @@ -1,12 +1,15 @@ from io import BufferedReader, BytesIO -from typing import Dict +from typing import Dict, Final +from unittest.mock import Mock, patch +import httpx import pytest from litellm import image_edit from litellm.images.utils import ImageEditRequestUtils from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageObject @pytest.fixture @@ -14,6 +17,103 @@ def image_edit_config() -> OpenAIImageEditConfig: return OpenAIImageEditConfig() +@pytest.mark.parametrize( + ("usage_fields", "expected_fields"), + [ + pytest.param( + {"input_tokens": None, "output_tokens": None, "total_tokens": None}, + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + id="null-counts", + ), + pytest.param( + {"input_tokens_details": None}, + {"input_tokens_details": {"image_tokens": 0, "text_tokens": 0}}, + id="null-input-details", + ), + pytest.param( + {"input_tokens_details": {"image_tokens": None, "text_tokens": None}}, + {"input_tokens_details": {"image_tokens": 0, "text_tokens": 0}}, + id="null-input-detail-counts", + ), + pytest.param( + {"input_tokens_details": {"image_tokens": None, "text_tokens": 5}}, + {"input_tokens_details": {"image_tokens": 0, "text_tokens": 5}}, + id="mixed-input-detail-counts", + ), + pytest.param( + {"output_tokens_details": {"image_tokens": None, "text_tokens": None}}, + {"output_tokens_details": {"image_tokens": None, "text_tokens": None}}, + id="null-output-detail-counts", + ), + pytest.param( + {"output_tokens_details": {"image_tokens": 18, "text_tokens": None}}, + {"output_tokens_details": {"image_tokens": 18, "text_tokens": None}}, + id="mixed-output-detail-counts", + ), + pytest.param( + {"output_tokens_details": None}, + {"output_tokens_details": None}, + id="null-output-details", + ), + pytest.param( + {"input_tokens_details": {"image_tokens": 6, "text_tokens": 4, "cached_tokens_details": None}}, + {"input_tokens_details": {"image_tokens": 6, "text_tokens": 4, "cached_tokens_details": None}}, + id="null-cached-details", + ), + pytest.param( + {"output_tokens_details": {"image_tokens": 20}}, + {"output_tokens_details": {"image_tokens": 20}}, + id="sparse-output-details", + ), + pytest.param({"total_tokens": None}, {"total_tokens": 30}, id="calculated-total"), + pytest.param({}, {}, id="integer-usage-unchanged"), + ], +) +def test_transform_image_edit_response_normalizes_null_usage( + image_edit_config: OpenAIImageEditConfig, + usage_fields: dict[str, object], + expected_fields: dict[str, object], +) -> None: + usage: Final = { + "input_tokens": 10, + "output_tokens": 20, + "total_tokens": 30, + "input_tokens_details": {"image_tokens": 6, "text_tokens": 4}, + "output_tokens_details": {"image_tokens": 18, "text_tokens": 2}, + } + raw_response: Final = httpx.Response( + status_code=200, + json={"created": 1234567890, "data": [{"b64_json": "edited-image"}], "usage": {**usage, **usage_fields}}, + ) + + result: Final = image_edit_config.transform_image_edit_response( + model="gpt-image-1", raw_response=raw_response, logging_obj=Mock() + ) + + assert result.usage is not None + assert result.usage.model_dump() == {**usage, **expected_fields} + assert result.data[0].b64_json == "edited-image" + + +def test_transform_image_edit_response_defaults_created_and_converts_image_objects( + image_edit_config: OpenAIImageEditConfig, +) -> None: + raw_response: Final = httpx.Response( + status_code=200, + json={"data": [{"b64_json": "edited-image"}]}, + ) + + with patch("time.time", return_value=1234567890): + result: Final = image_edit_config.transform_image_edit_response( + model="gpt-image-1", raw_response=raw_response, logging_obj=Mock() + ) + + assert result.created == 1234567890 + assert len(result.data) == 1 + assert isinstance(result.data[0], ImageObject) + assert result.data[0].b64_json == "edited-image" + + def test_transform_image_edit_request_basic(image_edit_config: OpenAIImageEditConfig): """Test basic image edit request transformation with image and prompt""" model = "dall-e-2" From 55ae32934e3bb77220e36a0127cc3e46f1a50347 Mon Sep 17 00:00:00 2001 From: pandalaohe Date: Thu, 15 Oct 2026 19:24:42 +0800 Subject: [PATCH 2/3] fix(images): validate image usage without casts Validate image usage mappings before normalization and calculate null totals only for integer counts Co-Authored-By: Claude Opus 5.5 --- .../convert_dict_to_response.py | 23 ++- .../llms/openai/image_edit/transformation.py | 4 +- .../test_convert_dict_to_response.py | 195 ++++++++++-------- 3 files changed, 129 insertions(+), 93 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index ba6c41eef45..127a7508188 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -485,22 +485,27 @@ _handle_invalid_parallel_tool_calls = handle_invalid_parallel_tool_calls def normalize_image_response_usage(response: Mapping[str, object]) -> Mapping[str, object]: - usage: Final = cast(Mapping[str, object] | None, response.get("usage")) - if usage is None: + raw_usage: Final = response.get("usage") + if not isinstance(raw_usage, Mapping): return {**response} - input_tokens: Final = cast(int, usage["input_tokens"] if usage.get("input_tokens") is not None else 0) - output_tokens: Final = cast(int, usage["output_tokens"] if usage.get("output_tokens") is not None else 0) - input_details: Final = cast(Mapping[str, object], usage.get("input_tokens_details") or {}) + usage: Final = _FIELD_MAP.validate_python(raw_usage) + input_tokens: Final = usage["input_tokens"] if usage.get("input_tokens") is not None else 0 + output_tokens: Final = usage["output_tokens"] if usage.get("output_tokens") is not None else 0 + total_tokens: Final = ( + input_tokens + output_tokens if isinstance(input_tokens, int) and isinstance(output_tokens, int) else None + ) + raw_input_details: Final = usage.get("input_tokens_details") + input_details: Final = ( + _FIELD_MAP.validate_python(raw_input_details) if isinstance(raw_input_details, Mapping) else {} + ) return { **response, "usage": { **usage, "input_tokens": input_tokens, "output_tokens": output_tokens, - "total_tokens": usage["total_tokens"] - if usage.get("total_tokens") is not None - else input_tokens + output_tokens, + "total_tokens": usage["total_tokens"] if usage.get("total_tokens") is not None else total_tokens, "input_tokens_details": { **input_details, "image_tokens": input_details["image_tokens"] if input_details.get("image_tokens") is not None else 0, @@ -519,7 +524,7 @@ class LiteLLMResponseObjectHandler: ) -> ImageResponse: response_object.update( { - **normalize_image_response_usage(cast(Mapping[str, object], response_object)), + **normalize_image_response_usage(_FIELD_MAP.validate_python(response_object)), "hidden_params": hidden_params, } ) diff --git a/litellm/llms/openai/image_edit/transformation.py b/litellm/llms/openai/image_edit/transformation.py index 36917532045..919aadd78c4 100644 --- a/litellm/llms/openai/image_edit/transformation.py +++ b/litellm/llms/openai/image_edit/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from io import BufferedReader from typing import TYPE_CHECKING, Any, Final, cast @@ -153,7 +154,8 @@ class OpenAIImageEditConfig(BaseImageEditConfig): raw_response_json: Final = raw_response.json() except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) - return ImageResponse(**cast(dict[str, Any], normalize_image_response_usage(raw_response_json))) + normalized_response: Final[Mapping[str, Any]] = normalize_image_response_usage(raw_response_json) + return ImageResponse(**normalized_response) def validate_environment( self, diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 0f4bc086a94..6b994fa6df0 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,6 +1,7 @@ import asyncio import importlib import json +from collections.abc import Mapping from copy import deepcopy from datetime import datetime, timedelta from typing import Final @@ -18,7 +19,19 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo safe_convert_created_field, should_convert_tool_call_to_json_mode, ) -from litellm.types.utils import ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, CompletionTokensDetailsWrapper, Function, ImageObject, ImageResponse, Message, ModelResponse, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Choices, + CompletionTokensDetailsWrapper, + Function, + ImageObject, + ImageResponse, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome FIXED_TIME: Final[datetime] = datetime(2025, 1, 1, 12, 0, 0) @@ -41,7 +54,7 @@ OPENAI_CUSTOM_TOOL_CALL_RESPONSE = { "type": "custom", "custom": { "name": "ApplyPatch", - "input": "*** Begin Patch\n*** Update File: main.py\n@@\n+def hello():\n+ print(\"Hello\")\n*** End Patch\n", + "input": '*** Begin Patch\n*** Update File: main.py\n@@\n+def hello():\n+ print("Hello")\n*** End Patch\n', }, } ], @@ -225,6 +238,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -234,6 +248,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -267,6 +282,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -281,6 +297,7 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_basic(): # Test basic conversion with minimal input @@ -295,6 +312,7 @@ def test_convert_to_image_response_basic(): assert result.created == 1234567890 assert result.data[0].url == "http://example.com/image.jpg" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_hidden_params(): # Test with hidden params @@ -308,6 +326,7 @@ def test_convert_to_image_response_with_hidden_params(): assert result._hidden_params == {"api_key": "test_key"} + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_multiple_images(): # Test handling multiple images in response @@ -325,6 +344,7 @@ def test_convert_to_image_response_multiple_images(): assert result.data[0].url == "http://example.com/image1.jpg" assert result.data[1].url == "http://example.com/image2.jpg" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_b64_json(): # Test handling b64_json in response @@ -337,6 +357,7 @@ def test_convert_to_image_response_with_b64_json(): assert result.data[0].b64_json == "base64encodedstring" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_extra_fields(): response_dict = { @@ -358,6 +379,7 @@ def test_convert_to_image_response_with_extra_fields(): assert result.data[0].url == "http://example.com/image1.jpg" assert result.data[1].url == "http://example.com/image2.jpg" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_extra_fields_2(): """ @@ -391,6 +413,31 @@ def test_normalize_image_response_without_usage_returns_copy(response: dict[str, assert normalized is not response +@pytest.mark.parametrize( + "counts", + [ + {"input_tokens": "10", "output_tokens": 20}, + {"input_tokens": 10, "output_tokens": "20"}, + {"input_tokens": 1.5, "output_tokens": 20}, + {"input_tokens": 10, "output_tokens": []}, + ], +) +def test_normalize_image_response_preserves_non_integer_counts(counts: Mapping[str, object]) -> None: + response: Final = {"usage": {**counts, "total_tokens": None}} + original: Final = deepcopy(response) + + normalized: Final = normalize_image_response_usage(response) + + assert normalized == { + "usage": { + **counts, + "total_tokens": None, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + } + } + assert response == original + + @pytest.mark.parametrize( ("usage_fields", "expected_fields"), [ @@ -510,6 +557,7 @@ def test_convert_to_image_response_with_none_usage_fields(): assert result.usage.input_tokens_details.image_tokens == 0 assert result.usage.input_tokens_details.text_tokens == 0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_partial_none_usage_fields(): """ @@ -543,6 +591,7 @@ def test_convert_to_image_response_with_partial_none_usage_fields(): assert result.usage.input_tokens_details.image_tokens == 0 assert result.usage.input_tokens_details.text_tokens == 0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_valid_usage_fields(): """ @@ -577,6 +626,7 @@ def test_convert_to_image_response_with_valid_usage_fields(): assert result.usage.input_tokens_details.image_tokens == 30 assert result.usage.input_tokens_details.text_tokens == 20 + def test_convert_to_model_response_object_basic(): """Test basic conversion with all fields present.""" response_object = { @@ -639,12 +689,8 @@ def test_convert_to_model_response_object_basic(): assert result.usage.prompt_tokens == 19 assert result.usage.completion_tokens == 10 assert result.usage.total_tokens == 29 - assert result.usage.prompt_tokens_details == PromptTokensDetailsWrapper( - cached_tokens=0 - ) - assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper( - reasoning_tokens=0 - ) + assert result.usage.prompt_tokens_details == PromptTokensDetailsWrapper(cached_tokens=0) + assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0) # Other fields assert result.system_fingerprint == "fp_6b68a8204b" @@ -652,6 +698,7 @@ def test_convert_to_model_response_object_basic(): # hidden params assert result._hidden_params is not None + def test_convert_image_input_dict_response_to_chat_completion_response(): """Test conversion on a response with an image input.""" response_object = { @@ -702,21 +749,17 @@ def test_convert_image_input_dict_response_to_chat_completion_response(): assert choice.index == 0 assert isinstance(choice.message, Message) assert choice.message.role == "assistant" - assert ( - choice.message.content - == "\n\nThis image shows a wooden boardwalk extending through a lush green marshland." - ) + assert choice.message.content == "\n\nThis image shows a wooden boardwalk extending through a lush green marshland." assert choice.finish_reason == "stop" assert result.usage.prompt_tokens == 9 assert result.usage.completion_tokens == 12 assert result.usage.total_tokens == 21 - assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper( - reasoning_tokens=0 - ) + assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0) assert result._hidden_params is not None + def test_convert_to_model_response_object_tool_calls_invalid_json_arguments(): """ Critical test - this is a basic response from OpenAI API @@ -779,13 +822,8 @@ def test_convert_to_model_response_object_tool_calls_invalid_json_arguments(): assert len(result.choices) == 1 assert result.choices[0].message.content is None assert len(result.choices[0].message.tool_calls) == 1 - assert ( - result.choices[0].message.tool_calls[0].function.name == "get_current_weather" - ) - assert ( - result.choices[0].message.tool_calls[0].function.arguments - == '{"location":"Boston, MA","unit":"fahren' - ) + assert result.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert result.choices[0].message.tool_calls[0].function.arguments == '{"location":"Boston, MA","unit":"fahren' assert result.choices[0].finish_reason == "length" assert result.model == "gpt-4o-2024-08-06" assert result.created == 1729337288 @@ -794,6 +832,7 @@ def test_convert_to_model_response_object_tool_calls_invalid_json_arguments(): assert result.usage.total_tokens == 102 assert result.system_fingerprint == "fp_45c6de4934" + def test_convert_to_model_response_object_tool_calls_valid_json_arguments(): """ Critical test - this is a basic response from OpenAI API @@ -856,13 +895,8 @@ def test_convert_to_model_response_object_tool_calls_valid_json_arguments(): assert len(result.choices) == 1 assert result.choices[0].message.content is None assert len(result.choices[0].message.tool_calls) == 1 - assert ( - result.choices[0].message.tool_calls[0].function.name == "get_current_weather" - ) - assert ( - result.choices[0].message.tool_calls[0].function.arguments - == '{"location":"Boston, MA","unit":"fahrenheit"}' - ) + assert result.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert result.choices[0].message.tool_calls[0].function.arguments == '{"location":"Boston, MA","unit":"fahrenheit"}' assert result.choices[0].finish_reason == "length" assert result.model == "gpt-4o-2024-08-06" assert result.created == 1729337288 @@ -871,6 +905,7 @@ def test_convert_to_model_response_object_tool_calls_valid_json_arguments(): assert result.usage.total_tokens == 102 assert result.system_fingerprint == "fp_45c6de4934" + def test_convert_to_model_response_object_json_mode(): """ This test is verifying that when convert_tool_call_to_json_mode is True, a single tool call's arguments are correctly converted into the message content of the response. @@ -921,6 +956,7 @@ def test_convert_to_model_response_object_json_mode(): assert result.usage.prompt_tokens == 5 assert result.usage.completion_tokens == 5 + def test_convert_to_model_response_object_with_logprobs(): """ @@ -1156,13 +1192,12 @@ def test_convert_to_model_response_object_with_logprobs(): assert result.usage.prompt_tokens == 9 assert result.usage.completion_tokens == 9 assert result.usage.total_tokens == 18 - assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper( - reasoning_tokens=0 - ) + assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0) assert result.system_fingerprint is None assert result._hidden_params is not None + def test_convert_to_model_response_object_error(): """Test error handling for None response object.""" with pytest.raises(Exception, match="Error in response object format"): @@ -1177,6 +1212,7 @@ def test_convert_to_model_response_object_error(): convert_tool_call_to_json_mode=False, ) + def test_image_generation_openai_with_pydantic_warning(caplog): try: import logging @@ -1214,6 +1250,7 @@ def test_image_generation_openai_with_pydantic_warning(caplog): except Exception as e: pytest.fail(f"Test failed with exception: {e}") + def test_convert_to_model_response_object_with_empty_str(): """Test that convert_to_model_response_object handles empty strings correctly.""" @@ -1317,6 +1354,7 @@ def test_convert_to_model_response_object_with_empty_str(): assert resp is not None assert resp.choices[0].message.content is not None + def test_convert_to_model_response_object_with_thinking_content(): """Test that convert_to_model_response_object handles thinking content correctly.""" @@ -1364,6 +1402,7 @@ def test_convert_to_model_response_object_with_thinking_content(): assert resp is not None assert resp.choices[0].message.reasoning_content is not None + def test_convert_to_model_response_object_with_empty_error_object(): """ Test that convert_to_model_response_object handles empty error objects gracefully. @@ -1421,9 +1460,8 @@ def test_convert_to_model_response_object_with_empty_error_object(): assert isinstance(result, ModelResponse) assert result.model == "minimax-m2.1" assert len(result.choices) == 1 - assert ( - result.choices[0].message.content == "Hey! I'm doing well, thanks for asking!" - ) + assert result.choices[0].message.content == "Hey! I'm doing well, thanks for asking!" + def test_convert_to_model_response_object_with_real_error(): """ @@ -1456,6 +1494,7 @@ def test_convert_to_model_response_object_with_real_error(): assert hasattr(exc_info.value, "message") assert "Rate limit exceeded" in str(exc_info.value.message) + def test_convert_to_model_response_object_with_empty_dict_error(): """ Test that convert_to_model_response_object handles completely empty error dict. @@ -1495,6 +1534,7 @@ def test_convert_to_model_response_object_with_empty_dict_error(): assert isinstance(result, ModelResponse) assert result.choices[0].message.content == "Hello!" + def test_convert_to_model_response_object_preserves_provider_specific_fields_from_proxy(): """ Test that provider_specific_fields (e.g. Anthropic citations) are preserved @@ -1572,18 +1612,13 @@ def test_convert_to_model_response_object_preserves_provider_specific_fields_fro assert result.id == "chatcmpl-proxy-123" choice = result.choices[0] - assert ( - choice.message.content - == "Based on current reviews, the Sony WH-1000XM5 remains one of the best headphones." - ) + assert choice.message.content == "Based on current reviews, the Sony WH-1000XM5 remains one of the best headphones." assert choice.message.provider_specific_fields is not None assert "citations" in choice.message.provider_specific_fields assert choice.message.provider_specific_fields["citations"] == citations assert "web_search_results" in choice.message.provider_specific_fields - assert ( - choice.message.provider_specific_fields["web_search_results"] - == web_search_results - ) + assert choice.message.provider_specific_fields["web_search_results"] == web_search_results + def test_convert_to_model_response_object_keeps_reasoning_items_from_a_cached_response(): reasoning_items: Final = [ @@ -1671,6 +1706,7 @@ def test_convert_to_model_response_object_provider_specific_fields_merges_extra_ assert "custom_extra_field" in psf assert psf["custom_extra_field"] == "extra_value" + def test_convert_to_model_response_object_no_provider_specific_fields_still_works(): """ Test that responses without provider_specific_fields continue to work as before. @@ -1716,6 +1752,7 @@ def test_convert_to_model_response_object_no_provider_specific_fields_still_work assert result.choices[0].message.refusal is None assert result.choices[0].message.provider_specific_fields is None + def test_convert_to_model_response_object_with_error_code_only(): """ Test that errors with only a code (no message) are still treated as real errors. @@ -1741,6 +1778,7 @@ def test_convert_to_model_response_object_with_error_code_only(): assert exc_info.value.status_code == 500 + def test_model_prefix_preservation(): """ Test that when model_response_object has a prefix like 'openai/gpt-4' @@ -1769,6 +1807,7 @@ def test_model_prefix_preservation(): assert result.model == "openai/gpt-4o" + def test_model_without_prefix(): """ Test that when model_response_object has no prefix (e.g. 'gpt-4'), @@ -1797,6 +1836,7 @@ def test_model_without_prefix(): assert result.model == "gpt-4" + def test_extra_response_fields_preserved(): """ Test that extra response fields (e.g. service_tier) are preserved @@ -1826,6 +1866,7 @@ def test_extra_response_fields_preserved(): assert result.service_tier == "default" + def test_hidden_params_and_response_headers_set(): """ Test that _hidden_params and _response_headers are correctly set @@ -1860,6 +1901,7 @@ def test_hidden_params_and_response_headers_set(): assert "additional_headers" in result._hidden_params assert result._response_headers == response_headers + def test_response_ms_computed(): """ Test that _response_ms is computed correctly from start_time and end_time. @@ -1889,6 +1931,7 @@ def test_response_ms_computed(): assert result._response_ms == pytest.approx(250.0) + def test_error_message_includes_function_args(): """ Test that when an exception occurs, the error message includes @@ -1901,7 +1944,7 @@ def test_error_message_includes_function_args(): "choices": [{"index": 0}], } - with pytest.raises(Exception, match='in convert_to_model_response_object') as exc_info: + with pytest.raises(Exception, match="in convert_to_model_response_object") as exc_info: convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, @@ -1915,6 +1958,7 @@ def test_error_message_includes_function_args(): assert "response_object" in error_msg assert "response_type" in error_msg + @pytest.mark.parametrize("falsy_id", [None, ""]) def test_convert_to_model_response_object_falsy_id_preserves_auto_generated(falsy_id): """Test that a falsy id in response_object preserves the auto-generated id.""" @@ -1942,6 +1986,7 @@ def test_convert_to_model_response_object_falsy_id_preserves_auto_generated(fals assert result.id == original_id assert result.id.startswith("chatcmpl-") + def test_convert_to_model_response_object_default_usage_overwritten(): """ Regression test: convert_to_model_response_object must properly set Usage @@ -1985,6 +2030,7 @@ def test_convert_to_model_response_object_default_usage_overwritten(): assert result.usage.completion_tokens == 7 assert result.usage.total_tokens == 22 + def test_convert_to_model_response_object_with_null_top_logprobs(): """ Test that convert_to_model_response_object handles null top_logprobs @@ -2076,6 +2122,7 @@ def test_convert_to_model_response_object_with_null_top_logprobs(): assert token_logprob.top_logprobs == [] assert isinstance(token_logprob.top_logprobs, list) + class TestMissingChoicesGuard: """ Tests for the defense-in-depth guard that raises APIError when a provider @@ -2201,9 +2248,7 @@ class TestMissingChoicesGuard: async def consume(): chunks = [] - async for chunk in convert_to_streaming_response_async( - response_object=response_object - ): + async for chunk in convert_to_streaming_response_async(response_object=response_object): chunks.append(chunk) return chunks @@ -2231,6 +2276,7 @@ class TestMissingChoicesGuard: assert "copilot_usage" in exc_info.value.message + class TestNormalizeImagesForMessage: def test_none_returns_none(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2266,6 +2312,7 @@ class TestNormalizeImagesForMessage: result = _normalize_images_for_message(images) assert result[0]["index"] == 5 + class TestSafeConvertCreatedField: def test_none_returns_current_time(self): import time @@ -2308,6 +2355,7 @@ class TestSafeConvertCreatedField: result = safe_convert_created_field("not-a-number") assert abs(result - int(time.time())) <= 1 + class TestConvertToStreamingResponse: def test_none_raises(self): @@ -2400,9 +2448,7 @@ class TestConvertToStreamingResponse: async def run(): chunks = [] - async for chunk in convert_to_streaming_response_async( - response_object=response_object - ): + async for chunk in convert_to_streaming_response_async(response_object=response_object): chunks.append(chunk) return chunks @@ -2411,6 +2457,7 @@ class TestConvertToStreamingResponse: assert chunks[0].choices[0].delta.tool_calls[0].id == "call_1" assert chunks[0].choices[0].delta.tool_calls[0].function.name == "get_weather" + class TestConvertToStreamingResponseAsync: def test_none_raises(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2448,9 +2495,7 @@ class TestConvertToStreamingResponseAsync: async def run(): chunks = [] - async for chunk in convert_to_streaming_response_async( - response_object=response_object - ): + async for chunk in convert_to_streaming_response_async(response_object=response_object): chunks.append(chunk) return chunks @@ -2465,6 +2510,7 @@ class TestConvertToStreamingResponseAsync: assert chunks[-1].choices[0].finish_reason == "stop" assert chunks[-1].usage.prompt_tokens == 3 + class TestHandleInvalidParallelToolCalls: def test_none_input(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2544,6 +2590,7 @@ class TestHandleInvalidParallelToolCalls: assert len(result) == 1 assert result[0].id == "call_1" + class TestShouldConvertToolCallToJsonMode: def test_returns_true_when_conditions_met(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2552,12 +2599,7 @@ class TestShouldConvertToolCallToJsonMode: from litellm.constants import RESPONSE_FORMAT_TOOL_NAME tool_calls = [{"function": {"name": RESPONSE_FORMAT_TOOL_NAME}}] - assert ( - should_convert_tool_call_to_json_mode( - tool_calls=tool_calls, convert_tool_call_to_json_mode=True - ) - is True - ) + assert should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) is True def test_returns_false_when_flag_off(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2567,10 +2609,7 @@ class TestShouldConvertToolCallToJsonMode: tool_calls = [{"function": {"name": RESPONSE_FORMAT_TOOL_NAME}}] assert ( - should_convert_tool_call_to_json_mode( - tool_calls=tool_calls, convert_tool_call_to_json_mode=False - ) - is False + should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=False) is False ) def test_returns_false_when_wrong_tool_name(self): @@ -2580,10 +2619,7 @@ class TestShouldConvertToolCallToJsonMode: tool_calls = [{"function": {"name": "some_other_tool"}}] assert ( - should_convert_tool_call_to_json_mode( - tool_calls=tool_calls, convert_tool_call_to_json_mode=True - ) - is False + should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) is False ) def test_returns_false_when_multiple_tool_calls(self): @@ -2597,10 +2633,7 @@ class TestShouldConvertToolCallToJsonMode: {"function": {"name": "other"}}, ] assert ( - should_convert_tool_call_to_json_mode( - tool_calls=tool_calls, convert_tool_call_to_json_mode=True - ) - is False + should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) is False ) def test_returns_false_when_none(self): @@ -2608,12 +2641,8 @@ class TestShouldConvertToolCallToJsonMode: should_convert_tool_call_to_json_mode, ) - assert ( - should_convert_tool_call_to_json_mode( - tool_calls=None, convert_tool_call_to_json_mode=True - ) - is False - ) + assert should_convert_tool_call_to_json_mode(tool_calls=None, convert_tool_call_to_json_mode=True) is False + class TestConvertToolCallToJsonMode: def test_converts_when_should(self): @@ -2633,9 +2662,7 @@ class TestConvertToolCallToJsonMode: ), ) ] - message, finish_reason = convert_fn( - tool_calls=tool_calls, convert_tool_call_to_json_mode=True - ) + message, finish_reason = convert_fn(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) assert message is not None assert message.content == '{"key": "value"}' assert finish_reason == "stop" @@ -2657,12 +2684,11 @@ class TestConvertToolCallToJsonMode: ), ) ] - message, finish_reason = convert_fn( - tool_calls=tool_calls, convert_tool_call_to_json_mode=False - ) + message, finish_reason = convert_fn(tool_calls=tool_calls, convert_tool_call_to_json_mode=False) assert message is None assert finish_reason is None + class TestConvertToModelResponseObjectEmbedding: def test_basic_embedding_response(self): from litellm.types.utils import EmbeddingResponse @@ -2688,6 +2714,7 @@ class TestConvertToModelResponseObjectEmbedding: assert result.data == [{"embedding": [0.1, 0.2, 0.3], "index": 0}] assert result.usage.prompt_tokens == 5 + class TestConvertToModelResponseObjectAudioTranscription: def test_basic_transcription(self): from litellm.types.utils import TranscriptionResponse @@ -2747,6 +2774,7 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 + class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse @@ -2765,6 +2793,7 @@ class TestConvertToModelResponseObjectRerank: assert result.id == "rerank-123" assert result.results[0]["relevance_score"] == 0.9 + class TestConvertToModelResponseObjectCompletion: def test_tool_calls_finish_reason_override(self): response_object = { From 607d3fff6f9ca9ea178118c9cc8eb68d81a46006 Mon Sep 17 00:00:00 2001 From: pandalaohe Date: Thu, 15 Oct 2026 19:30:07 +0800 Subject: [PATCH 3/3] test(images): keep the existing test file formatting Restore the formatting from cab48123d1 while preserving the new parametrized test and its Mapping import Co-Authored-By: Claude Opus 5.5 --- .../test_convert_dict_to_response.py | 169 +++++++++--------- 1 file changed, 83 insertions(+), 86 deletions(-) diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 6b994fa6df0..68e102b2d90 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -19,19 +19,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo safe_convert_created_field, should_convert_tool_call_to_json_mode, ) -from litellm.types.utils import ( - ChatCompletionMessageCustomToolCall, - ChatCompletionMessageToolCall, - Choices, - CompletionTokensDetailsWrapper, - Function, - ImageObject, - ImageResponse, - Message, - ModelResponse, - PromptTokensDetailsWrapper, - Usage, -) +from litellm.types.utils import ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, CompletionTokensDetailsWrapper, Function, ImageObject, ImageResponse, Message, ModelResponse, PromptTokensDetailsWrapper, Usage from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome FIXED_TIME: Final[datetime] = datetime(2025, 1, 1, 12, 0, 0) @@ -54,7 +42,7 @@ OPENAI_CUSTOM_TOOL_CALL_RESPONSE = { "type": "custom", "custom": { "name": "ApplyPatch", - "input": '*** Begin Patch\n*** Update File: main.py\n@@\n+def hello():\n+ print("Hello")\n*** End Patch\n', + "input": "*** Begin Patch\n*** Update File: main.py\n@@\n+def hello():\n+ print(\"Hello\")\n*** End Patch\n", }, } ], @@ -238,7 +226,6 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) - @pytest.fixture(scope="session") def event_loop(): try: @@ -248,7 +235,6 @@ def event_loop(): yield loop loop.close() - @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -282,7 +268,6 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) - _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -297,7 +282,6 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_basic(): # Test basic conversion with minimal input @@ -312,7 +296,6 @@ def test_convert_to_image_response_basic(): assert result.created == 1234567890 assert result.data[0].url == "http://example.com/image.jpg" - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_hidden_params(): # Test with hidden params @@ -326,7 +309,6 @@ def test_convert_to_image_response_with_hidden_params(): assert result._hidden_params == {"api_key": "test_key"} - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_multiple_images(): # Test handling multiple images in response @@ -344,7 +326,6 @@ def test_convert_to_image_response_multiple_images(): assert result.data[0].url == "http://example.com/image1.jpg" assert result.data[1].url == "http://example.com/image2.jpg" - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_b64_json(): # Test handling b64_json in response @@ -357,7 +338,6 @@ def test_convert_to_image_response_with_b64_json(): assert result.data[0].b64_json == "base64encodedstring" - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_extra_fields(): response_dict = { @@ -379,7 +359,6 @@ def test_convert_to_image_response_with_extra_fields(): assert result.data[0].url == "http://example.com/image1.jpg" assert result.data[1].url == "http://example.com/image2.jpg" - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_extra_fields_2(): """ @@ -557,7 +536,6 @@ def test_convert_to_image_response_with_none_usage_fields(): assert result.usage.input_tokens_details.image_tokens == 0 assert result.usage.input_tokens_details.text_tokens == 0 - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_partial_none_usage_fields(): """ @@ -591,7 +569,6 @@ def test_convert_to_image_response_with_partial_none_usage_fields(): assert result.usage.input_tokens_details.image_tokens == 0 assert result.usage.input_tokens_details.text_tokens == 0 - @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_convert_to_image_response_with_valid_usage_fields(): """ @@ -626,7 +603,6 @@ def test_convert_to_image_response_with_valid_usage_fields(): assert result.usage.input_tokens_details.image_tokens == 30 assert result.usage.input_tokens_details.text_tokens == 20 - def test_convert_to_model_response_object_basic(): """Test basic conversion with all fields present.""" response_object = { @@ -689,8 +665,12 @@ def test_convert_to_model_response_object_basic(): assert result.usage.prompt_tokens == 19 assert result.usage.completion_tokens == 10 assert result.usage.total_tokens == 29 - assert result.usage.prompt_tokens_details == PromptTokensDetailsWrapper(cached_tokens=0) - assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0) + assert result.usage.prompt_tokens_details == PromptTokensDetailsWrapper( + cached_tokens=0 + ) + assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper( + reasoning_tokens=0 + ) # Other fields assert result.system_fingerprint == "fp_6b68a8204b" @@ -698,7 +678,6 @@ def test_convert_to_model_response_object_basic(): # hidden params assert result._hidden_params is not None - def test_convert_image_input_dict_response_to_chat_completion_response(): """Test conversion on a response with an image input.""" response_object = { @@ -749,17 +728,21 @@ def test_convert_image_input_dict_response_to_chat_completion_response(): assert choice.index == 0 assert isinstance(choice.message, Message) assert choice.message.role == "assistant" - assert choice.message.content == "\n\nThis image shows a wooden boardwalk extending through a lush green marshland." + assert ( + choice.message.content + == "\n\nThis image shows a wooden boardwalk extending through a lush green marshland." + ) assert choice.finish_reason == "stop" assert result.usage.prompt_tokens == 9 assert result.usage.completion_tokens == 12 assert result.usage.total_tokens == 21 - assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0) + assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper( + reasoning_tokens=0 + ) assert result._hidden_params is not None - def test_convert_to_model_response_object_tool_calls_invalid_json_arguments(): """ Critical test - this is a basic response from OpenAI API @@ -822,8 +805,13 @@ def test_convert_to_model_response_object_tool_calls_invalid_json_arguments(): assert len(result.choices) == 1 assert result.choices[0].message.content is None assert len(result.choices[0].message.tool_calls) == 1 - assert result.choices[0].message.tool_calls[0].function.name == "get_current_weather" - assert result.choices[0].message.tool_calls[0].function.arguments == '{"location":"Boston, MA","unit":"fahren' + assert ( + result.choices[0].message.tool_calls[0].function.name == "get_current_weather" + ) + assert ( + result.choices[0].message.tool_calls[0].function.arguments + == '{"location":"Boston, MA","unit":"fahren' + ) assert result.choices[0].finish_reason == "length" assert result.model == "gpt-4o-2024-08-06" assert result.created == 1729337288 @@ -832,7 +820,6 @@ def test_convert_to_model_response_object_tool_calls_invalid_json_arguments(): assert result.usage.total_tokens == 102 assert result.system_fingerprint == "fp_45c6de4934" - def test_convert_to_model_response_object_tool_calls_valid_json_arguments(): """ Critical test - this is a basic response from OpenAI API @@ -895,8 +882,13 @@ def test_convert_to_model_response_object_tool_calls_valid_json_arguments(): assert len(result.choices) == 1 assert result.choices[0].message.content is None assert len(result.choices[0].message.tool_calls) == 1 - assert result.choices[0].message.tool_calls[0].function.name == "get_current_weather" - assert result.choices[0].message.tool_calls[0].function.arguments == '{"location":"Boston, MA","unit":"fahrenheit"}' + assert ( + result.choices[0].message.tool_calls[0].function.name == "get_current_weather" + ) + assert ( + result.choices[0].message.tool_calls[0].function.arguments + == '{"location":"Boston, MA","unit":"fahrenheit"}' + ) assert result.choices[0].finish_reason == "length" assert result.model == "gpt-4o-2024-08-06" assert result.created == 1729337288 @@ -905,7 +897,6 @@ def test_convert_to_model_response_object_tool_calls_valid_json_arguments(): assert result.usage.total_tokens == 102 assert result.system_fingerprint == "fp_45c6de4934" - def test_convert_to_model_response_object_json_mode(): """ This test is verifying that when convert_tool_call_to_json_mode is True, a single tool call's arguments are correctly converted into the message content of the response. @@ -956,7 +947,6 @@ def test_convert_to_model_response_object_json_mode(): assert result.usage.prompt_tokens == 5 assert result.usage.completion_tokens == 5 - def test_convert_to_model_response_object_with_logprobs(): """ @@ -1192,12 +1182,13 @@ def test_convert_to_model_response_object_with_logprobs(): assert result.usage.prompt_tokens == 9 assert result.usage.completion_tokens == 9 assert result.usage.total_tokens == 18 - assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0) + assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper( + reasoning_tokens=0 + ) assert result.system_fingerprint is None assert result._hidden_params is not None - def test_convert_to_model_response_object_error(): """Test error handling for None response object.""" with pytest.raises(Exception, match="Error in response object format"): @@ -1212,7 +1203,6 @@ def test_convert_to_model_response_object_error(): convert_tool_call_to_json_mode=False, ) - def test_image_generation_openai_with_pydantic_warning(caplog): try: import logging @@ -1250,7 +1240,6 @@ def test_image_generation_openai_with_pydantic_warning(caplog): except Exception as e: pytest.fail(f"Test failed with exception: {e}") - def test_convert_to_model_response_object_with_empty_str(): """Test that convert_to_model_response_object handles empty strings correctly.""" @@ -1354,7 +1343,6 @@ def test_convert_to_model_response_object_with_empty_str(): assert resp is not None assert resp.choices[0].message.content is not None - def test_convert_to_model_response_object_with_thinking_content(): """Test that convert_to_model_response_object handles thinking content correctly.""" @@ -1402,7 +1390,6 @@ def test_convert_to_model_response_object_with_thinking_content(): assert resp is not None assert resp.choices[0].message.reasoning_content is not None - def test_convert_to_model_response_object_with_empty_error_object(): """ Test that convert_to_model_response_object handles empty error objects gracefully. @@ -1460,8 +1447,9 @@ def test_convert_to_model_response_object_with_empty_error_object(): assert isinstance(result, ModelResponse) assert result.model == "minimax-m2.1" assert len(result.choices) == 1 - assert result.choices[0].message.content == "Hey! I'm doing well, thanks for asking!" - + assert ( + result.choices[0].message.content == "Hey! I'm doing well, thanks for asking!" + ) def test_convert_to_model_response_object_with_real_error(): """ @@ -1494,7 +1482,6 @@ def test_convert_to_model_response_object_with_real_error(): assert hasattr(exc_info.value, "message") assert "Rate limit exceeded" in str(exc_info.value.message) - def test_convert_to_model_response_object_with_empty_dict_error(): """ Test that convert_to_model_response_object handles completely empty error dict. @@ -1534,7 +1521,6 @@ def test_convert_to_model_response_object_with_empty_dict_error(): assert isinstance(result, ModelResponse) assert result.choices[0].message.content == "Hello!" - def test_convert_to_model_response_object_preserves_provider_specific_fields_from_proxy(): """ Test that provider_specific_fields (e.g. Anthropic citations) are preserved @@ -1612,13 +1598,18 @@ def test_convert_to_model_response_object_preserves_provider_specific_fields_fro assert result.id == "chatcmpl-proxy-123" choice = result.choices[0] - assert choice.message.content == "Based on current reviews, the Sony WH-1000XM5 remains one of the best headphones." + assert ( + choice.message.content + == "Based on current reviews, the Sony WH-1000XM5 remains one of the best headphones." + ) assert choice.message.provider_specific_fields is not None assert "citations" in choice.message.provider_specific_fields assert choice.message.provider_specific_fields["citations"] == citations assert "web_search_results" in choice.message.provider_specific_fields - assert choice.message.provider_specific_fields["web_search_results"] == web_search_results - + assert ( + choice.message.provider_specific_fields["web_search_results"] + == web_search_results + ) def test_convert_to_model_response_object_keeps_reasoning_items_from_a_cached_response(): reasoning_items: Final = [ @@ -1706,7 +1697,6 @@ def test_convert_to_model_response_object_provider_specific_fields_merges_extra_ assert "custom_extra_field" in psf assert psf["custom_extra_field"] == "extra_value" - def test_convert_to_model_response_object_no_provider_specific_fields_still_works(): """ Test that responses without provider_specific_fields continue to work as before. @@ -1752,7 +1742,6 @@ def test_convert_to_model_response_object_no_provider_specific_fields_still_work assert result.choices[0].message.refusal is None assert result.choices[0].message.provider_specific_fields is None - def test_convert_to_model_response_object_with_error_code_only(): """ Test that errors with only a code (no message) are still treated as real errors. @@ -1778,7 +1767,6 @@ def test_convert_to_model_response_object_with_error_code_only(): assert exc_info.value.status_code == 500 - def test_model_prefix_preservation(): """ Test that when model_response_object has a prefix like 'openai/gpt-4' @@ -1807,7 +1795,6 @@ def test_model_prefix_preservation(): assert result.model == "openai/gpt-4o" - def test_model_without_prefix(): """ Test that when model_response_object has no prefix (e.g. 'gpt-4'), @@ -1836,7 +1823,6 @@ def test_model_without_prefix(): assert result.model == "gpt-4" - def test_extra_response_fields_preserved(): """ Test that extra response fields (e.g. service_tier) are preserved @@ -1866,7 +1852,6 @@ def test_extra_response_fields_preserved(): assert result.service_tier == "default" - def test_hidden_params_and_response_headers_set(): """ Test that _hidden_params and _response_headers are correctly set @@ -1901,7 +1886,6 @@ def test_hidden_params_and_response_headers_set(): assert "additional_headers" in result._hidden_params assert result._response_headers == response_headers - def test_response_ms_computed(): """ Test that _response_ms is computed correctly from start_time and end_time. @@ -1931,7 +1915,6 @@ def test_response_ms_computed(): assert result._response_ms == pytest.approx(250.0) - def test_error_message_includes_function_args(): """ Test that when an exception occurs, the error message includes @@ -1944,7 +1927,7 @@ def test_error_message_includes_function_args(): "choices": [{"index": 0}], } - with pytest.raises(Exception, match="in convert_to_model_response_object") as exc_info: + with pytest.raises(Exception, match='in convert_to_model_response_object') as exc_info: convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, @@ -1958,7 +1941,6 @@ def test_error_message_includes_function_args(): assert "response_object" in error_msg assert "response_type" in error_msg - @pytest.mark.parametrize("falsy_id", [None, ""]) def test_convert_to_model_response_object_falsy_id_preserves_auto_generated(falsy_id): """Test that a falsy id in response_object preserves the auto-generated id.""" @@ -1986,7 +1968,6 @@ def test_convert_to_model_response_object_falsy_id_preserves_auto_generated(fals assert result.id == original_id assert result.id.startswith("chatcmpl-") - def test_convert_to_model_response_object_default_usage_overwritten(): """ Regression test: convert_to_model_response_object must properly set Usage @@ -2030,7 +2011,6 @@ def test_convert_to_model_response_object_default_usage_overwritten(): assert result.usage.completion_tokens == 7 assert result.usage.total_tokens == 22 - def test_convert_to_model_response_object_with_null_top_logprobs(): """ Test that convert_to_model_response_object handles null top_logprobs @@ -2122,7 +2102,6 @@ def test_convert_to_model_response_object_with_null_top_logprobs(): assert token_logprob.top_logprobs == [] assert isinstance(token_logprob.top_logprobs, list) - class TestMissingChoicesGuard: """ Tests for the defense-in-depth guard that raises APIError when a provider @@ -2248,7 +2227,9 @@ class TestMissingChoicesGuard: async def consume(): chunks = [] - async for chunk in convert_to_streaming_response_async(response_object=response_object): + async for chunk in convert_to_streaming_response_async( + response_object=response_object + ): chunks.append(chunk) return chunks @@ -2276,7 +2257,6 @@ class TestMissingChoicesGuard: assert "copilot_usage" in exc_info.value.message - class TestNormalizeImagesForMessage: def test_none_returns_none(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2312,7 +2292,6 @@ class TestNormalizeImagesForMessage: result = _normalize_images_for_message(images) assert result[0]["index"] == 5 - class TestSafeConvertCreatedField: def test_none_returns_current_time(self): import time @@ -2355,7 +2334,6 @@ class TestSafeConvertCreatedField: result = safe_convert_created_field("not-a-number") assert abs(result - int(time.time())) <= 1 - class TestConvertToStreamingResponse: def test_none_raises(self): @@ -2448,7 +2426,9 @@ class TestConvertToStreamingResponse: async def run(): chunks = [] - async for chunk in convert_to_streaming_response_async(response_object=response_object): + async for chunk in convert_to_streaming_response_async( + response_object=response_object + ): chunks.append(chunk) return chunks @@ -2457,7 +2437,6 @@ class TestConvertToStreamingResponse: assert chunks[0].choices[0].delta.tool_calls[0].id == "call_1" assert chunks[0].choices[0].delta.tool_calls[0].function.name == "get_weather" - class TestConvertToStreamingResponseAsync: def test_none_raises(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2495,7 +2474,9 @@ class TestConvertToStreamingResponseAsync: async def run(): chunks = [] - async for chunk in convert_to_streaming_response_async(response_object=response_object): + async for chunk in convert_to_streaming_response_async( + response_object=response_object + ): chunks.append(chunk) return chunks @@ -2510,7 +2491,6 @@ class TestConvertToStreamingResponseAsync: assert chunks[-1].choices[0].finish_reason == "stop" assert chunks[-1].usage.prompt_tokens == 3 - class TestHandleInvalidParallelToolCalls: def test_none_input(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2590,7 +2570,6 @@ class TestHandleInvalidParallelToolCalls: assert len(result) == 1 assert result[0].id == "call_1" - class TestShouldConvertToolCallToJsonMode: def test_returns_true_when_conditions_met(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2599,7 +2578,12 @@ class TestShouldConvertToolCallToJsonMode: from litellm.constants import RESPONSE_FORMAT_TOOL_NAME tool_calls = [{"function": {"name": RESPONSE_FORMAT_TOOL_NAME}}] - assert should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) is True + assert ( + should_convert_tool_call_to_json_mode( + tool_calls=tool_calls, convert_tool_call_to_json_mode=True + ) + is True + ) def test_returns_false_when_flag_off(self): from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -2609,7 +2593,10 @@ class TestShouldConvertToolCallToJsonMode: tool_calls = [{"function": {"name": RESPONSE_FORMAT_TOOL_NAME}}] assert ( - should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=False) is False + should_convert_tool_call_to_json_mode( + tool_calls=tool_calls, convert_tool_call_to_json_mode=False + ) + is False ) def test_returns_false_when_wrong_tool_name(self): @@ -2619,7 +2606,10 @@ class TestShouldConvertToolCallToJsonMode: tool_calls = [{"function": {"name": "some_other_tool"}}] assert ( - should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) is False + should_convert_tool_call_to_json_mode( + tool_calls=tool_calls, convert_tool_call_to_json_mode=True + ) + is False ) def test_returns_false_when_multiple_tool_calls(self): @@ -2633,7 +2623,10 @@ class TestShouldConvertToolCallToJsonMode: {"function": {"name": "other"}}, ] assert ( - should_convert_tool_call_to_json_mode(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) is False + should_convert_tool_call_to_json_mode( + tool_calls=tool_calls, convert_tool_call_to_json_mode=True + ) + is False ) def test_returns_false_when_none(self): @@ -2641,8 +2634,12 @@ class TestShouldConvertToolCallToJsonMode: should_convert_tool_call_to_json_mode, ) - assert should_convert_tool_call_to_json_mode(tool_calls=None, convert_tool_call_to_json_mode=True) is False - + assert ( + should_convert_tool_call_to_json_mode( + tool_calls=None, convert_tool_call_to_json_mode=True + ) + is False + ) class TestConvertToolCallToJsonMode: def test_converts_when_should(self): @@ -2662,7 +2659,9 @@ class TestConvertToolCallToJsonMode: ), ) ] - message, finish_reason = convert_fn(tool_calls=tool_calls, convert_tool_call_to_json_mode=True) + message, finish_reason = convert_fn( + tool_calls=tool_calls, convert_tool_call_to_json_mode=True + ) assert message is not None assert message.content == '{"key": "value"}' assert finish_reason == "stop" @@ -2684,11 +2683,12 @@ class TestConvertToolCallToJsonMode: ), ) ] - message, finish_reason = convert_fn(tool_calls=tool_calls, convert_tool_call_to_json_mode=False) + message, finish_reason = convert_fn( + tool_calls=tool_calls, convert_tool_call_to_json_mode=False + ) assert message is None assert finish_reason is None - class TestConvertToModelResponseObjectEmbedding: def test_basic_embedding_response(self): from litellm.types.utils import EmbeddingResponse @@ -2714,7 +2714,6 @@ class TestConvertToModelResponseObjectEmbedding: assert result.data == [{"embedding": [0.1, 0.2, 0.3], "index": 0}] assert result.usage.prompt_tokens == 5 - class TestConvertToModelResponseObjectAudioTranscription: def test_basic_transcription(self): from litellm.types.utils import TranscriptionResponse @@ -2774,7 +2773,6 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 - class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse @@ -2793,7 +2791,6 @@ class TestConvertToModelResponseObjectRerank: assert result.id == "rerank-123" assert result.results[0]["relevance_score"] == 0.9 - class TestConvertToModelResponseObjectCompletion: def test_tool_calls_finish_reason_override(self): response_object = {