diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 0c1eaaaae26..73f40f5e0e4 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -11,13 +11,14 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer import openai -from openai import AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream from openai._base_client import make_request_options from openai._constants import RAW_RESPONSE_HEADER from openai._legacy_response import LegacyAPIResponse from openai._types import RequestOptions from openai.types import CreateEmbeddingResponse from openai.types.beta.assistant_deleted import AssistantDeleted +from openai.types.chat import ChatCompletion, ChatCompletionChunk from openai.types.file_deleted import FileDeleted from pydantic import BaseModel, TypeAdapter from typing_extensions import overload @@ -354,7 +355,14 @@ _NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({}) _SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body")) -def _embedding_request_without_sdk_defaults( +def _raw_chat_response( + result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk], +) -> LegacyAPIResponse[ChatCompletion]: + assert isinstance(result, LegacyAPIResponse) + return result + + +def _request_without_sdk_transform( data: Mapping[str, object], timeout: float | httpx.Timeout ) -> tuple[Mapping[str, object], RequestOptions]: body: Final = {k: v for k, v in data.items() if k not in _SDK_OPTION_KEYS} @@ -517,7 +525,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): """ start_time: Final = time.time() try: - raw_response = await openai_aclient.chat.completions.with_raw_response.create(**data, timeout=timeout) + body, options = _request_without_sdk_transform(data, timeout) + raw_response = _raw_chat_response( + await openai_aclient.post( + "/chat/completions", + body=body, + cast_to=ChatCompletion, + options=options, + stream=bool(data.get("stream")), + stream_cls=AsyncStream[ChatCompletionChunk], + ) + ) end_time = time.time() if hasattr(raw_response, "headers"): @@ -558,7 +576,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): """ raw_response = None try: - raw_response = openai_client.chat.completions.with_raw_response.create(**data, timeout=timeout) + body, options = _request_without_sdk_transform(data, timeout) + raw_response = _raw_chat_response( + openai_client.post( + "/chat/completions", + body=body, + cast_to=ChatCompletion, + options=options, + stream=bool(data.get("stream")), + stream_cls=Stream[ChatCompletionChunk], + ) + ) if hasattr(raw_response, "headers"): headers = dict(raw_response.headers) @@ -1257,7 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj: LiteLLMLoggingObj, ) -> LegacyAPIResponse[CreateEmbeddingResponse]: if "encoding_format" not in data: - body, options = _embedding_request_without_sdk_defaults(data, timeout) + body, options = _request_without_sdk_transform(data, timeout) bypass_response: Final = await openai_aclient.post( "/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse ) @@ -1274,7 +1302,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj: LiteLLMLoggingObj, ) -> LegacyAPIResponse[CreateEmbeddingResponse]: if "encoding_format" not in data: - body, options = _embedding_request_without_sdk_defaults(data, timeout) + body, options = _request_without_sdk_transform(data, timeout) bypass_response: Final = openai_client.post( "/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse ) diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index e891706df13..3cbc3591009 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -25,13 +25,13 @@ from typing import Any, Dict, Final, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import pytest - +from openai._legacy_response import LegacyAPIResponse import litellm -from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.code_interpreter_interception.handler import ( CodeInterpreterInterceptionLogger, ) +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.chat_completion_agentic_loop import ( maybe_run_chat_completion_agentic_loop, ) @@ -45,9 +45,9 @@ from litellm.types.integrations.custom_logger import ( AgenticLoopRequestPatch, ) from litellm.types.utils import ( + ChatCompletionMessageToolCall, Choices, Function, - ChatCompletionMessageToolCall, Message, ModelResponse, ) @@ -142,7 +142,7 @@ def _raw_response_for(model_response: ModelResponse) -> MagicMock: (an object exposing `.headers` and `.parse()` -> something with model_dump).""" parsed = MagicMock() parsed.model_dump.return_value = model_response.model_dump() - raw = MagicMock() + raw = MagicMock(spec=LegacyAPIResponse) raw.headers = {} raw.parse.return_value = parsed return raw @@ -171,7 +171,7 @@ async def test_internal_control_fields_never_leak_into_provider_body(restore_cal ] ) mock_client = MagicMock() - mock_client.chat.completions.with_raw_response.create = create + mock_client.post = create response = await litellm.acompletion( model="openai/gpt-4o-mini", diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 02ebae6dcf6..b24348cfaa8 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -2462,7 +2462,6 @@ def test_openai_gateway_timeout_error(): Test that the OpenAI gateway timeout error is raised """ openai_client = OpenAI() - mapped_target = openai_client.chat.completions.with_raw_response # type: ignore def _return_exception(*args, **kwargs): @@ -2498,8 +2497,8 @@ def test_openai_gateway_timeout_error(): with pytest.raises(litellm.Timeout) as exc_info: with patch.object( - mapped_target, - "create", + openai_client, + "post", side_effect=_return_exception, ): litellm.completion( diff --git a/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py index 3630ef1e354..1cdc288ff87 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py @@ -1,9 +1,10 @@ import json import traceback -from typing import Callable, Optional +from typing import Callable, Final, Optional from unittest.mock import MagicMock, patch import pytest +from openai._legacy_response import LegacyAPIResponse import litellm from litellm import ModelResponse @@ -130,7 +131,7 @@ def test_azure_o3_streaming(): api_version="2024-02-15-preview", ) - with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with patch.object(client, "post") as mock_create: try: completion( model="azure/o3-mini", @@ -141,7 +142,7 @@ def test_azure_o3_streaming(): except Exception as e: print(e) assert mock_create.call_count == 1 - assert "stream" in mock_create.call_args.kwargs + assert "stream" in mock_create.call_args.kwargs["body"] def test_azure_o_series_routing(): @@ -157,7 +158,7 @@ def test_azure_o_series_routing(): api_version="2024-02-15-preview", ) - with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with patch.object(client, "post") as mock_create: try: completion( model="azure/o_series/my-random-deployment-name", @@ -168,17 +169,19 @@ def test_azure_o_series_routing(): except Exception as e: print(e) assert mock_create.call_count == 1 - assert "stream" not in mock_create.call_args.kwargs + assert "stream" not in mock_create.call_args.kwargs["body"] @patch("litellm.main.azure_o1_chat_completions._get_openai_client") def test_openai_o_series_max_retries_0(mock_get_openai_client): import litellm - mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {} - mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = ( - ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) - ) + raw_response: Final = MagicMock(spec=LegacyAPIResponse) + raw_response.headers = {} + raw_response.parse.return_value = ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) + client_stub: Final = MagicMock() + client_stub.post.return_value = raw_response + mock_get_openai_client.return_value = client_stub litellm.set_verbose = True response = litellm.completion( model="azure/o1-preview", @@ -233,7 +236,7 @@ async def test_azure_o1_series_response_format_extra_params(): response_format = {"type": "json_object"} tool_choice = "auto" with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: await litellm.acompletion( @@ -251,7 +254,7 @@ async def test_azure_o1_series_response_format_extra_params(): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] print("request_body: ", json.dumps(request_body, indent=4)) assert request_body["tools"] == tools diff --git a/tests/unit/llms/azure/test_azure.py b/tests/unit/llms/azure/test_azure.py index 7907344ae64..7d5afde68ef 100644 --- a/tests/unit/llms/azure/test_azure.py +++ b/tests/unit/llms/azure/test_azure.py @@ -313,7 +313,7 @@ def test_azure_openai_gpt_4o_naming(monkeypatch): number: str = Field(description="total number of days in a week") days: list[str] = Field(description="name of days in a week") - with patch.object(client.chat.completions.with_raw_response, "create") as mock_post: + with patch.object(client, "post") as mock_post: try: completion( model="azure/gpt4o", @@ -326,9 +326,9 @@ def test_azure_openai_gpt_4o_naming(monkeypatch): mock_post.assert_called_once() - print(mock_post.call_args.kwargs) + print(mock_post.call_args.kwargs["body"]) - assert "tool_calls" not in mock_post.call_args.kwargs + assert "tool_calls" not in mock_post.call_args.kwargs["body"] @pytest.mark.parametrize( @@ -382,7 +382,7 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version): } ] - with patch.object(client.chat.completions.with_raw_response, "create") as mock_post: + with patch.object(client, "post") as mock_post: mock_post.return_value.headers = {} mock_post.return_value.parse.return_value = litellm.ModelResponse( choices=[{"message": {"role": "assistant", "content": InvestigationOutput().model_dump_json()}}] @@ -411,9 +411,9 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version): mock_post.assert_called_once() if api_version == "2024-10-21": - assert "response_format" in mock_post.call_args.kwargs + assert "response_format" in mock_post.call_args.kwargs["body"] else: - assert "response_format" not in mock_post.call_args.kwargs + assert "response_format" not in mock_post.call_args.kwargs["body"] assert response.choices[0].message.content == InvestigationOutput().model_dump_json() diff --git a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py index 86fb6fa15d8..9b8788d3769 100644 --- a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py +++ b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py @@ -1,15 +1,16 @@ import json +import re from io import BytesIO from pathlib import Path from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx +import pytest from openai import AsyncOpenAI, OpenAI +from openai._legacy_response import LegacyAPIResponse from openai.types import CreateEmbeddingResponse, Embedding from openai.types.create_embedding_response import Usage -import pytest -import re import litellm from litellm import completion @@ -18,7 +19,6 @@ from litellm.llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfi from tests._vcr_conftest_common import rewound_new_episodes_cassette from tests.capturing_transport import CapturingTransport - _GATEWAY_EMBEDDING_RESPONSE: Final = CreateEmbeddingResponse( object="list", data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), @@ -98,7 +98,7 @@ async def test_litellm_gateway_from_sdk(): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions.with_raw_response, "create", new=MagicMock() + openai_client, "post", new=MagicMock() ) as mock_call: try: completion( @@ -114,9 +114,9 @@ async def test_litellm_gateway_from_sdk(): mock_call.assert_called_once() - print("Call KWARGS - {}".format(mock_call.call_args.kwargs)) + print("Call KWARGS - {}".format(mock_call.call_args.kwargs["body"])) - assert "hello" in mock_call.call_args.kwargs["extra_body"] + assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"] @pytest.mark.parametrize("is_async", (False, True)) @@ -264,7 +264,7 @@ async def test_litellm_gateway_from_sdk_rerank(is_async): patch_target = client.post with patch.object(client, "post", new=mock_method): - mock_response = MagicMock() + mock_response = MagicMock(spec=LegacyAPIResponse) # Create a mock response similar to OpenAI's rerank response mock_response.text = json.dumps( @@ -408,7 +408,7 @@ async def test_litellm_gateway_from_sdk_structured_output(): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions, "create", new=MagicMock() + openai_client, "post", new=MagicMock() ) as mock_call: try: litellm.completion( @@ -427,8 +427,8 @@ async def test_litellm_gateway_from_sdk_structured_output(): mock_call.assert_called_once() - print("Call KWARGS - {}".format(mock_call.call_args.kwargs)) - json_schema = mock_call.call_args.kwargs["response_format"] + print("Call KWARGS - {}".format(mock_call.call_args.kwargs["body"])) + json_schema = mock_call.call_args.kwargs["body"]["response_format"] assert "json_schema" in json_schema @@ -486,7 +486,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers(): openai_client = OpenAI(api_key="fake-key") # Create mock response object - mock_response = MagicMock() + mock_response = MagicMock(spec=LegacyAPIResponse) mock_response.headers = {"x-litellm-response-cost": "120"} mock_response.parse.return_value = litellm.ModelResponse( **{ @@ -528,8 +528,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers(): ) with patch.object( - openai_client.chat.completions.with_raw_response, - "create", + openai_client, "post", return_value=mock_response, ) as mock_call: response = litellm.completion( @@ -564,7 +563,7 @@ async def test_litellm_gateway_image_generation_direct(is_async): "created": 1, "data": [{"url": "https://example.com/image.png"}], } - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.parse.return_value = mock_openai_response mock_raw_response.headers = {} diff --git a/tests/unit/llms/nvidia_nim/test_nvidia_nim.py b/tests/unit/llms/nvidia_nim/test_nvidia_nim.py index e5298c7ba39..3633ac97265 100644 --- a/tests/unit/llms/nvidia_nim/test_nvidia_nim.py +++ b/tests/unit/llms/nvidia_nim/test_nvidia_nim.py @@ -94,7 +94,7 @@ def test_chat_completion_nvidia_nim_with_tools(): ] with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: completion( @@ -116,7 +116,7 @@ def test_chat_completion_nvidia_nim_with_tools(): # Add assertions to check the request mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] print("request_body: ", request_body) @@ -208,7 +208,7 @@ def test_completion_nvidia_nim(): api_key="fake-api-key", ) - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object(client, "post") as mock_client: try: completion( model=model_name, @@ -226,7 +226,7 @@ def test_completion_nvidia_nim(): print(e) mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] assert request_body["messages"] == [ { diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index 6ee882b7d32..121fab66c15 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -70,7 +70,7 @@ async def test_o1_handle_system_role(model): client = AsyncOpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: await litellm.acompletion( @@ -83,7 +83,7 @@ async def test_o1_handle_system_role(model): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] print("request_body: ", request_body) @@ -141,7 +141,7 @@ async def test_o1_max_completion_tokens(model: str): client = AsyncOpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: await litellm.acompletion( @@ -154,7 +154,7 @@ async def test_o1_max_completion_tokens(model: str): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] print("request_body: ", request_body) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index d7ca9319f3a..11d7c7d6d3c 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -5,12 +5,13 @@ import os from collections.abc import Mapping from types import ModuleType from typing import Final, Literal, Protocol -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import httpx import pytest import respx from openai import AsyncOpenAI, AzureOpenAI, OpenAI +from openai._legacy_response import LegacyAPIResponse from openai._types import Response as SDKResponse from pydantic import BaseModel, ConfigDict, JsonValue @@ -1669,7 +1670,7 @@ async def test_openai_prediction_param_mock(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: await litellm.acompletion( @@ -1688,7 +1689,7 @@ async def test_openai_prediction_param_mock(): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] # Verify the request contains the prediction parameter assert "prediction" in request_body @@ -1700,10 +1701,12 @@ async def test_openai_prediction_param_mock(): def test_openai_max_retries_0(mock_get_openai_client): import litellm - mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {} - mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = ( - ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) - ) + raw_response: Final = MagicMock(spec=LegacyAPIResponse) + raw_response.headers = {} + raw_response.parse.return_value = ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) + client_stub: Final = MagicMock() + client_stub.post.return_value = raw_response + mock_get_openai_client.return_value = client_stub litellm.set_verbose = True response = litellm.completion( model="gpt-4o-mini", @@ -1812,7 +1815,7 @@ async def test_openai_safety_identifier_parameter(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: await litellm.acompletion( @@ -1825,7 +1828,7 @@ async def test_openai_safety_identifier_parameter(): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] # Verify the request contains the safety_identifier parameter assert "safety_identifier" in request_body @@ -1841,7 +1844,7 @@ def test_openai_safety_identifier_parameter_sync(): client = OpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: litellm.completion( @@ -1854,7 +1857,7 @@ def test_openai_safety_identifier_parameter_sync(): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] # Verify the request contains the safety_identifier parameter assert "safety_identifier" in request_body @@ -1871,7 +1874,7 @@ async def test_openai_service_tier_parameter(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: await litellm.acompletion( @@ -1884,7 +1887,7 @@ async def test_openai_service_tier_parameter(): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] # Verify the request contains the service_tier parameter assert "service_tier" in request_body, "service_tier should be in request body" @@ -1902,7 +1905,7 @@ def test_openai_service_tier_parameter_sync(): client = OpenAI(api_key="fake-api-key") with patch.object( - client.chat.completions.with_raw_response, "create" + client, "post" ) as mock_client: try: litellm.completion( @@ -1915,7 +1918,7 @@ def test_openai_service_tier_parameter_sync(): print(f"Error: {e}") mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] # Verify the request contains the service_tier parameter assert "service_tier" in request_body, "service_tier should be in request body" @@ -1947,3 +1950,63 @@ def test_responses_gpt54_with_xhigh_reasoning(): assert request_body["model"] == "openai/gpt-5.4" assert request_body["reasoning"] == {"effort": "xhigh"} + + +_CHAT_RESPONSE_JSON: Final = { + "id": "chatcmpl-wire", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.4", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +_WIRE_REQUEST: Final = { + "model": "gpt-5.4", + "messages": [ + {"role": "system", "content": "sys"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "what is in the image"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png", "detail": "low"}}, + ], + }, + ], + "tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}], + "tool_choice": "auto", + "response_format": {"type": "json_schema", "json_schema": {"name": "r", "schema": {"type": "object"}}}, + "max_completion_tokens": 7, + "reasoning_effort": "none", + "n": 2, + "seed": 3, +} + + +@pytest.mark.parametrize("is_async", (False, True)) +@respx.mock +async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_headers_unchanged( + is_async: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE_JSON) + ) + kwargs: Final = { + **_WIRE_REQUEST, + "model": "openai/gpt-5.4", + "api_base": "https://api.openai.com/v1", + "api_key": "sk-test", + "num_retries": 0, + "extra_body": {"vendor_flag": True}, + "extra_headers": {"x-caller": "unit"}, + } + + if is_async: + await litellm.acompletion(**kwargs) + else: + litellm.completion(**kwargs) + + sent: Final = route.calls.last.request + assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True} + assert sent.headers["x-caller"] == "unit" + assert sent.headers["authorization"] == "Bearer sk-test" diff --git a/tests/unit/llms/openai/test_openai_empty_response.py b/tests/unit/llms/openai/test_openai_empty_response.py index 26b28f967db..1505d9ef049 100644 --- a/tests/unit/llms/openai/test_openai_empty_response.py +++ b/tests/unit/llms/openai/test_openai_empty_response.py @@ -2,13 +2,13 @@ Test for issue #17209: Clearer error when LLM endpoint returns empty response """ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest +from openai._legacy_response import LegacyAPIResponse - -from litellm.llms.openai.openai import OpenAIChatCompletion from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.openai.openai import OpenAIChatCompletion class TestEmptyResponseHandling: @@ -22,12 +22,12 @@ class TestEmptyResponseHandling: openai_chat = OpenAIChatCompletion() # Mock the raw response to return an empty string from parse() - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = {} mock_raw_response.parse.return_value = "" # Empty string response mock_client = MagicMock() - mock_client.chat.completions.with_raw_response.create.return_value = ( + mock_client.post.return_value = ( mock_raw_response ) @@ -48,12 +48,12 @@ class TestEmptyResponseHandling: """Test that None response also produces a clear error""" openai_chat = OpenAIChatCompletion() - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = {} mock_raw_response.parse.return_value = None mock_client = MagicMock() - mock_client.chat.completions.with_raw_response.create.return_value = ( + mock_client.post.return_value = ( mock_raw_response ) @@ -75,12 +75,12 @@ class TestEmptyResponseHandling: mock_response = MagicMock() mock_response.model_dump.return_value = {"choices": []} - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = {"x-request-id": "123"} mock_raw_response.parse.return_value = mock_response mock_client = MagicMock() - mock_client.chat.completions.with_raw_response.create.return_value = ( + mock_client.post.return_value = ( mock_raw_response ) @@ -104,12 +104,12 @@ class TestEmptyResponseHandling: # Create a mock response WITHOUT model_dump (like an AsyncStream/Iterator) mock_stream = MagicMock(spec=[]) # spec=[] means no attributes - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = {"x-request-id": "123"} mock_raw_response.parse.return_value = mock_stream mock_client = MagicMock() - mock_client.chat.completions.with_raw_response.create.return_value = ( + mock_client.post.return_value = ( mock_raw_response ) diff --git a/tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py b/tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py index 7013afc7a5f..58f1ff60ba5 100644 --- a/tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py +++ b/tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py @@ -9,6 +9,7 @@ body. OpenAI/Anthropic reject unknown body params with HTTP 400. from unittest.mock import MagicMock +from openai._legacy_response import LegacyAPIResponse import litellm from litellm.types.utils import all_litellm_params @@ -47,12 +48,12 @@ def test_completion_does_not_leak_flag_into_provider_request_body(): }, } - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = {} mock_raw_response.parse.return_value = mock_response mock_client = MagicMock() - mock_client.chat.completions.with_raw_response.create.return_value = ( + mock_client.post.return_value = ( mock_raw_response ) @@ -65,7 +66,7 @@ def test_completion_does_not_leak_flag_into_provider_request_body(): ) create_kwargs = ( - mock_client.chat.completions.with_raw_response.create.call_args.kwargs + mock_client.post.call_args.kwargs ) assert "use_chat_completions_api" not in create_kwargs assert "use_chat_completions_api" not in (create_kwargs.get("extra_body") or {}) diff --git a/tests/unit/llms/perplexity/chat/test_transformation.py b/tests/unit/llms/perplexity/chat/test_transformation.py index 6a70fd1232d..0cb6335f5a8 100644 --- a/tests/unit/llms/perplexity/chat/test_transformation.py +++ b/tests/unit/llms/perplexity/chat/test_transformation.py @@ -4,6 +4,7 @@ import os from unittest.mock import MagicMock, patch import pytest +from openai._legacy_response import LegacyAPIResponse import litellm from litellm import completion @@ -92,7 +93,7 @@ class TestPerplexityReasoning: pydantic_obj = ChatCompletion(**response_object) def _return_pydantic_obj(*args, **kwargs): - new_response = MagicMock() + new_response = MagicMock(spec=LegacyAPIResponse) new_response.headers = {"content-type": "application/json"} new_response.parse.return_value = pydantic_obj return new_response @@ -100,8 +101,7 @@ class TestPerplexityReasoning: openai_client = OpenAI(api_key="fake-api-key") with patch.object( - openai_client.chat.completions.with_raw_response, - "create", + openai_client, "post", side_effect=_return_pydantic_obj, ) as mock_client: response = completion( @@ -121,7 +121,7 @@ class TestPerplexityReasoning: # Get the request data from the mock call call_args = mock_client.call_args - request_data = call_args.kwargs + request_data = call_args.kwargs["body"] # Verify reasoning_effort was included in the request assert "reasoning_effort" in request_data diff --git a/tests/unit/llms/volcengine/test_volcengine.py b/tests/unit/llms/volcengine/test_volcengine.py index d472d52d2db..dff9a0c62b2 100644 --- a/tests/unit/llms/volcengine/test_volcengine.py +++ b/tests/unit/llms/volcengine/test_volcengine.py @@ -2,6 +2,7 @@ import os import sys from unittest.mock import MagicMock, patch +from openai._legacy_response import LegacyAPIResponse from pydantic import BaseModel from litellm.llms.volcengine.chat.transformation import ( @@ -117,7 +118,7 @@ class TestVolcEngineConfig: client = OpenAI(api_key="test_api_key") - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = { "x-request-id": "123", "openai-organization": "org-123", @@ -126,9 +127,7 @@ class TestVolcEngineConfig: } mock_raw_response.parse.return_value = ModelResponse() - with patch.object( - client.chat.completions.with_raw_response, "create", mock_raw_response - ) as mock_create: + with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create: completion( model="volcengine/doubao-seed-1.6", messages=[ @@ -144,11 +143,6 @@ class TestVolcEngineConfig: ) mock_create.assert_called_once() - print(mock_create.call_args.kwargs) + print(mock_create.call_args.kwargs["body"]) # Fixed: thinking disabled should appear in extra_body with original structure - assert ( - "extra_body" in mock_create.call_args.kwargs - and "thinking" in mock_create.call_args.kwargs.get("extra_body", {}) - and mock_create.call_args.kwargs.get("extra_body", {})["thinking"] - == {"type": "disabled"} - ) + assert mock_create.call_args.kwargs["options"]["extra_json"]["thinking"] == {"type": "disabled"} diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 14cea8f0c2c..bb0e8db7f1c 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -21,15 +21,22 @@ import httpx import pytest import respx from openai import APITimeoutError, AsyncOpenAI +from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion import litellm -from litellm import acompletion, completion -from litellm import acompletion_with_retries, aresponses_with_retries, completion_with_retries, responses_with_retries +from litellm import ( + acompletion, + acompletion_with_retries, + aresponses_with_retries, + completion, + completion_with_retries, + responses_with_retries, +) from litellm import main as litellm_main -from litellm.constants import CONTROL_OPTIONS_KEY from litellm.caching.base_cache import BaseCache from litellm.caching.caching import Cache +from litellm.constants import CONTROL_OPTIONS_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs @@ -130,7 +137,7 @@ def test_completion_missing_role(openai_api_response): client = OpenAI(api_key="test_api_key") - mock_raw_response = MagicMock() + mock_raw_response = MagicMock(spec=LegacyAPIResponse) mock_raw_response.headers = { "x-request-id": "123", "openai-organization": "org-123", @@ -142,7 +149,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") with patch.object( - client.chat.completions.with_raw_response, "create", MagicMock(return_value=mock_raw_response) + client, "post", MagicMock(return_value=mock_raw_response) ) as mock_create: litellm.completion( model="gpt-4o-mini", @@ -218,7 +225,7 @@ async def test_url_with_format_param_openai(model, sync_mode): } ], } - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object(client, "post") as mock_client: try: if sync_mode: response = completion(**args, client=client) @@ -232,7 +239,7 @@ async def test_url_with_format_param_openai(model, sync_mode): print(mock_client.call_args.kwargs) - json_str = json.dumps(mock_client.call_args.kwargs) + json_str = json.dumps(mock_client.call_args.kwargs["body"]) assert "format" not in json_str @@ -3005,9 +3012,10 @@ def test_stream_chunk_builder_thinking_blocks(): assert response.choices[0].message.thinking_blocks is not None -from litellm.llms.openai.openai import OpenAIChatCompletion import traceback +from litellm.llms.openai.openai import OpenAIChatCompletion + user_message = "Write a short poem about the sky" messages = [{"content": user_message, "role": "user"}] @@ -3777,7 +3785,7 @@ def test_completion_forwards_store_and_prompt_cache_key_to_openai(): client = OpenAI(api_key="fake-api-key") - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object(client, "post") as mock_client: try: litellm.completion( model="openai/gpt-4o", @@ -3790,7 +3798,7 @@ def test_completion_forwards_store_and_prompt_cache_key_to_openai(): print(e) mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] assert request_body["store"] is False assert request_body["prompt_cache_key"] == "test-cache-key" @@ -3805,7 +3813,7 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): client = AsyncOpenAI(api_key="fake-api-key") - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object(client, "post") as mock_client: try: await litellm.acompletion( model="openai/gpt-4o", @@ -3818,7 +3826,7 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): print(e) mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] assert request_body["store"] is False assert request_body["prompt_cache_key"] == "test-cache-key" @@ -3832,7 +3840,7 @@ def test_completion_omits_store_and_prompt_cache_key_when_not_passed(): client = OpenAI(api_key="fake-api-key") - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object(client, "post") as mock_client: try: litellm.completion( model="openai/gpt-4o", @@ -3843,7 +3851,7 @@ def test_completion_omits_store_and_prompt_cache_key_when_not_passed(): print(e) mock_client.assert_called_once() - request_body = mock_client.call_args.kwargs + request_body = mock_client.call_args.kwargs["body"] assert "store" not in request_body assert "prompt_cache_key" not in request_body @@ -4038,8 +4046,7 @@ def test_openai_model_without_a_provider_still_routes_to_openai(): from openai import OpenAI client = OpenAI(api_key="fake-key") - raw_response = client.chat.completions.with_raw_response - with patch.object(raw_response, "create") as mock_create, contextlib.suppress(Exception): + with patch.object(client, "post") as mock_create, contextlib.suppress(Exception): litellm.completion( model="gpt-4o", messages=[{"role": "user", "content": "hello"}], @@ -4050,7 +4057,7 @@ def test_openai_model_without_a_provider_still_routes_to_openai(): def _openai_chat_create_kwargs(client, **completion_kwargs): - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object(client, "post") as mock_client: with contextlib.suppress(Exception): litellm.completion( messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], @@ -4075,7 +4082,7 @@ def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6 from openai import OpenAI client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") - request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1")["body"] assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) @@ -4087,7 +4094,7 @@ def test_completion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6 from openai import OpenAI client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") - request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1")["body"] assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) @@ -4100,7 +4107,7 @@ async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_ from openai import AsyncOpenAI client = AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") - with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with patch.object(client, "post") as mock_create: with contextlib.suppress(Exception): await litellm.acompletion( model="gpt-5.6", @@ -4111,7 +4118,7 @@ async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_ ) mock_create.assert_called_once() - request_body = mock_create.call_args.kwargs + request_body = mock_create.call_args.kwargs["body"] assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) @@ -4123,12 +4130,12 @@ def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6() from openai import OpenAI client = OpenAI(api_key="fake-api-key") - request_body = _openai_chat_create_kwargs(client, model="gpt-5.6") + request = _openai_chat_create_kwargs(client, model="gpt-5.6") - assert request_body["messages"][0]["content"] == [ + assert request["body"]["messages"][0]["content"] == [ {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} ] - assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "implicit"} + assert request["options"]["extra_json"]["prompt_cache_options"] == {"mode": "implicit"} _SUBSCRIPTION_OAUTH_CREDENTIAL = "Bearer sk-ant-oat01-fake-subscription-token-for-testing-0123456789" @@ -5141,6 +5148,7 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None: def test_acompletion_params(): import inspect + from litellm.types.completion import CompletionRequest acompletion_params_odict = inspect.signature(acompletion).parameters @@ -5171,7 +5179,7 @@ def test_acompletion_params(): def _openai_mock_response(*args: object, **kwargs: object) -> MagicMock: - new_response: Final = MagicMock() + new_response: Final = MagicMock(spec=LegacyAPIResponse) new_response.headers = {"hello": "world"} response_object: Final = { "id": "chatcmpl-123", @@ -5206,7 +5214,7 @@ def test_null_role_response(): openai_client = openai.OpenAI() with patch.object( - openai_client.chat.completions, "create", side_effect=_openai_mock_response + openai_client, "post", side_effect=_openai_mock_response ) as mock_response: response = litellm.completion( model="gpt-3.5-turbo", @@ -5296,7 +5304,7 @@ def test_completion_perplexity_api(): pydantic_obj = ChatCompletion(**response_object) def _return_pydantic_obj(*args, **kwargs): - new_response = MagicMock() + new_response = MagicMock(spec=LegacyAPIResponse) new_response.headers = {"hello": "world"} new_response.parse.return_value = pydantic_obj @@ -5305,8 +5313,7 @@ def test_completion_perplexity_api(): openai_client = OpenAI() with patch.object( - openai_client.chat.completions.with_raw_response, - "create", + openai_client, "post", side_effect=_return_pydantic_obj, ) as mock_client: # litellm.set_verbose= True @@ -5350,7 +5357,7 @@ async def test_openai_compatible_custom_api_base(provider): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions, "create", new=MagicMock() + openai_client, "post", new=MagicMock() ) as mock_call: try: completion( @@ -5368,7 +5375,7 @@ async def test_openai_compatible_custom_api_base(provider): print("Call KWARGS - {}".format(mock_call.call_args.kwargs)) - assert "hello" in mock_call.call_args.kwargs["extra_body"] + assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"] @pytest.mark.parametrize( @@ -5401,7 +5408,7 @@ async def test_openai_compatible_custom_api_video(provider): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions, "create", new=MagicMock() + openai_client, "post", new=MagicMock() ) as mock_call: try: completion( @@ -7264,7 +7271,7 @@ def _openai_hallucinated_tool_call_mock_response( *args: object, **kwargs: object, ) -> MagicMock: - new_response: Final = MagicMock() + new_response: Final = MagicMock(spec=LegacyAPIResponse) new_response.headers = {"hello": "world"} response_object: Final = { "id": "chatcmpl-123", @@ -7322,8 +7329,8 @@ def test_openai_hallucinated_tool_call(): openai_client = openai.OpenAI() with patch.object( - openai_client.chat.completions, - "create", + openai_client, + "post", side_effect=_openai_hallucinated_tool_call_mock_response, ) as mock_response: response = litellm.completion( @@ -7445,7 +7452,7 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions.with_raw_response, "create" + openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse) ) as mock_call: mock_call.return_value.headers = {} mock_call.return_value.parse.return_value = litellm.ModelResponse( @@ -7464,11 +7471,11 @@ def test_completion_novita_ai(): # Verify model is passed correctly assert ( - mock_call.call_args.kwargs["model"] + mock_call.call_args.kwargs["body"]["model"] == "meta-llama/llama-3.3-70b-instruct" ) # Verify messages are passed correctly - assert mock_call.call_args.kwargs["messages"] == messages + assert mock_call.call_args.kwargs["body"]["messages"] == messages except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -7491,8 +7498,8 @@ def test_completion_novita_ai_dynamic_params(api_key): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions, - "create", + openai_client, + "post", side_effect=Exception("Invalid API key"), ) as mock_call: with pytest.raises(Exception, match="Invalid API key") as exc_info: @@ -7524,7 +7531,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features) with patch.object( - client.chat.completions.with_raw_response, "create", return_value=MagicMock() + client, "post", return_value=MagicMock() ) as mock_completion: try: resp = litellm.completion( @@ -7538,11 +7545,11 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): mock_completion.assert_called_once() if enable_preview_features: - assert mock_completion.call_args.kwargs["metadata"] == { + assert mock_completion.call_args.kwargs["body"]["metadata"] == { "my-test-key": "my-test-value" } else: - assert "metadata" not in mock_completion.call_args.kwargs + assert "metadata" not in mock_completion.call_args.kwargs["body"] AZURE_TTS_BASE: Final = "https://tts.example.azure.com"