scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 125

fix(openai): send chat completions through client.post so the SDK's per-message param transform stops holding the event loop (#46967)

Browse source 
* fix(openai): send chat completions through client.post so the SDK's per-message param transform stops holding the event loop

create() walks every message, content part and tool of the request through
openai._utils._transform before posting, 91 ms at 300k tokens and ~300 ms at
1M tokens on the event loop, and returns the same dict for the plain dicts
litellm builds. client.post is what create() calls after that walk, same
request, headers, retries, raw response and stream objects, so the chat
helpers now call it directly through the helper embeddings already used

* fix(openai): type the chat client.post result as LegacyAPIResponse so the type gate sees .headers and .parse

* fix(openai): assert the chat client.post result is a LegacyAPIResponse instead of casting it

* test(openai): stub client.post with a LegacyAPIResponse spec in the empty-response and no-leak tests

* test(core-utils): stub client.post with a LegacyAPIResponse spec in the agentic loop leak test

* test: every client.post stub on the OpenAI chat path returns a LegacyAPIResponse-spec'd mock

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-10-16 18:06:01 -07:00 • committed by GitHub
parent 6b3a5dc3ce
commit e8550c1454
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 234 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
litellm/llms/openai/openai.py
View file

@ -11,13 +11,14 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
import openai
from openai import AsyncOpenAI, OpenAI
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, TypeAdapter
from typing_extensions import overload
@ -354,7 +355,14 @@ _NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
_SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body"))
def _embedding_request_without_sdk_defaults(
def _raw_chat_response(
result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk],
) -> LegacyAPIResponse[ChatCompletion]:
assert isinstance(result, LegacyAPIResponse)
return result
def _request_without_sdk_transform(
data: Mapping[str, object], timeout: float | httpx.Timeout
) -> tuple[Mapping[str, object], RequestOptions]:
body: Final = {k: v for k, v in data.items() if k not in _SDK_OPTION_KEYS}
@ -517,7 +525,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"""
start_time: Final = time.time()
try:
raw_response = await openai_aclient.chat.completions.with_raw_response.create(**data, timeout=timeout)
body, options = _request_without_sdk_transform(data, timeout)
raw_response = _raw_chat_response(
await openai_aclient.post(
"/chat/completions",
body=body,
cast_to=ChatCompletion,
options=options,
stream=bool(data.get("stream")),
stream_cls=AsyncStream[ChatCompletionChunk],
)
)
end_time = time.time()
if hasattr(raw_response, "headers"):
@ -558,7 +576,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"""
raw_response = None
try:
raw_response = openai_client.chat.completions.with_raw_response.create(**data, timeout=timeout)
body, options = _request_without_sdk_transform(data, timeout)
raw_response = _raw_chat_response(
openai_client.post(
"/chat/completions",
body=body,
cast_to=ChatCompletion,
options=options,
stream=bool(data.get("stream")),
stream_cls=Stream[ChatCompletionChunk],
)
)
if hasattr(raw_response, "headers"):
headers = dict(raw_response.headers)
@ -1257,7 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj: LiteLLMLoggingObj,
) -> LegacyAPIResponse[CreateEmbeddingResponse]:
if "encoding_format" not in data:
body, options = _embedding_request_without_sdk_defaults(data, timeout)
body, options = _request_without_sdk_transform(data, timeout)
bypass_response: Final = await openai_aclient.post(
"/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
)
@ -1274,7 +1302,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj: LiteLLMLoggingObj,
) -> LegacyAPIResponse[CreateEmbeddingResponse]:
if "encoding_format" not in data:
body, options = _embedding_request_without_sdk_defaults(data, timeout)
body, options = _request_without_sdk_transform(data, timeout)
bypass_response: Final = openai_client.post(
"/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
)

10
tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py
View file

@ -25,13 +25,13 @@ from typing import Any, Dict, Final, List, Optional, Tuple
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openai._legacy_response import LegacyAPIResponse
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.code_interpreter_interception.handler import (
CodeInterpreterInterceptionLogger,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
maybe_run_chat_completion_agentic_loop,
)
@ -45,9 +45,9 @@ from litellm.types.integrations.custom_logger import (
AgenticLoopRequestPatch,
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Function,
ChatCompletionMessageToolCall,
Message,
ModelResponse,
)
@ -142,7 +142,7 @@ def _raw_response_for(model_response: ModelResponse) -> MagicMock:
(an object exposing `.headers` and `.parse()` -> something with model_dump)."""
parsed = MagicMock()
parsed.model_dump.return_value = model_response.model_dump()
raw = MagicMock()
raw = MagicMock(spec=LegacyAPIResponse)
raw.headers = {}
raw.parse.return_value = parsed
return raw
@ -171,7 +171,7 @@ async def test_internal_control_fields_never_leak_into_provider_body(restore_cal
]
)
mock_client = MagicMock()
mock_client.chat.completions.with_raw_response.create = create
mock_client.post = create
response = await litellm.acompletion(
model="openai/gpt-4o-mini",

5
tests/unit/litellm_core_utils/test_exception_mapping_utils.py
View file

@ -2462,7 +2462,6 @@ def test_openai_gateway_timeout_error():
Test that the OpenAI gateway timeout error is raised
"""
openai_client = OpenAI()
mapped_target = openai_client.chat.completions.with_raw_response # type: ignore
def _return_exception(*args, **kwargs):
@ -2498,8 +2497,8 @@ def test_openai_gateway_timeout_error():
with pytest.raises(litellm.Timeout) as exc_info:
with patch.object(
mapped_target,
"create",
openai_client,
"post",
side_effect=_return_exception,
):
litellm.completion(

25
tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py
View file

@ -1,9 +1,10 @@
import json
import traceback
from typing import Callable, Optional
from typing import Callable, Final, Optional
from unittest.mock import MagicMock, patch
import pytest
from openai._legacy_response import LegacyAPIResponse
import litellm
from litellm import ModelResponse
@ -130,7 +131,7 @@ def test_azure_o3_streaming():
api_version="2024-02-15-preview",
)
with patch.object(client.chat.completions.with_raw_response, "create") as mock_create:
with patch.object(client, "post") as mock_create:
try:
completion(
model="azure/o3-mini",
@ -141,7 +142,7 @@ def test_azure_o3_streaming():
except Exception as e:
print(e)
assert mock_create.call_count == 1
assert "stream" in mock_create.call_args.kwargs
assert "stream" in mock_create.call_args.kwargs["body"]
def test_azure_o_series_routing():
@ -157,7 +158,7 @@ def test_azure_o_series_routing():
api_version="2024-02-15-preview",
)
with patch.object(client.chat.completions.with_raw_response, "create") as mock_create:
with patch.object(client, "post") as mock_create:
try:
completion(
model="azure/o_series/my-random-deployment-name",
@ -168,17 +169,19 @@ def test_azure_o_series_routing():
except Exception as e:
print(e)
assert mock_create.call_count == 1
assert "stream" not in mock_create.call_args.kwargs
assert "stream" not in mock_create.call_args.kwargs["body"]
@patch("litellm.main.azure_o1_chat_completions._get_openai_client")
def test_openai_o_series_max_retries_0(mock_get_openai_client):
import litellm
mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {}
mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = (
ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}])
)
raw_response: Final = MagicMock(spec=LegacyAPIResponse)
raw_response.headers = {}
raw_response.parse.return_value = ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}])
client_stub: Final = MagicMock()
client_stub.post.return_value = raw_response
mock_get_openai_client.return_value = client_stub
litellm.set_verbose = True
response = litellm.completion(
model="azure/o1-preview",
@ -233,7 +236,7 @@ async def test_azure_o1_series_response_format_extra_params():
response_format = {"type": "json_object"}
tool_choice = "auto"
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
await litellm.acompletion(
@ -251,7 +254,7 @@ async def test_azure_o1_series_response_format_extra_params():
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
print("request_body: ", json.dumps(request_body, indent=4))
assert request_body["tools"] == tools

12
tests/unit/llms/azure/test_azure.py
View file

@ -313,7 +313,7 @@ def test_azure_openai_gpt_4o_naming(monkeypatch):
number: str = Field(description="total number of days in a week")
days: list[str] = Field(description="name of days in a week")
with patch.object(client.chat.completions.with_raw_response, "create") as mock_post:
with patch.object(client, "post") as mock_post:
try:
completion(
model="azure/gpt4o",
@ -326,9 +326,9 @@ def test_azure_openai_gpt_4o_naming(monkeypatch):
mock_post.assert_called_once()
print(mock_post.call_args.kwargs)
print(mock_post.call_args.kwargs["body"])
assert "tool_calls" not in mock_post.call_args.kwargs
assert "tool_calls" not in mock_post.call_args.kwargs["body"]
@pytest.mark.parametrize(
@ -382,7 +382,7 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version):
}
]
with patch.object(client.chat.completions.with_raw_response, "create") as mock_post:
with patch.object(client, "post") as mock_post:
mock_post.return_value.headers = {}
mock_post.return_value.parse.return_value = litellm.ModelResponse(
choices=[{"message": {"role": "assistant", "content": InvestigationOutput().model_dump_json()}}]
@ -411,9 +411,9 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version):
mock_post.assert_called_once()
if api_version == "2024-10-21":
assert "response_format" in mock_post.call_args.kwargs
assert "response_format" in mock_post.call_args.kwargs["body"]
else:
assert "response_format" not in mock_post.call_args.kwargs
assert "response_format" not in mock_post.call_args.kwargs["body"]
assert response.choices[0].message.content == InvestigationOutput().model_dump_json()

27
tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py
View file

@ -1,15 +1,16 @@
import json
import re
from io import BytesIO
from pathlib import Path
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from openai import AsyncOpenAI, OpenAI
from openai._legacy_response import LegacyAPIResponse
from openai.types import CreateEmbeddingResponse, Embedding
from openai.types.create_embedding_response import Usage
import pytest
import re
import litellm
from litellm import completion
@ -18,7 +19,6 @@ from litellm.llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfi
from tests._vcr_conftest_common import rewound_new_episodes_cassette
from tests.capturing_transport import CapturingTransport
_GATEWAY_EMBEDDING_RESPONSE: Final = CreateEmbeddingResponse(
object="list",
data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),),
@ -98,7 +98,7 @@ async def test_litellm_gateway_from_sdk():
openai_client = OpenAI(api_key="fake-key")
with patch.object(
openai_client.chat.completions.with_raw_response, "create", new=MagicMock()
openai_client, "post", new=MagicMock()
) as mock_call:
try:
completion(
@ -114,9 +114,9 @@ async def test_litellm_gateway_from_sdk():
mock_call.assert_called_once()
print("Call KWARGS - {}".format(mock_call.call_args.kwargs))
print("Call KWARGS - {}".format(mock_call.call_args.kwargs["body"]))
assert "hello" in mock_call.call_args.kwargs["extra_body"]
assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"]
@pytest.mark.parametrize("is_async", (False, True))
@ -264,7 +264,7 @@ async def test_litellm_gateway_from_sdk_rerank(is_async):
patch_target = client.post
with patch.object(client, "post", new=mock_method):
mock_response = MagicMock()
mock_response = MagicMock(spec=LegacyAPIResponse)
# Create a mock response similar to OpenAI's rerank response
mock_response.text = json.dumps(
@ -408,7 +408,7 @@ async def test_litellm_gateway_from_sdk_structured_output():
openai_client = OpenAI(api_key="fake-key")
with patch.object(
openai_client.chat.completions, "create", new=MagicMock()
openai_client, "post", new=MagicMock()
) as mock_call:
try:
litellm.completion(
@ -427,8 +427,8 @@ async def test_litellm_gateway_from_sdk_structured_output():
mock_call.assert_called_once()
print("Call KWARGS - {}".format(mock_call.call_args.kwargs))
json_schema = mock_call.call_args.kwargs["response_format"]
print("Call KWARGS - {}".format(mock_call.call_args.kwargs["body"]))
json_schema = mock_call.call_args.kwargs["body"]["response_format"]
assert "json_schema" in json_schema
@ -486,7 +486,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers():
openai_client = OpenAI(api_key="fake-key")
# Create mock response object
mock_response = MagicMock()
mock_response = MagicMock(spec=LegacyAPIResponse)
mock_response.headers = {"x-litellm-response-cost": "120"}
mock_response.parse.return_value = litellm.ModelResponse(
**{
@ -528,8 +528,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers():
)
with patch.object(
openai_client.chat.completions.with_raw_response,
"create",
openai_client, "post",
return_value=mock_response,
) as mock_call:
response = litellm.completion(
@ -564,7 +563,7 @@ async def test_litellm_gateway_image_generation_direct(is_async):
"created": 1,
"data": [{"url": "https://example.com/image.png"}],
}
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.parse.return_value = mock_openai_response
mock_raw_response.headers = {}

8
tests/unit/llms/nvidia_nim/test_nvidia_nim.py
View file

@ -94,7 +94,7 @@ def test_chat_completion_nvidia_nim_with_tools():
]
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
completion(
@ -116,7 +116,7 @@ def test_chat_completion_nvidia_nim_with_tools():
# Add assertions to check the request
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
print("request_body: ", request_body)
@ -208,7 +208,7 @@ def test_completion_nvidia_nim():
api_key="fake-api-key",
)
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
with patch.object(client, "post") as mock_client:
try:
completion(
model=model_name,
@ -226,7 +226,7 @@ def test_completion_nvidia_nim():
print(e)
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
assert request_body["messages"] == [
{

8
tests/unit/llms/openai/test_o_series_transformation.py
View file

@ -70,7 +70,7 @@ async def test_o1_handle_system_role(model):
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
await litellm.acompletion(
@ -83,7 +83,7 @@ async def test_o1_handle_system_role(model):
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
print("request_body: ", request_body)
@ -141,7 +141,7 @@ async def test_o1_max_completion_tokens(model: str):
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
await litellm.acompletion(
@ -154,7 +154,7 @@ async def test_o1_max_completion_tokens(model: str):
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
print("request_body: ", request_body)

93
tests/unit/llms/openai/test_openai.py
View file

@ -5,12 +5,13 @@ import os
from collections.abc import Mapping
from types import ModuleType
from typing import Final, Literal, Protocol
from unittest.mock import Mock, patch
from unittest.mock import MagicMock, Mock, patch
import httpx
import pytest
import respx
from openai import AsyncOpenAI, AzureOpenAI, OpenAI
from openai._legacy_response import LegacyAPIResponse
from openai._types import Response as SDKResponse
from pydantic import BaseModel, ConfigDict, JsonValue
@ -1669,7 +1670,7 @@ async def test_openai_prediction_param_mock():
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
await litellm.acompletion(
@ -1688,7 +1689,7 @@ async def test_openai_prediction_param_mock():
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
# Verify the request contains the prediction parameter
assert "prediction" in request_body
@ -1700,10 +1701,12 @@ async def test_openai_prediction_param_mock():
def test_openai_max_retries_0(mock_get_openai_client):
import litellm
mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {}
mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = (
ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}])
)
raw_response: Final = MagicMock(spec=LegacyAPIResponse)
raw_response.headers = {}
raw_response.parse.return_value = ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}])
client_stub: Final = MagicMock()
client_stub.post.return_value = raw_response
mock_get_openai_client.return_value = client_stub
litellm.set_verbose = True
response = litellm.completion(
model="gpt-4o-mini",
@ -1812,7 +1815,7 @@ async def test_openai_safety_identifier_parameter():
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
await litellm.acompletion(
@ -1825,7 +1828,7 @@ async def test_openai_safety_identifier_parameter():
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
# Verify the request contains the safety_identifier parameter
assert "safety_identifier" in request_body
@ -1841,7 +1844,7 @@ def test_openai_safety_identifier_parameter_sync():
client = OpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
litellm.completion(
@ -1854,7 +1857,7 @@ def test_openai_safety_identifier_parameter_sync():
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
# Verify the request contains the safety_identifier parameter
assert "safety_identifier" in request_body
@ -1871,7 +1874,7 @@ async def test_openai_service_tier_parameter():
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
await litellm.acompletion(
@ -1884,7 +1887,7 @@ async def test_openai_service_tier_parameter():
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
# Verify the request contains the service_tier parameter
assert "service_tier" in request_body, "service_tier should be in request body"
@ -1902,7 +1905,7 @@ def test_openai_service_tier_parameter_sync():
client = OpenAI(api_key="fake-api-key")
with patch.object(
client.chat.completions.with_raw_response, "create"
client, "post"
) as mock_client:
try:
litellm.completion(
@ -1915,7 +1918,7 @@ def test_openai_service_tier_parameter_sync():
print(f"Error: {e}")
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
# Verify the request contains the service_tier parameter
assert "service_tier" in request_body, "service_tier should be in request body"
@ -1947,3 +1950,63 @@ def test_responses_gpt54_with_xhigh_reasoning():
assert request_body["model"] == "openai/gpt-5.4"
assert request_body["reasoning"] == {"effort": "xhigh"}
_CHAT_RESPONSE_JSON: Final = {
"id": "chatcmpl-wire",
"object": "chat.completion",
"created": 1,
"model": "gpt-5.4",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
_WIRE_REQUEST: Final = {
"model": "gpt-5.4",
"messages": [
{"role": "system", "content": "sys"},
{
"role": "user",
"content": [
{"type": "text", "text": "what is in the image"},
{"type": "image_url", "image_url": {"url": "https://example.com/a.png", "detail": "low"}},
],
},
],
"tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}],
"tool_choice": "auto",
"response_format": {"type": "json_schema", "json_schema": {"name": "r", "schema": {"type": "object"}}},
"max_completion_tokens": 7,
"reasoning_effort": "none",
"n": 2,
"seed": 3,
}
@pytest.mark.parametrize("is_async", (False, True))
@respx.mock
async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_headers_unchanged(
is_async: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_RESPONSE_JSON)
)
kwargs: Final = {
**_WIRE_REQUEST,
"model": "openai/gpt-5.4",
"api_base": "https://api.openai.com/v1",
"api_key": "sk-test",
"num_retries": 0,
"extra_body": {"vendor_flag": True},
"extra_headers": {"x-caller": "unit"},
}
if is_async:
await litellm.acompletion(**kwargs)
else:
litellm.completion(**kwargs)
sent: Final = route.calls.last.request
assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True}
assert sent.headers["x-caller"] == "unit"
assert sent.headers["authorization"] == "Bearer sk-test"

22
tests/unit/llms/openai/test_openai_empty_response.py
View file

@ -2,13 +2,13 @@
Test for issue #17209: Clearer error when LLM endpoint returns empty response
"""
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import pytest
from openai._legacy_response import LegacyAPIResponse
from litellm.llms.openai.openai import OpenAIChatCompletion
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.openai import OpenAIChatCompletion
class TestEmptyResponseHandling:
@ -22,12 +22,12 @@ class TestEmptyResponseHandling:
openai_chat = OpenAIChatCompletion()
# Mock the raw response to return an empty string from parse()
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {}
mock_raw_response.parse.return_value = "" # Empty string response
mock_client = MagicMock()
mock_client.chat.completions.with_raw_response.create.return_value = (
mock_client.post.return_value = (
mock_raw_response
)
@ -48,12 +48,12 @@ class TestEmptyResponseHandling:
"""Test that None response also produces a clear error"""
openai_chat = OpenAIChatCompletion()
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {}
mock_raw_response.parse.return_value = None
mock_client = MagicMock()
mock_client.chat.completions.with_raw_response.create.return_value = (
mock_client.post.return_value = (
mock_raw_response
)
@ -75,12 +75,12 @@ class TestEmptyResponseHandling:
mock_response = MagicMock()
mock_response.model_dump.return_value = {"choices": []}
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {"x-request-id": "123"}
mock_raw_response.parse.return_value = mock_response
mock_client = MagicMock()
mock_client.chat.completions.with_raw_response.create.return_value = (
mock_client.post.return_value = (
mock_raw_response
)
@ -104,12 +104,12 @@ class TestEmptyResponseHandling:
# Create a mock response WITHOUT model_dump (like an AsyncStream/Iterator)
mock_stream = MagicMock(spec=[]) # spec=[] means no attributes
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {"x-request-id": "123"}
mock_raw_response.parse.return_value = mock_stream
mock_client = MagicMock()
mock_client.chat.completions.with_raw_response.create.return_value = (
mock_client.post.return_value = (
mock_raw_response
)

7
tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py
View file

@ -9,6 +9,7 @@ body. OpenAI/Anthropic reject unknown body params with HTTP 400.
from unittest.mock import MagicMock
from openai._legacy_response import LegacyAPIResponse
import litellm
from litellm.types.utils import all_litellm_params
@ -47,12 +48,12 @@ def test_completion_does_not_leak_flag_into_provider_request_body():
},
}
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {}
mock_raw_response.parse.return_value = mock_response
mock_client = MagicMock()
mock_client.chat.completions.with_raw_response.create.return_value = (
mock_client.post.return_value = (
mock_raw_response
)
@ -65,7 +66,7 @@ def test_completion_does_not_leak_flag_into_provider_request_body():
)
create_kwargs = (
mock_client.chat.completions.with_raw_response.create.call_args.kwargs
mock_client.post.call_args.kwargs
)
assert "use_chat_completions_api" not in create_kwargs
assert "use_chat_completions_api" not in (create_kwargs.get("extra_body") or {})

8
tests/unit/llms/perplexity/chat/test_transformation.py
View file

@ -4,6 +4,7 @@ import os
from unittest.mock import MagicMock, patch
import pytest
from openai._legacy_response import LegacyAPIResponse
import litellm
from litellm import completion
@ -92,7 +93,7 @@ class TestPerplexityReasoning:
pydantic_obj = ChatCompletion(**response_object)
def _return_pydantic_obj(*args, **kwargs):
new_response = MagicMock()
new_response = MagicMock(spec=LegacyAPIResponse)
new_response.headers = {"content-type": "application/json"}
new_response.parse.return_value = pydantic_obj
return new_response
@ -100,8 +101,7 @@ class TestPerplexityReasoning:
openai_client = OpenAI(api_key="fake-api-key")
with patch.object(
openai_client.chat.completions.with_raw_response,
"create",
openai_client, "post",
side_effect=_return_pydantic_obj,
) as mock_client:
response = completion(
@ -121,7 +121,7 @@ class TestPerplexityReasoning:
# Get the request data from the mock call
call_args = mock_client.call_args
request_data = call_args.kwargs
request_data = call_args.kwargs["body"]
# Verify reasoning_effort was included in the request
assert "reasoning_effort" in request_data

16
tests/unit/llms/volcengine/test_volcengine.py
View file

@ -2,6 +2,7 @@ import os
import sys
from unittest.mock import MagicMock, patch
from openai._legacy_response import LegacyAPIResponse
from pydantic import BaseModel
from litellm.llms.volcengine.chat.transformation import (
@ -117,7 +118,7 @@ class TestVolcEngineConfig:
client = OpenAI(api_key="test_api_key")
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {
"x-request-id": "123",
"openai-organization": "org-123",
@ -126,9 +127,7 @@ class TestVolcEngineConfig:
}
mock_raw_response.parse.return_value = ModelResponse()
with patch.object(
client.chat.completions.with_raw_response, "create", mock_raw_response
) as mock_create:
with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create:
completion(
model="volcengine/doubao-seed-1.6",
messages=[
@ -144,11 +143,6 @@ class TestVolcEngineConfig:
)
mock_create.assert_called_once()
print(mock_create.call_args.kwargs)
print(mock_create.call_args.kwargs["body"])
# Fixed: thinking disabled should appear in extra_body with original structure
assert (
"extra_body" in mock_create.call_args.kwargs
and "thinking" in mock_create.call_args.kwargs.get("extra_body", {})
and mock_create.call_args.kwargs.get("extra_body", {})["thinking"]
== {"type": "disabled"}
)
assert mock_create.call_args.kwargs["options"]["extra_json"]["thinking"] == {"type": "disabled"}

93
tests/unit/test_main.py
View file

@ -21,15 +21,22 @@ import httpx
import pytest
import respx
from openai import APITimeoutError, AsyncOpenAI
from openai._legacy_response import LegacyAPIResponse
from openai.types.chat.chat_completion import ChatCompletion
import litellm
from litellm import acompletion, completion
from litellm import acompletion_with_retries, aresponses_with_retries, completion_with_retries, responses_with_retries
from litellm import (
acompletion,
acompletion_with_retries,
aresponses_with_retries,
completion,
completion_with_retries,
responses_with_retries,
)
from litellm import main as litellm_main
from litellm.constants import CONTROL_OPTIONS_KEY
from litellm.caching.base_cache import BaseCache
from litellm.caching.caching import Cache
from litellm.constants import CONTROL_OPTIONS_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
@ -130,7 +137,7 @@ def test_completion_missing_role(openai_api_response):
client = OpenAI(api_key="test_api_key")
mock_raw_response = MagicMock()
mock_raw_response = MagicMock(spec=LegacyAPIResponse)
mock_raw_response.headers = {
"x-request-id": "123",
"openai-organization": "org-123",
@ -142,7 +149,7 @@ def test_completion_missing_role(openai_api_response):
print(f"openai_api_response: {openai_api_response}")
with patch.object(
client.chat.completions.with_raw_response, "create", MagicMock(return_value=mock_raw_response)
client, "post", MagicMock(return_value=mock_raw_response)
) as mock_create:
litellm.completion(
model="gpt-4o-mini",
@ -218,7 +225,7 @@ async def test_url_with_format_param_openai(model, sync_mode):
}
],
}
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
with patch.object(client, "post") as mock_client:
try:
if sync_mode:
response = completion(**args, client=client)
@ -232,7 +239,7 @@ async def test_url_with_format_param_openai(model, sync_mode):
print(mock_client.call_args.kwargs)
json_str = json.dumps(mock_client.call_args.kwargs)
json_str = json.dumps(mock_client.call_args.kwargs["body"])
assert "format" not in json_str
@ -3005,9 +3012,10 @@ def test_stream_chunk_builder_thinking_blocks():
assert response.choices[0].message.thinking_blocks is not None
from litellm.llms.openai.openai import OpenAIChatCompletion
import traceback
from litellm.llms.openai.openai import OpenAIChatCompletion
user_message = "Write a short poem about the sky"
messages = [{"content": user_message, "role": "user"}]
@ -3777,7 +3785,7 @@ def test_completion_forwards_store_and_prompt_cache_key_to_openai():
client = OpenAI(api_key="fake-api-key")
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
with patch.object(client, "post") as mock_client:
try:
litellm.completion(
model="openai/gpt-4o",
@ -3790,7 +3798,7 @@ def test_completion_forwards_store_and_prompt_cache_key_to_openai():
print(e)
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
assert request_body["store"] is False
assert request_body["prompt_cache_key"] == "test-cache-key"
@ -3805,7 +3813,7 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai():
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
with patch.object(client, "post") as mock_client:
try:
await litellm.acompletion(
model="openai/gpt-4o",
@ -3818,7 +3826,7 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai():
print(e)
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
assert request_body["store"] is False
assert request_body["prompt_cache_key"] == "test-cache-key"
@ -3832,7 +3840,7 @@ def test_completion_omits_store_and_prompt_cache_key_when_not_passed():
client = OpenAI(api_key="fake-api-key")
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
with patch.object(client, "post") as mock_client:
try:
litellm.completion(
model="openai/gpt-4o",
@ -3843,7 +3851,7 @@ def test_completion_omits_store_and_prompt_cache_key_when_not_passed():
print(e)
mock_client.assert_called_once()
request_body = mock_client.call_args.kwargs
request_body = mock_client.call_args.kwargs["body"]
assert "store" not in request_body
assert "prompt_cache_key" not in request_body
@ -4038,8 +4046,7 @@ def test_openai_model_without_a_provider_still_routes_to_openai():
from openai import OpenAI
client = OpenAI(api_key="fake-key")
raw_response = client.chat.completions.with_raw_response
with patch.object(raw_response, "create") as mock_create, contextlib.suppress(Exception):
with patch.object(client, "post") as mock_create, contextlib.suppress(Exception):
litellm.completion(
model="gpt-4o",
messages=[{"role": "user", "content": "hello"}],
@ -4050,7 +4057,7 @@ def test_openai_model_without_a_provider_still_routes_to_openai():
def _openai_chat_create_kwargs(client, **completion_kwargs):
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
with patch.object(client, "post") as mock_client:
with contextlib.suppress(Exception):
litellm.completion(
messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}],
@ -4075,7 +4082,7 @@ def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6
from openai import OpenAI
client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1")
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1")["body"]
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
@ -4087,7 +4094,7 @@ def test_completion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6
from openai import OpenAI
client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1")
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1")["body"]
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
@ -4100,7 +4107,7 @@ async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_
from openai import AsyncOpenAI
client = AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
with patch.object(client.chat.completions.with_raw_response, "create") as mock_create:
with patch.object(client, "post") as mock_create:
with contextlib.suppress(Exception):
await litellm.acompletion(
model="gpt-5.6",
@ -4111,7 +4118,7 @@ async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_
)
mock_create.assert_called_once()
request_body = mock_create.call_args.kwargs
request_body = mock_create.call_args.kwargs["body"]
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
@ -4123,12 +4130,12 @@ def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6()
from openai import OpenAI
client = OpenAI(api_key="fake-api-key")
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6")
request = _openai_chat_create_kwargs(client, model="gpt-5.6")
assert request_body["messages"][0]["content"] == [
assert request["body"]["messages"][0]["content"] == [
{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}
]
assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "implicit"}
assert request["options"]["extra_json"]["prompt_cache_options"] == {"mode": "implicit"}
_SUBSCRIPTION_OAUTH_CREDENTIAL = "Bearer sk-ant-oat01-fake-subscription-token-for-testing-0123456789"
@ -5141,6 +5148,7 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None:
def test_acompletion_params():
import inspect
from litellm.types.completion import CompletionRequest
acompletion_params_odict = inspect.signature(acompletion).parameters
@ -5171,7 +5179,7 @@ def test_acompletion_params():
def _openai_mock_response(*args: object, **kwargs: object) -> MagicMock:
new_response: Final = MagicMock()
new_response: Final = MagicMock(spec=LegacyAPIResponse)
new_response.headers = {"hello": "world"}
response_object: Final = {
"id": "chatcmpl-123",
@ -5206,7 +5214,7 @@ def test_null_role_response():
openai_client = openai.OpenAI()
with patch.object(
openai_client.chat.completions, "create", side_effect=_openai_mock_response
openai_client, "post", side_effect=_openai_mock_response
) as mock_response:
response = litellm.completion(
model="gpt-3.5-turbo",
@ -5296,7 +5304,7 @@ def test_completion_perplexity_api():
pydantic_obj = ChatCompletion(**response_object)
def _return_pydantic_obj(*args, **kwargs):
new_response = MagicMock()
new_response = MagicMock(spec=LegacyAPIResponse)
new_response.headers = {"hello": "world"}
new_response.parse.return_value = pydantic_obj
@ -5305,8 +5313,7 @@ def test_completion_perplexity_api():
openai_client = OpenAI()
with patch.object(
openai_client.chat.completions.with_raw_response,
"create",
openai_client, "post",
side_effect=_return_pydantic_obj,
) as mock_client:
# litellm.set_verbose= True
@ -5350,7 +5357,7 @@ async def test_openai_compatible_custom_api_base(provider):
openai_client = OpenAI(api_key="fake-key")
with patch.object(
openai_client.chat.completions, "create", new=MagicMock()
openai_client, "post", new=MagicMock()
) as mock_call:
try:
completion(
@ -5368,7 +5375,7 @@ async def test_openai_compatible_custom_api_base(provider):
print("Call KWARGS - {}".format(mock_call.call_args.kwargs))
assert "hello" in mock_call.call_args.kwargs["extra_body"]
assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"]
@pytest.mark.parametrize(
@ -5401,7 +5408,7 @@ async def test_openai_compatible_custom_api_video(provider):
openai_client = OpenAI(api_key="fake-key")
with patch.object(
openai_client.chat.completions, "create", new=MagicMock()
openai_client, "post", new=MagicMock()
) as mock_call:
try:
completion(
@ -7264,7 +7271,7 @@ def _openai_hallucinated_tool_call_mock_response(
*args: object,
**kwargs: object,
) -> MagicMock:
new_response: Final = MagicMock()
new_response: Final = MagicMock(spec=LegacyAPIResponse)
new_response.headers = {"hello": "world"}
response_object: Final = {
"id": "chatcmpl-123",
@ -7322,8 +7329,8 @@ def test_openai_hallucinated_tool_call():
openai_client = openai.OpenAI()
with patch.object(
openai_client.chat.completions,
"create",
openai_client,
"post",
side_effect=_openai_hallucinated_tool_call_mock_response,
) as mock_response:
response = litellm.completion(
@ -7445,7 +7452,7 @@ def test_completion_novita_ai():
openai_client = OpenAI(api_key="fake-key")
with patch.object(
openai_client.chat.completions.with_raw_response, "create"
openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)
) as mock_call:
mock_call.return_value.headers = {}
mock_call.return_value.parse.return_value = litellm.ModelResponse(
@ -7464,11 +7471,11 @@ def test_completion_novita_ai():
# Verify model is passed correctly
assert (
mock_call.call_args.kwargs["model"]
mock_call.call_args.kwargs["body"]["model"]
== "meta-llama/llama-3.3-70b-instruct"
)
# Verify messages are passed correctly
assert mock_call.call_args.kwargs["messages"] == messages
assert mock_call.call_args.kwargs["body"]["messages"] == messages
except Exception as e:
pytest.fail(f"Error occurred: {e}")
@ -7491,8 +7498,8 @@ def test_completion_novita_ai_dynamic_params(api_key):
openai_client = OpenAI(api_key="fake-key")
with patch.object(
openai_client.chat.completions,
"create",
openai_client,
"post",
side_effect=Exception("Invalid API key"),
) as mock_call:
with pytest.raises(Exception, match="Invalid API key") as exc_info:
@ -7524,7 +7531,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features):
monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features)
with patch.object(
client.chat.completions.with_raw_response, "create", return_value=MagicMock()
client, "post", return_value=MagicMock()
) as mock_completion:
try:
resp = litellm.completion(
@ -7538,11 +7545,11 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features):
mock_completion.assert_called_once()
if enable_preview_features:
assert mock_completion.call_args.kwargs["metadata"] == {
assert mock_completion.call_args.kwargs["body"]["metadata"] == {
"my-test-key": "my-test-value"
}
else:
assert "metadata" not in mock_completion.call_args.kwargs
assert "metadata" not in mock_completion.call_args.kwargs["body"]
AZURE_TTS_BASE: Final = "https://tts.example.azure.com"