scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge pull request #34549 from BerriAI/litellm_fix_stream_options_responses_api

Browse source 
fix(responses): strip include_usage from stream_options instead of dropping the param
This commit is contained in:
Mateo Wang 2026-07-24 17:12:26 -07:00 • committed by GitHub
commit cfb7edb54e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 217 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -35,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
record_output_item_chunk,
record_output_text_chunk,
)
from litellm.responses.utils import normalize_responses_api_stream_options
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
@ -320,6 +321,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
responses_api_request["tool_choice"] = ( # type: ignore[assignment]
self._normalize_tool_choice_for_responses_api(value)
)
elif key == "stream_options":
stream_options = normalize_responses_api_stream_options(value)
if stream_options is not None:
responses_api_request["stream_options"] = stream_options
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key == "previous_response_id":
@ -360,8 +365,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
continue
if key == "instructions" and instructions:
request_data["instructions"] = instructions
elif key == "stream_options" and isinstance(value, dict):
request_data["stream_options"] = value.get("include_obfuscation")
elif key == "user" and isinstance(value, str):
# OpenAI API requires user param to be max 64 chars - truncate if longer
if len(value) <= 64:

25
litellm/responses/utils.py
View file

@ -5,6 +5,7 @@ from typing import (
Dict,
Iterable,
List,
Mapping,
Optional,
Type,
Union,
@ -24,6 +25,7 @@ from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamOptions,
ResponseText,
)
from litellm.types.responses.main import DecodedResponseId
@ -35,6 +37,17 @@ from litellm.types.utils import (
)
def normalize_responses_api_stream_options(
stream_options: object,
) -> ResponsesAPIStreamOptions | None:
if not isinstance(stream_options, Mapping):
return None
include_obfuscation = stream_options.get("include_obfuscation")
if not isinstance(include_obfuscation, bool):
return None
return ResponsesAPIStreamOptions(include_obfuscation=include_obfuscation)
class ResponsesAPIRequestUtils:
"""Helper utils for constructing ResponseAPI requests"""
@ -156,15 +169,19 @@ class ResponsesAPIRequestUtils:
drop_params=should_drop_params,
)
stream_options = normalize_responses_api_stream_options(mapped_params.get("stream_options"))
params_with_normalized_stream_options = {
**{key: value for key, value in mapped_params.items() if key != "stream_options"},
**({} if stream_options is None else {"stream_options": stream_options}),
}
# add any allowed_openai_params to the mapped_params
mapped_params = _apply_openai_param_overrides(
optional_params=mapped_params,
return _apply_openai_param_overrides(
optional_params=params_with_normalized_stream_options,
non_default_params=non_default_params,
allowed_openai_params=allowed_openai_params or [],
)
return mapped_params
@staticmethod
def get_requested_response_api_optional_param(
params: Dict[str, Any],

6
litellm/types/llms/openai.py
View file

@ -1145,6 +1145,10 @@ class ContextManagementEntry(TypedDict, total=False):
"""Token threshold at which compaction is triggered for this entry. Minimum 1000."""
class ResponsesAPIStreamOptions(TypedDict, total=False):
include_obfuscation: bool
class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
"""TypedDict for Optional parameters supported by the responses API."""
@ -1171,7 +1175,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
max_tool_calls: Optional[int]
prompt_cache_key: Optional[str]
prompt_cache_retention: Optional[str]
stream_options: Optional[dict]
stream_options: Optional[ResponsesAPIStreamOptions]
top_logprobs: Optional[int]
partial_images: Optional[int] # Number of partial images to generate (1-3) for streaming image generation
context_management: Optional[List[ContextManagementEntry]]

74
tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
View file

@ -2853,3 +2853,77 @@ def test_streaming_function_call_tool_id_for_degenerate_call_id():
assert stream_tool_id("fc_unique_abc123", "call_0") == "fc_unique_abc123"
assert stream_tool_id("fc_2", "call_tokyo") == "call_tokyo"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"stream_options,expected_wire_stream_options",
[
({"include_usage": True, "include_obfuscation": False}, {"include_obfuscation": False}),
({"include_usage": True}, None),
],
)
async def test_acompletion_bridge_normalizes_stream_options_on_the_wire(
stream_options, expected_wire_stream_options
):
"""include_usage must be stripped from the /v1/responses body; include_obfuscation must survive as a dict."""
from unittest.mock import AsyncMock
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
responses_payload = {
"id": "resp_bridge_stream_options",
"object": "response",
"created_at": 1734366691,
"status": "completed",
"model": "gpt-5.5",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
}
],
"parallel_tool_calls": True,
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
"error": None,
"incomplete_details": None,
"instructions": None,
"metadata": None,
"temperature": None,
"tool_choice": "auto",
"tools": [],
"top_p": None,
"max_output_tokens": None,
"previous_response_id": None,
"reasoning": None,
"truncation": None,
"user": None,
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = json.dumps(responses_payload)
mock_response.headers = httpx.Headers({})
mock_response.json.return_value = responses_payload
with patch.object(AsyncHTTPHandler, "post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
await litellm.acompletion(
model="openai/responses/gpt-5.5",
messages=[{"role": "user", "content": "hi"}],
api_key="fake-api-key",
stream_options=stream_options,
)
mock_post.assert_called_once()
post_kwargs = mock_post.call_args.kwargs
request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"])
if expected_wire_stream_options is None:
assert "stream_options" not in request_body
else:
assert request_body["stream_options"] == expected_wire_stream_options

64
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -2,6 +2,7 @@ import json
import os
import sys
import httpx
import pytest
from fastapi.testclient import TestClient
@ -9,6 +10,7 @@ sys.path.insert(0, os.path.abspath("../../../../.."))
from unittest.mock import AsyncMock, MagicMock, patch
import litellm
from litellm.anthropic_interface import messages
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import Delta, ModelResponse, StreamingChoices
@ -37,6 +39,68 @@ def test_anthropic_experimental_pass_through_messages_handler():
assert mock_responses.call_args.kwargs["api_key"] == "test-api-key"
@pytest.mark.asyncio
async def test_openai_model_does_not_forward_stream_options_to_responses_api():
"""
Regression test for LIT-4779. `always_include_stream_usage` injects
stream_options={'include_usage': True} into every streaming request, but OpenAI
models on /v1/messages go to the Responses API, which 400s on that param.
"""
responses_payload = {
"id": "resp_stream_options",
"object": "response",
"created_at": 1734366691,
"status": "completed",
"model": "gpt-5.5",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
}
],
"parallel_tool_calls": True,
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
"error": None,
"incomplete_details": None,
"instructions": None,
"metadata": None,
"temperature": None,
"tool_choice": "auto",
"tools": [],
"top_p": None,
"max_output_tokens": None,
"previous_response_id": None,
"reasoning": None,
"truncation": None,
"user": None,
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = json.dumps(responses_payload)
mock_response.headers = httpx.Headers({})
mock_response.json.return_value = responses_payload
with patch.object(AsyncHTTPHandler, "post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
await litellm.anthropic.messages.acreate(
max_tokens=100,
messages=[{"role": "user", "content": "Hello, how are you?"}],
model="openai/gpt-5.5",
api_key="test-api-key",
stream_options={"include_usage": True},
)
mock_post.assert_called_once()
post_kwargs = mock_post.call_args.kwargs
request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"])
assert "stream_options" not in request_body
def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values():
"""
Test that api key, api base, and extra kwargs are forwarded to litellm.completion for Azure models.

48
tests/test_litellm/responses/test_responses_api_request_body.py
View file

@ -198,6 +198,54 @@ async def test_aresponses_azure_shell_tool_400_maps_to_bad_request_error():
assert "not supported" in str(excinfo.value).lower()
@pytest.mark.asyncio
async def test_aresponses_drops_stream_options():
"""The Responses API rejects include_usage, so include_usage-only stream_options must never reach the wire."""
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock,
) as mock_post:
mock_post.return_value = MockResponse(
_minimal_responses_api_payload("resp_stream_options_test", "gpt-5.5"), 200
)
await litellm.aresponses(
model="openai/gpt-5.5",
api_key="fake-api-key",
input="hi",
stream_options={"include_usage": True},
)
mock_post.assert_called_once()
post_kwargs = mock_post.call_args.kwargs
request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"])
assert "stream_options" not in request_body
@pytest.mark.asyncio
async def test_aresponses_keeps_include_obfuscation_in_stream_options():
"""include_obfuscation is a valid Responses API stream option and must survive the include_usage strip."""
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock,
) as mock_post:
mock_post.return_value = MockResponse(
_minimal_responses_api_payload("resp_stream_options_obfuscation", "gpt-5.5"), 200
)
await litellm.aresponses(
model="openai/gpt-5.5",
api_key="fake-api-key",
input="hi",
stream_options={"include_usage": True, "include_obfuscation": False},
)
mock_post.assert_called_once()
post_kwargs = mock_post.call_args.kwargs
request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"])
assert request_body["stream_options"] == {"include_obfuscation": False}
@pytest.mark.asyncio
async def test_aresponses_request_level_drop_params_drops_bedrock_mantle_service_tier(
monkeypatch,