mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(utils): strip stream_options.include_usage for non-streaming requests
include_usage in stream_options is only meaningful for streaming requests. OpenAI ignores it otherwise, but OpenAI-compatible backends such as vLLM reject the call with a 400. Strip just that key when stream is not True; other stream_options keys are preserved. Fixes #29431 Signed-off-by: Chenglun Hu <chenglunhu@gmail.com>
This commit is contained in:
parent
dddbaa803d
commit
eba074cceb
2 changed files with 41 additions and 0 deletions
|
|
@ -5043,6 +5043,20 @@ def get_optional_params(
|
|||
model=model,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
# include_usage in stream_options is only meaningful for streaming requests;
|
||||
# OpenAI-compatible backends (e.g. vLLM) reject it with a 400 when stream is
|
||||
# not True. Strip just that key for non-streaming requests (other keys such as
|
||||
# include_obfuscation stay). Fixes #29431.
|
||||
if stream is not True:
|
||||
_params: Final = cast(dict[str, object], non_default_params) # cast-ok: bare dict from the preprocessor
|
||||
_stream_options: Final = _params.get("stream_options")
|
||||
if isinstance(_stream_options, dict) and "include_usage" in _stream_options:
|
||||
_options: Final = cast(dict[str, object], _stream_options) # cast-ok: caller-supplied stream_options
|
||||
_without_usage: Final = {k: v for k, v in _options.items() if k != "include_usage"}
|
||||
if _without_usage:
|
||||
_params["stream_options"] = _without_usage
|
||||
else:
|
||||
_params.pop("stream_options", None)
|
||||
optional_params = pre_process_optional_params(
|
||||
passed_params=passed_params,
|
||||
non_default_params=non_default_params,
|
||||
|
|
|
|||
|
|
@ -10840,3 +10840,30 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_
|
|||
)
|
||||
|
||||
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stream,passed,expected",
|
||||
[
|
||||
(False, {"include_usage": True}, None),
|
||||
(None, {"include_usage": True}, None),
|
||||
(True, {"include_usage": True}, {"include_usage": True}),
|
||||
(False, {"include_usage": True, "include_obfuscation": True}, {"include_obfuscation": True}),
|
||||
(False, {"include_obfuscation": True}, {"include_obfuscation": True}),
|
||||
],
|
||||
)
|
||||
def test_get_optional_params_strips_include_usage_when_not_streaming(stream, passed, expected):
|
||||
"""`stream_options.include_usage` only means anything for a streaming request.
|
||||
OpenAI ignores it otherwise, but OpenAI-compatible backends such as vLLM reject
|
||||
the call with a 400 (#29431). Only that key is stripped; the rest are kept.
|
||||
"""
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
optional_params = get_optional_params(
|
||||
model="gpt-4o",
|
||||
custom_llm_provider="openai",
|
||||
stream=stream,
|
||||
stream_options=dict(passed),
|
||||
)
|
||||
|
||||
assert optional_params.get("stream_options") == expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue