mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(headroom): drop stream_options when converting a stream to non-streaming
When the Headroom CCR guardrail converts a streaming /chat/completions call to non-streaming (async_pre_call_deployment_hook sets stream=False), it left the client's stream_options in the request. Strict providers reject stream_options without stream=true — DeepSeek returns HTTP 400 'stream_options should be set along with stream = true' — so streaming requests failed whenever Headroom actually compressed and triggered CCR. Pop stream_options from the converted request alongside stream=False. Fixes #40068
This commit is contained in:
parent
2ea43214c9
commit
f7bb8c836a
2 changed files with 25 additions and 1 deletions
|
|
@ -890,11 +890,16 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
return base_result
|
||||
if not has_headroom_retrieve_tool(effective.get("tools")):
|
||||
return base_result
|
||||
return { # mutable-ok: the hook contract is a plain dict the router merges into the request kwargs
|
||||
converted: Final = { # mutable-ok: the hook contract is a plain dict the router merges into the request kwargs
|
||||
**effective,
|
||||
"stream": False,
|
||||
HEADROOM_CONVERTED_STREAM_KEY: True,
|
||||
}
|
||||
# stream_options is only valid alongside stream=true; leaving it after the
|
||||
# internal conversion makes strict providers (e.g. DeepSeek) reject the
|
||||
# request with "stream_options should be set along with stream = true".
|
||||
converted.pop("stream_options", None)
|
||||
return converted
|
||||
|
||||
async def async_should_run_agentic_loop(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2171,6 +2171,25 @@ async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_comple
|
|||
assert kwargs["stream"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_deployment_hook_drops_stream_options_when_converting(guardrail: HeadroomGuardrail):
|
||||
"""stream_options is only valid with stream=true. When the CCR conversion flips stream to
|
||||
False it must also drop stream_options, or strict providers (e.g. DeepSeek) 400 with
|
||||
'stream_options should be set along with stream = true'."""
|
||||
kwargs = {
|
||||
"model": "gpt-4o",
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"tools": [_retrieve_tool_definition()],
|
||||
}
|
||||
|
||||
result = await guardrail.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acompletion)
|
||||
|
||||
assert result is not None
|
||||
assert result["stream"] is False
|
||||
assert "stream_options" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_deployment_hook_leaves_background_streams_alone(guardrail: HeadroomGuardrail):
|
||||
kwargs = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue