mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(responses_id_security): handle a non-streaming response in the streaming iterator hook
async_post_call_streaming_iterator_hook did 'async for chunk in response' unconditionally. A streaming chat request can still hand it a single, non-streaming ModelResponse — e.g. when an agentic loop (websearch_interception) resolves the turn without streaming. That object has no __aiter__, so the hook raised TypeError. Wrap a single non-streaming response as a one-item async stream (_single_chunk) and iterate only when the response is actually async iterable. Fixes #41411
This commit is contained in:
parent
2ea43214c9
commit
fbe92192fa
2 changed files with 49 additions and 1 deletions
|
|
@ -104,6 +104,12 @@ def _is_responses_api_create_route(request_route: str | None) -> bool:
|
|||
return canonical in _RESPONSES_API_CREATE_ROUTES
|
||||
|
||||
|
||||
async def _single_chunk(response: object) -> AsyncGenerator[object, None]:
|
||||
"""Wrap one non-streaming response as a one-item async stream so the
|
||||
streaming iterator hook can treat it uniformly."""
|
||||
yield response
|
||||
|
||||
|
||||
class ResponsesIDSecurity(CustomLogger):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -333,7 +339,13 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
# Create a request-scoped cache for consistent encryption across streaming chunks.
|
||||
request_encryption_cache: Final[dict[str, str]] = {}
|
||||
|
||||
async for chunk in response:
|
||||
# A streaming request can still yield a single non-streaming ModelResponse
|
||||
# here — e.g. when an agentic loop (websearch_interception) resolves the
|
||||
# turn without streaming. Such an object is not async-iterable, so collect
|
||||
# the chunks from the real stream, or fall back to the one response.
|
||||
chunks = response if hasattr(response, "__aiter__") else _single_chunk(response)
|
||||
|
||||
async for chunk in chunks:
|
||||
if (
|
||||
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
|
||||
and _is_responses_api_create_route(user_api_key_dict.request_route)
|
||||
|
|
|
|||
|
|
@ -1094,3 +1094,39 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization:
|
|||
|
||||
assert result["response_id"] == "resp_strangerownprovideridcccccccc"
|
||||
assert result["response_id"] != victim_provider_id
|
||||
|
||||
|
||||
class TestStreamingIteratorNonStreamingResponse:
|
||||
"""A streaming chat request can still hand this hook a single, non-streaming
|
||||
ModelResponse — e.g. when an agentic loop (websearch_interception) resolves the
|
||||
turn without streaming. That object has no __aiter__, so `async for` used to
|
||||
raise `TypeError: 'async for' requires ... got ModelResponse` (#41411). The hook
|
||||
must pass such a response through as one item."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streaming_response_passed_through(self, monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij")
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
response = ModelResponse(
|
||||
choices=[Choices(index=0, message=Message(role="assistant", content="hi"))]
|
||||
)
|
||||
assert not hasattr(response, "__aiter__")
|
||||
|
||||
mock_auth = MagicMock()
|
||||
mock_auth.user_id = "user-a"
|
||||
mock_auth.team_id = "team-a"
|
||||
mock_auth.request_route = "/chat/completions"
|
||||
|
||||
hook = ResponsesIDSecurity()
|
||||
collected = [
|
||||
out
|
||||
async for out in hook.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=mock_auth,
|
||||
response=response,
|
||||
request_data={},
|
||||
)
|
||||
]
|
||||
|
||||
assert len(collected) == 1
|
||||
assert collected[0] is response
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue