mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-31 06:56:59 +00:00
d06b889d41
* fix(vertex_ai): split Gemini SSE lines on CR/LF only so U+2028 in text keeps the stream The Gemini and Vertex AI streaming path read the SSE body with httpx iter_lines()/aiter_lines(). Those follow str.splitlines(), which also breaks on U+0085, U+2028, U+2029, \x0b, \x0c and \x1c-\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a data: line whose text holds one of them was cut in two. The tail is not a data: line and the head never parses, and the accumulate-and-retry fallback then carries the unparsed prefix forward, so every later event is appended to it and the stream ends with no more content and no usage. SSE lines end only at CRLF, LF or CR. The new iter_sse_lines/aiter_sse_lines split decoded text on exactly those, treating a CR at the end of one chunk and an LF at the start of the next as one break. The first request and every long-decode hop now read through them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(streaming): keep merged think tags balanced and split shared SSE lines only on CR/LF Close an open <think> block on the first content or tool-call delta and at stream end, reopen it when reasoning returns, and route the shared chat completions stream through the SSE line splitter so U+2028 and U+2029 inside JSON no longer drop chunks. * fix(streaming): exact-match the SSE [DONE] marker in BaseModelResponseIterator * fix(streaming): split every SSE reader on CR/LF only and keep the logged think block closed Switch the remaining httpx stream readers (Anthropic chat, Google GenAI, Interactions, Codestral, openai_like, Predibase, LangGraph, Vertex Agent Engine, A2A providers, Azure AI agents, sandboxes, the A2A endpoint, and others) from iter_lines/aiter_lines to iter_sse_lines/aiter_sse_lines, so a raw U+2028 or U+2029 inside a JSON string no longer splits an event. Keep the chunk that closes a think block in the logged and cached response when a stream ends without a finish_reason, close the block before a legacy function_call, and make sync Codestral FIM streaming send stream=true and use the text-completion-codestral stream parser. Test fakes that only mocked iter_lines now stream real httpx bytes and assert the content they feed in. * fix(streaming): split buffered SSE parsers only on CR and LF, shard the codestral tests Buffered SSE and JSONL parsers used str.splitlines(), which also breaks on U+2028, U+2029 and U+0085 that JSON allows raw inside strings. They now use split_sse_lines from sse_lines.py, which also gains a str.split fast path. Adds tests/unit/llms/codestral to the unit workflow shard. * fix(streaming): keep U+2028 in restamped Anthropic frames and fold CRLF before splitting SSE lines * test: type the new stream tests, drop a comment, and benchmark the SSE line splitter * test: type the iterator class parameters in the line-separator stream tests * fix(responses): stream tool call arguments opened in a text chunk * fix(responses): hold a tool call that shares a chunk with text until the text ends * test: type the a2a SSE handler helper and the sandbox fake responses --------- Co-authored-by: Tim Vink <vinktim@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: michelligabriele <gabriele.michelli@icloud.com>
63 lines
2 KiB
Python
63 lines
2 KiB
Python
"""
|
|
Test for Groq streaming ASCII encoding issue fix.
|
|
|
|
This test verifies that the OpenAI-like handler correctly handles
|
|
UTF-8 encoded content in streaming responses, specifically fixing
|
|
the ASCII encoding error described in issue #12660.
|
|
"""
|
|
|
|
import json
|
|
from typing import Final
|
|
from unittest.mock import Mock
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
|
from litellm.llms.openai_like.chat.handler import make_call, make_sync_call
|
|
|
|
_UNICODE_TEXTS: Final = (
|
|
"The symbol µ represents micro",
|
|
"Testing © character",
|
|
"Testing ™ character",
|
|
"Testing € character",
|
|
"Testing 北京 character",
|
|
"Testing 🚀 character",
|
|
"Testing Ñoño character",
|
|
)
|
|
|
|
|
|
def _utf8_stream_transport(text: str) -> httpx.MockTransport:
|
|
chunk: Final = {"choices": [{"index": 0, "delta": {"content": text}}]}
|
|
body: Final = f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n".encode()
|
|
return httpx.MockTransport(lambda request: httpx.Response(200, stream=httpx.ByteStream(body)))
|
|
|
|
|
|
def _call_args() -> dict[str, object]:
|
|
return {
|
|
"api_base": "https://test.com/v1/chat/completions",
|
|
"headers": {"Authorization": "Bearer test"},
|
|
"data": '{"model": "test", "messages": []}',
|
|
"model": "test-model",
|
|
"messages": [],
|
|
"logging_obj": Mock(),
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("text", _UNICODE_TEXTS)
|
|
def test_utf8_streaming_sync(text: str) -> None:
|
|
completion_stream: Final = make_sync_call(
|
|
client=HTTPHandler(client=httpx.Client(transport=_utf8_stream_transport(text))), **_call_args()
|
|
)
|
|
|
|
assert "".join(chunk["text"] for chunk in completion_stream) == text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("text", _UNICODE_TEXTS)
|
|
async def test_utf8_streaming_async(text: str) -> None:
|
|
completion_stream: Final = await make_call(
|
|
client=AsyncHTTPHandler(transport=_utf8_stream_transport(text)), **_call_args()
|
|
)
|
|
|
|
assert "".join([chunk["text"] async for chunk in completion_stream]) == text
|