This commit is contained in:
Tim Vink 2026-10-27 02:24:38 +00:00 • committed by GitHub
commit e787a9f4c4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 164 additions and 11 deletions

View file

@ -0,0 +1,56 @@
"""
Split a decoded Server-Sent Events body into lines the way the SSE spec does.
httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028,
U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a
`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR
(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream)
"""
import io
import re
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator
from typing import Final
_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
class _SSELineSplitter:
def __init__(self) -> None:
self._partial_line: io.StringIO = io.StringIO()
self._after_cr: bool = False
def feed(self, text: str) -> tuple[str, ...]:
if not text:
return ()
fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text
self._after_cr = fresh.endswith("\r")
*ended, tail = _SSE_LINE_BREAK.split(fresh)
if not ended:
_ = self._partial_line.write(tail)
return ()
first: Final = self._partial_line.getvalue() + ended[0]
self._partial_line = io.StringIO()
_ = self._partial_line.write(tail)
return (first, *ended[1:])
def flush(self) -> tuple[str, ...]:
rest: Final = self._partial_line.getvalue()
self._partial_line = io.StringIO()
return (rest,) if rest else ()
def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]:
splitter: Final = _SSELineSplitter()
for text in text_chunks:
yield from splitter.feed(text)
yield from splitter.flush()
async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]:
splitter: Final = _SSELineSplitter()
async for text in text_chunks:
for line in splitter.feed(text):
yield line
for line in splitter.flush():
yield line

View file

@ -41,6 +41,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
get_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
@ -3241,7 +3242,7 @@ async def make_call(
raise
completion_stream: Final = ModelResponseIterator(
streaming_response=response.aiter_lines(),
streaming_response=aiter_sse_lines(response.aiter_text()),
sync_stream=False,
logging_obj=logging_obj,
response_headers=response.headers,
@ -3312,7 +3313,7 @@ def make_sync_call(
raise
completion_stream: Final = ModelResponseIterator(
streaming_response=response.iter_lines(),
streaming_response=iter_sse_lines(response.iter_text()),
sync_stream=True,
logging_obj=logging_obj,
response_headers=response.headers,
@ -4190,7 +4191,7 @@ class ModelResponseIterator:
log_failed_hop(e)
return self._truncated_stream_end(token)
self._start_hop(hop_response, token)
self.streaming_response = hop_response.aiter_lines()
self.streaming_response = aiter_sse_lines(hop_response.aiter_text())
self.async_response_iterator = self.streaming_response
return None
@ -4204,7 +4205,7 @@ class ModelResponseIterator:
log_failed_hop(e)
return self._truncated_stream_end(token)
self._start_hop(hop_response, token)
self.streaming_response = hop_response.iter_lines()
self.streaming_response = iter_sse_lines(hop_response.iter_text())
self.response_iterator = self.streaming_response
return None

View file

@ -7704,7 +7704,7 @@ def gemini_mock_post_streaming(url, **kwargs):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.headers = {"Content-Type": "text/event-stream"}
mock_response.iter_lines = MagicMock(return_value=stream_response())
mock_response.iter_text = MagicMock(return_value=stream_response())
return mock_response

View file

@ -0,0 +1,38 @@
from collections.abc import AsyncIterator, Sequence
from typing import Final
import pytest
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
CASES: Final = (
pytest.param(("data: a
b\r\n\r\n",), ("data: a
b", ""), id="u2028-stays-inside-the-line"),
pytest.param(
("data: \u0085
\x0b\x0c\x1c\x1d\x1e\n",),
("data: \u0085
\x0b\x0c\x1c\x1d\x1e",),
id="no-splitlines-only-separator-breaks-a-line",
),
pytest.param(("a\r\nb\nc\rd\n",), ("a", "b", "c", "d"), id="crlf-lf-and-cr-each-end-a-line"),
pytest.param(("a\r", "\nb\n"), ("a", "b"), id="crlf-split-across-chunks-is-one-break"),
pytest.param(("a\r", "\n\r\n"), ("a", ""), id="crlf-split-then-blank-line"),
pytest.param(("a\r", "", "\nb\n"), ("a", "b"), id="empty-chunk-between-cr-and-lf"),
pytest.param(("a\r", "\r"), ("a", ""), id="cr-then-cr-is-two-breaks"),
pytest.param(("da", "ta: x", "y\n"), ("data: xy",), id="line-spanning-chunks-is-joined"),
pytest.param(("data: x",), ("data: x",), id="unterminated-last-line-is-kept"),
pytest.param(("",), (), id="empty-body"),
)
async def _async_chunks(chunks: Sequence[str]) -> AsyncIterator[str]:
for chunk in chunks:
yield chunk
@pytest.mark.parametrize(("chunks", "expected"), CASES)
def test_iter_sse_lines_splits_only_on_cr_lf_and_crlf(chunks: Sequence[str], expected: Sequence[str]) -> None:
assert tuple(iter_sse_lines(chunks)) == tuple(expected)
@pytest.mark.parametrize(("chunks", "expected"), CASES)
async def test_aiter_sse_lines_splits_only_on_cr_lf_and_crlf(chunks: Sequence[str], expected: Sequence[str]) -> None:
assert tuple([line async for line in aiter_sse_lines(_async_chunks(chunks))]) == tuple(expected)

View file

@ -3,8 +3,9 @@ import json
import logging
import re
import threading
from collections.abc import Awaitable, Callable, Iterator, Mapping
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping
from copy import deepcopy
from itertools import pairwise
from typing import Final, List, Literal, cast, get_args
from unittest.mock import MagicMock, Mock, patch
@ -10091,7 +10092,7 @@ def test_vertex_ai_streaming_response_id():
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.iter_lines = MagicMock(return_value=stream_response())
mock_response.iter_text = MagicMock(return_value=stream_response())
return mock_response
logging_obj = MagicMock()
@ -10654,3 +10655,60 @@ async def test_gemini_sends_a_boolean_for_a_gateway_default_written_as_text_or_n
sent: Final = await _sent_thinking_config("gemini-3.8-flash", {"reasoning_effort": "high"})
assert sent["includeThoughts"] is expected
class _BodyInChunks(httpx.SyncByteStream, httpx.AsyncByteStream):
def __init__(self, chunks: tuple[bytes, ...]) -> None:
self._chunks: Final = chunks
def __iter__(self) -> Iterator[bytes]:
yield from self._chunks
async def __aiter__(self) -> AsyncIterator[bytes]:
for chunk in self._chunks:
yield chunk
def _split_at(body: bytes, cuts: tuple[int, ...]) -> tuple[bytes, ...]:
return tuple(body[start:end] for start, end in pairwise((0, *sorted(cuts), len(body))))
async def _gemini_stream(is_async: bool, chunks: tuple[bytes, ...]) -> list:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, stream=_BodyInChunks(chunks), headers={"content-type": "text/event-stream"})
kwargs: Final = {
"model": "gemini/gemini-2.5-flash",
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
"api_key": "fake-gemini-key",
}
if is_async:
stream: Final = await litellm.acompletion(
**kwargs, client=AsyncHTTPHandler(transport=httpx.MockTransport(handler))
)
return [chunk async for chunk in stream]
return list(
litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))))
)
@pytest.mark.parametrize("is_async", [True, False])
async def test_gemini_stream_keeps_text_and_usage_when_a_part_contains_a_unicode_line_separator(is_async: bool):
events: Final = (
{"candidates": [{"content": {"role": "model", "parts": [{"text": "line one\u2028line two"}]}}]},
{
"candidates": [{"content": {"role": "model", "parts": [{"text": " and three"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 8, "totalTokenCount": 18},
},
)
body: Final = "".join(f"data: {json.dumps(event, ensure_ascii=False)}\r\n\r\n" for event in events).encode()
cut_inside_separator_and_crlf: Final = (body.index(b"\xe2\x80\xa8") + 1, body.index(b"\r") + 1)
received: Final = await _gemini_stream(is_async, _split_at(body, cut_inside_separator_and_crlf))
usage: Final = next(chunk.usage for chunk in reversed(received) if getattr(chunk, "usage", None))
text: Final = "".join(c.choices[0].delta.content or "" for c in received if c.choices)
assert text == "line one\u2028line two and three"
assert (usage.prompt_tokens, usage.completion_tokens) == (10, 8)

View file

@ -39,8 +39,8 @@ class TestVertexAIHTTPStatus201(unittest.TestCase):
# Create a mock response with status code 201
mock_response = MagicMock()
mock_response.status_code = 201
mock_response.aiter_lines = MagicMock()
mock_response.aiter_lines.return_value = ["test response"]
mock_response.aiter_text = MagicMock()
mock_response.aiter_text.return_value = ["test response"]
# Setup mock client
mock_client = MagicMock()
@ -76,8 +76,8 @@ class TestVertexAIHTTPStatus201(unittest.TestCase):
# Create a mock response with status code 201
mock_response = MagicMock()
mock_response.status_code = 201
mock_response.iter_lines = MagicMock()
mock_response.iter_lines.return_value = ["test response"]
mock_response.iter_text = MagicMock()
mock_response.iter_text.return_value = ["test response"]
mock_post.return_value = mock_response
# Call the make_sync_call function