mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge e6d4f29f1d into e16d79d635
This commit is contained in:
commit
e787a9f4c4
6 changed files with 164 additions and 11 deletions
56
litellm/llms/custom_httpx/sse_lines.py
Normal file
56
litellm/llms/custom_httpx/sse_lines.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""
|
||||
Split a decoded Server-Sent Events body into lines the way the SSE spec does.
|
||||
|
||||
httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028,
|
||||
U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a
|
||||
`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR
|
||||
(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream)
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator
|
||||
from typing import Final
|
||||
|
||||
_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
|
||||
|
||||
|
||||
class _SSELineSplitter:
|
||||
def __init__(self) -> None:
|
||||
self._partial_line: io.StringIO = io.StringIO()
|
||||
self._after_cr: bool = False
|
||||
|
||||
def feed(self, text: str) -> tuple[str, ...]:
|
||||
if not text:
|
||||
return ()
|
||||
fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text
|
||||
self._after_cr = fresh.endswith("\r")
|
||||
*ended, tail = _SSE_LINE_BREAK.split(fresh)
|
||||
if not ended:
|
||||
_ = self._partial_line.write(tail)
|
||||
return ()
|
||||
first: Final = self._partial_line.getvalue() + ended[0]
|
||||
self._partial_line = io.StringIO()
|
||||
_ = self._partial_line.write(tail)
|
||||
return (first, *ended[1:])
|
||||
|
||||
def flush(self) -> tuple[str, ...]:
|
||||
rest: Final = self._partial_line.getvalue()
|
||||
self._partial_line = io.StringIO()
|
||||
return (rest,) if rest else ()
|
||||
|
||||
|
||||
def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]:
|
||||
splitter: Final = _SSELineSplitter()
|
||||
for text in text_chunks:
|
||||
yield from splitter.feed(text)
|
||||
yield from splitter.flush()
|
||||
|
||||
|
||||
async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]:
|
||||
splitter: Final = _SSELineSplitter()
|
||||
async for text in text_chunks:
|
||||
for line in splitter.feed(text):
|
||||
yield line
|
||||
for line in splitter.flush():
|
||||
yield line
|
||||
|
|
@ -41,6 +41,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
get_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
||||
|
|
@ -3241,7 +3242,7 @@ async def make_call(
|
|||
raise
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
logging_obj=logging_obj,
|
||||
response_headers=response.headers,
|
||||
|
|
@ -3312,7 +3313,7 @@ def make_sync_call(
|
|||
raise
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(response.iter_text()),
|
||||
sync_stream=True,
|
||||
logging_obj=logging_obj,
|
||||
response_headers=response.headers,
|
||||
|
|
@ -4190,7 +4191,7 @@ class ModelResponseIterator:
|
|||
log_failed_hop(e)
|
||||
return self._truncated_stream_end(token)
|
||||
self._start_hop(hop_response, token)
|
||||
self.streaming_response = hop_response.aiter_lines()
|
||||
self.streaming_response = aiter_sse_lines(hop_response.aiter_text())
|
||||
self.async_response_iterator = self.streaming_response
|
||||
return None
|
||||
|
||||
|
|
@ -4204,7 +4205,7 @@ class ModelResponseIterator:
|
|||
log_failed_hop(e)
|
||||
return self._truncated_stream_end(token)
|
||||
self._start_hop(hop_response, token)
|
||||
self.streaming_response = hop_response.iter_lines()
|
||||
self.streaming_response = iter_sse_lines(hop_response.iter_text())
|
||||
self.response_iterator = self.streaming_response
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -7704,7 +7704,7 @@ def gemini_mock_post_streaming(url, **kwargs):
|
|||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"Content-Type": "text/event-stream"}
|
||||
mock_response.iter_lines = MagicMock(return_value=stream_response())
|
||||
mock_response.iter_text = MagicMock(return_value=stream_response())
|
||||
|
||||
return mock_response
|
||||
|
||||
|
|
|
|||
38
tests/unit/llms/custom_httpx/test_sse_lines.py
Normal file
38
tests/unit/llms/custom_httpx/test_sse_lines.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from collections.abc import AsyncIterator, Sequence
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
|
||||
CASES: Final = (
|
||||
pytest.param(("data: a
b\r\n\r\n",), ("data: a
b", ""), id="u2028-stays-inside-the-line"),
|
||||
pytest.param(
|
||||
("data: \u0085
\x0b\x0c\x1c\x1d\x1e\n",),
|
||||
("data: \u0085
\x0b\x0c\x1c\x1d\x1e",),
|
||||
id="no-splitlines-only-separator-breaks-a-line",
|
||||
),
|
||||
pytest.param(("a\r\nb\nc\rd\n",), ("a", "b", "c", "d"), id="crlf-lf-and-cr-each-end-a-line"),
|
||||
pytest.param(("a\r", "\nb\n"), ("a", "b"), id="crlf-split-across-chunks-is-one-break"),
|
||||
pytest.param(("a\r", "\n\r\n"), ("a", ""), id="crlf-split-then-blank-line"),
|
||||
pytest.param(("a\r", "", "\nb\n"), ("a", "b"), id="empty-chunk-between-cr-and-lf"),
|
||||
pytest.param(("a\r", "\r"), ("a", ""), id="cr-then-cr-is-two-breaks"),
|
||||
pytest.param(("da", "ta: x", "y\n"), ("data: xy",), id="line-spanning-chunks-is-joined"),
|
||||
pytest.param(("data: x",), ("data: x",), id="unterminated-last-line-is-kept"),
|
||||
pytest.param(("",), (), id="empty-body"),
|
||||
)
|
||||
|
||||
|
||||
async def _async_chunks(chunks: Sequence[str]) -> AsyncIterator[str]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("chunks", "expected"), CASES)
|
||||
def test_iter_sse_lines_splits_only_on_cr_lf_and_crlf(chunks: Sequence[str], expected: Sequence[str]) -> None:
|
||||
assert tuple(iter_sse_lines(chunks)) == tuple(expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("chunks", "expected"), CASES)
|
||||
async def test_aiter_sse_lines_splits_only_on_cr_lf_and_crlf(chunks: Sequence[str], expected: Sequence[str]) -> None:
|
||||
assert tuple([line async for line in aiter_sse_lines(_async_chunks(chunks))]) == tuple(expected)
|
||||
|
|
@ -3,8 +3,9 @@ import json
|
|||
import logging
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping
|
||||
from copy import deepcopy
|
||||
from itertools import pairwise
|
||||
from typing import Final, List, Literal, cast, get_args
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
|
|
@ -10091,7 +10092,7 @@ def test_vertex_ai_streaming_response_id():
|
|||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.iter_lines = MagicMock(return_value=stream_response())
|
||||
mock_response.iter_text = MagicMock(return_value=stream_response())
|
||||
return mock_response
|
||||
|
||||
logging_obj = MagicMock()
|
||||
|
|
@ -10654,3 +10655,60 @@ async def test_gemini_sends_a_boolean_for_a_gateway_default_written_as_text_or_n
|
|||
sent: Final = await _sent_thinking_config("gemini-3.8-flash", {"reasoning_effort": "high"})
|
||||
|
||||
assert sent["includeThoughts"] is expected
|
||||
|
||||
|
||||
class _BodyInChunks(httpx.SyncByteStream, httpx.AsyncByteStream):
|
||||
def __init__(self, chunks: tuple[bytes, ...]) -> None:
|
||||
self._chunks: Final = chunks
|
||||
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
yield from self._chunks
|
||||
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
for chunk in self._chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
def _split_at(body: bytes, cuts: tuple[int, ...]) -> tuple[bytes, ...]:
|
||||
return tuple(body[start:end] for start, end in pairwise((0, *sorted(cuts), len(body))))
|
||||
|
||||
|
||||
async def _gemini_stream(is_async: bool, chunks: tuple[bytes, ...]) -> list:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, stream=_BodyInChunks(chunks), headers={"content-type": "text/event-stream"})
|
||||
|
||||
kwargs: Final = {
|
||||
"model": "gemini/gemini-2.5-flash",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"api_key": "fake-gemini-key",
|
||||
}
|
||||
if is_async:
|
||||
stream: Final = await litellm.acompletion(
|
||||
**kwargs, client=AsyncHTTPHandler(transport=httpx.MockTransport(handler))
|
||||
)
|
||||
return [chunk async for chunk in stream]
|
||||
return list(
|
||||
litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))))
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
async def test_gemini_stream_keeps_text_and_usage_when_a_part_contains_a_unicode_line_separator(is_async: bool):
|
||||
events: Final = (
|
||||
{"candidates": [{"content": {"role": "model", "parts": [{"text": "line one\u2028line two"}]}}]},
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": " and three"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 8, "totalTokenCount": 18},
|
||||
},
|
||||
)
|
||||
body: Final = "".join(f"data: {json.dumps(event, ensure_ascii=False)}\r\n\r\n" for event in events).encode()
|
||||
cut_inside_separator_and_crlf: Final = (body.index(b"\xe2\x80\xa8") + 1, body.index(b"\r") + 1)
|
||||
|
||||
received: Final = await _gemini_stream(is_async, _split_at(body, cut_inside_separator_and_crlf))
|
||||
|
||||
usage: Final = next(chunk.usage for chunk in reversed(received) if getattr(chunk, "usage", None))
|
||||
text: Final = "".join(c.choices[0].delta.content or "" for c in received if c.choices)
|
||||
assert text == "line one\u2028line two and three"
|
||||
assert (usage.prompt_tokens, usage.completion_tokens) == (10, 8)
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ class TestVertexAIHTTPStatus201(unittest.TestCase):
|
|||
# Create a mock response with status code 201
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 201
|
||||
mock_response.aiter_lines = MagicMock()
|
||||
mock_response.aiter_lines.return_value = ["test response"]
|
||||
mock_response.aiter_text = MagicMock()
|
||||
mock_response.aiter_text.return_value = ["test response"]
|
||||
|
||||
# Setup mock client
|
||||
mock_client = MagicMock()
|
||||
|
|
@ -76,8 +76,8 @@ class TestVertexAIHTTPStatus201(unittest.TestCase):
|
|||
# Create a mock response with status code 201
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 201
|
||||
mock_response.iter_lines = MagicMock()
|
||||
mock_response.iter_lines.return_value = ["test response"]
|
||||
mock_response.iter_text = MagicMock()
|
||||
mock_response.iter_text.return_value = ["test response"]
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
# Call the make_sync_call function
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue