mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-23 05:34:41 +00:00
fix(proxy): ignore None-valued extras in the streaming fast path guard
The stream wrapper sets citations=None on every OpenAI-style chunk, so the previous guard sent almost every chunk to the slow path. exclude_none drops None extras on the slow path too, so only non-None extras need it.
This commit is contained in:
parent
cccfe196e5
commit
7d33d5b043
2 changed files with 15 additions and 2 deletions
|
|
@ -9568,9 +9568,11 @@ def _fast_serialize_simple_model_response_stream(
|
|||
# Fields set on the chunk outside its schema (e.g. by a streaming iterator
|
||||
# hook) live in ``model_extra``. The payload below only writes
|
||||
# ``service_tier``, so any other extra would be silently dropped here
|
||||
# while the canonical serializer keeps it.
|
||||
# while the canonical serializer keeps it. None-valued extras (e.g. the
|
||||
# ``citations=None`` the stream wrapper sets on every chunk) are dropped by
|
||||
# ``exclude_none`` there too, so they must not force the slow path.
|
||||
model_extra: Final = getattr(chunk, "model_extra", None)
|
||||
if model_extra and any(key != "service_tier" for key in model_extra):
|
||||
if model_extra and any(key != "service_tier" and value is not None for key, value in model_extra.items()):
|
||||
return None
|
||||
|
||||
choices: Final = getattr(chunk, "choices", None)
|
||||
|
|
|
|||
|
|
@ -558,6 +558,17 @@ def test_fast_serialize_simple_model_response_stream_with_extra_field_returns_no
|
|||
assert _fast_serialize_simple_model_response_stream(chunk) is None
|
||||
|
||||
|
||||
def test_fast_serialize_simple_model_response_stream_with_none_extra_uses_fast_path():
|
||||
"""The stream wrapper sets ``citations=None`` on every OpenAI-style chunk.
|
||||
``exclude_none`` drops it on the slow path too, so it must not disable
|
||||
the fast path."""
|
||||
chunk = _simple_chunk()
|
||||
chunk.citations = None
|
||||
fast = _fast_serialize_simple_model_response_stream(chunk)
|
||||
assert fast is not None
|
||||
assert "citations" not in json.loads(fast)
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_keeps_extra_field():
|
||||
chunk = _simple_chunk()
|
||||
chunk.gateway_metadata = {"served_by": "fallback-model"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue