From cccfe196e5ee5cf8ee600a3b49a647a15de8429b Mon Sep 17 00:00:00 2001 From: Aasif-Simpplr Date: Tue, 6 Oct 2026 19:53:35 +0530 Subject: [PATCH 1/2] fix(proxy): keep extra chunk fields on the streaming fast path _fast_serialize_simple_model_response_stream writes a fixed set of fields, so a field set on the chunk outside its schema (model_extra), for example by an async_post_call_streaming_iterator_hook, was silently dropped from plain text chunks. Fall back to the canonical serializer when the chunk carries an extra the fast payload does not write; service_tier is still served on the fast path. --- litellm/proxy/proxy_server.py | 8 ++++++++ .../proxy/proxy_server/test_streaming_helpers.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06bfb467b52..d9119388f89 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9565,6 +9565,14 @@ def _fast_serialize_simple_model_response_stream( ): return None + # Fields set on the chunk outside its schema (e.g. by a streaming iterator + # hook) live in ``model_extra``. The payload below only writes + # ``service_tier``, so any other extra would be silently dropped here + # while the canonical serializer keeps it. + model_extra: Final = getattr(chunk, "model_extra", None) + if model_extra and any(key != "service_tier" for key in model_extra): + return None + choices: Final = getattr(chunk, "choices", None) if not isinstance(choices, list) or len(choices) != 1: return None diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 69fa195e9d6..2a3eba51f86 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -550,6 +550,21 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv assert _fast_serialize_simple_model_response_stream(chunk) is None +def test_fast_serialize_simple_model_response_stream_with_extra_field_returns_none(): + """A field a callback attaches to the chunk is not in the fast payload, so + the fast path must bail rather than drop it.""" + chunk = _simple_chunk() + chunk.gateway_metadata = {"served_by": "fallback-model"} + assert _fast_serialize_simple_model_response_stream(chunk) is None + + +def test_serialize_streaming_chunk_keeps_extra_field(): + chunk = _simple_chunk() + chunk.gateway_metadata = {"served_by": "fallback-model"} + payload = json.loads(_serialize_streaming_chunk(chunk)) + assert payload["gateway_metadata"] == {"served_by": "fallback-model"} + + # --------------------------------------------------------------------------- # _serialize_streaming_chunk # --------------------------------------------------------------------------- From 7d33d5b0432bc6040592ec95b2030806aee187bf Mon Sep 17 00:00:00 2001 From: Aasif-Simpplr Date: Wed, 7 Oct 2026 14:47:33 +0530 Subject: [PATCH 2/2] fix(proxy): ignore None-valued extras in the streaming fast path guard The stream wrapper sets citations=None on every OpenAI-style chunk, so the previous guard sent almost every chunk to the slow path. exclude_none drops None extras on the slow path too, so only non-None extras need it. --- litellm/proxy/proxy_server.py | 6 ++++-- .../unit/proxy/proxy_server/test_streaming_helpers.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d9119388f89..92629528d37 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9568,9 +9568,11 @@ def _fast_serialize_simple_model_response_stream( # Fields set on the chunk outside its schema (e.g. by a streaming iterator # hook) live in ``model_extra``. The payload below only writes # ``service_tier``, so any other extra would be silently dropped here - # while the canonical serializer keeps it. + # while the canonical serializer keeps it. None-valued extras (e.g. the + # ``citations=None`` the stream wrapper sets on every chunk) are dropped by + # ``exclude_none`` there too, so they must not force the slow path. model_extra: Final = getattr(chunk, "model_extra", None) - if model_extra and any(key != "service_tier" for key in model_extra): + if model_extra and any(key != "service_tier" and value is not None for key, value in model_extra.items()): return None choices: Final = getattr(chunk, "choices", None) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 2a3eba51f86..075a43fab5d 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -558,6 +558,17 @@ def test_fast_serialize_simple_model_response_stream_with_extra_field_returns_no assert _fast_serialize_simple_model_response_stream(chunk) is None +def test_fast_serialize_simple_model_response_stream_with_none_extra_uses_fast_path(): + """The stream wrapper sets ``citations=None`` on every OpenAI-style chunk. + ``exclude_none`` drops it on the slow path too, so it must not disable + the fast path.""" + chunk = _simple_chunk() + chunk.citations = None + fast = _fast_serialize_simple_model_response_stream(chunk) + assert fast is not None + assert "citations" not in json.loads(fast) + + def test_serialize_streaming_chunk_keeps_extra_field(): chunk = _simple_chunk() chunk.gateway_metadata = {"served_by": "fallback-model"}