fix(streaming): widen JSONFragmentAccumulator's whitespace skip to match str.strip()

pop_next_value's whitespace skip only matched json.decoder.WHITESPACE's
ASCII set, narrower than str.strip() (Unicode-aware) which the O(1)-cursor
rewrite replaced. A non-ASCII separator like U+00A0 between two
concatenated JSON values on one SSE line made raw_decode fail on it, and
the buffer never advanced past that byte again, permanently stranding
everything after it for the rest of the stream. Use str.isspace() to
match str.strip()'s tolerance.
This commit is contained in:
Deepanshu 2026-08-25 17:31:42 -04:00
parent 34a204378a
commit bf119b8476
2 changed files with 20 additions and 3 deletions

View file

@ -1,8 +1,6 @@
import json
from typing import Final, cast # noqa: TID251 # raw_decode returns tuple[Any, int]; no cast-free unpack
_JSON_WHITESPACE: Final = " \t\n\r" # matches json.decoder.WHITESPACE's character class
class JSONFragmentAccumulator:
"""
@ -68,7 +66,7 @@ class JSONFragmentAccumulator:
self._materialize()
length: Final = len(self._buffer)
start = self._offset
while start < length and self._buffer[start] in _JSON_WHITESPACE:
while start < length and self._buffer[start].isspace():
start += 1
if start >= length:
self._offset = start # mutable-ok: see __init__

View file

@ -97,6 +97,25 @@ def test_pop_next_value_peels_one_value_and_keeps_remainder():
assert not accumulator
def test_pop_next_value_skips_non_ascii_whitespace_between_concatenated_values():
"""A separator like U+00A0 (non-breaking space) between two concatenated
values must not strand the second value forever. `raw_decode` only skips
the narrow `json.decoder.WHITESPACE` set, so the accumulator's own
whitespace skip must be as tolerant as `str.strip()` was before this
class replaced it, not merely match `raw_decode`'s narrower set."""
accumulator = JSONFragmentAccumulator()
accumulator.append('{"a": 1}' + "\xa0" + '{"a": 2}')
first_found, first_value = accumulator.pop_next_value()
assert first_found is True
assert first_value == {"a": 1}
second_found, second_value = accumulator.pop_next_value()
assert second_found is True, "the second value must not be permanently stranded"
assert second_value == {"a": 2}
assert not accumulator
def test_pop_next_value_advances_past_a_non_dict_leading_value():
accumulator = JSONFragmentAccumulator()
accumulator.append("[1, 2]" + '{"a": 1}')