diff --git a/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py b/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py index 29fa143831b..3247c846da8 100644 --- a/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py +++ b/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py @@ -1,3 +1,4 @@ +import gc import json import time import tracemalloc @@ -186,23 +187,14 @@ def _popped_values(accumulator: JSONFragmentAccumulator) -> Iterator[object]: yield value -def test_draining_many_concatenated_values_is_not_quadratic(): - """ - Regression guard: peeling N JSON values already sitting in one buffer, - one pop_next_value() call per value with no new fragments in between, - must advance a cursor instead of re-slicing the shrinking remainder. - - Measures peak allocation during the drain rather than wall-clock time: - a cursor drain only allocates per-value decode scratch, so its peak - stays far below the payload, while every re-slice copies the remainder, - so the peak reaches the payload size. Allocation is deterministic where - a timing ratio trips on a busy CI runner. - """ - count: Final = 40_000 - payload: Final = '{"a": 1}' * count +def _holding(value: str, count: int) -> JSONFragmentAccumulator: accumulator: Final = JSONFragmentAccumulator() - accumulator.append(payload) + accumulator.append(value * count) + return accumulator + +def _drain_peak_bytes(value: str, count: int) -> int: + accumulator: Final = _holding(value, count) already_tracing: Final = tracemalloc.is_tracing() if not already_tracing: tracemalloc.start() @@ -214,15 +206,69 @@ def test_draining_many_concatenated_values_is_not_quadratic(): finally: if not already_tracing: tracemalloc.stop() - assert drained == count - peak_bytes: Final = peak - baseline - assert peak_bytes < len(payload) // 4, ( - f"draining {count} values allocated a peak of {peak_bytes} bytes over a {len(payload)} byte payload; " + return peak - baseline + + +def test_draining_many_concatenated_values_never_copies_the_remainder(): + """ + Regression guard: peeling N JSON values already sitting in one buffer, + one pop_next_value() call per value with no new fragments in between, + must advance a cursor instead of re-slicing the shrinking remainder. + + Measures peak allocation during the drain rather than wall-clock time: + a cursor drain only allocates per-value decode scratch, a few KB however + large the payload, while every re-slice copies the remainder, so the peak + reaches the payload size. Runner load cannot move either number, and the + 1.28 MB payload leaves room for other threads' allocations under the bound. + """ + value: Final = '{"text": "' + "x" * 20 + '"}' + count: Final = 40_000 + payload_bytes: Final = len(value) * count + peak_bytes: Final = _drain_peak_bytes(value, count) + assert peak_bytes < payload_bytes // 4, ( + f"draining {count} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload; " "a cursor drain allocates per-value scratch only; re-slicing the remainder on every pop?" ) +def _drain_cpu_seconds(count: int) -> float: + accumulator: Final = _holding('{"a": 1}', count) + gc_was_enabled: Final = gc.isenabled() + gc.disable() + try: + started: Final = time.thread_time() + drained: Final = sum(1 for _ in _popped_values(accumulator)) + elapsed: Final = time.thread_time() - started + finally: + if gc_was_enabled: + gc.enable() + assert drained == count + return elapsed + + +def test_draining_many_concatenated_values_scales_linearly_in_cpu_time(): + """ + Regression guard for a drain that stays quadratic without copying, such as + a pop that re-scans the consumed prefix with a C string method. Sixteen + times the values costs about 16x the CPU time when the drain is linear and + 256x when it is quadratic, so the 64x bound sits 4x from both. + + Times the draining thread's own CPU time, the fastest of three runs, with + the cyclic GC paused, so neither other processes, other threads holding + the GIL, nor a collection over earlier tests' heap moves the ratio the way + they moved the wall-clock ratio this test replaced. + """ + small_count: Final = 2_500 + small: Final = min(_drain_cpu_seconds(small_count) for _ in range(3)) + large: Final = min(_drain_cpu_seconds(small_count * 16) for _ in range(3)) + ratio: Final = large / small + assert ratio < 64, ( + f"draining 16x the values took {ratio:.1f}x the CPU time ({small * 1000:.1f} ms -> {large * 1000:.1f} ms); " + "a linear drain takes about 16x; re-scanning the consumed prefix on every pop?" + ) + + def test_could_close_json_after_many_blank_fragments_is_not_quadratic(): """ Regression test: a hostile upstream can send malformed JSON that never