mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(json_fragment_accumulator): guard copying drains by peak memory and rescanning drains by CPU-time scaling
This commit is contained in:
parent
30a638e563
commit
bf62999c3e
1 changed files with 65 additions and 19 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import gc
|
||||
import json
|
||||
import time
|
||||
import tracemalloc
|
||||
|
|
@ -186,23 +187,14 @@ def _popped_values(accumulator: JSONFragmentAccumulator) -> Iterator[object]:
|
|||
yield value
|
||||
|
||||
|
||||
def test_draining_many_concatenated_values_is_not_quadratic():
|
||||
"""
|
||||
Regression guard: peeling N JSON values already sitting in one buffer,
|
||||
one pop_next_value() call per value with no new fragments in between,
|
||||
must advance a cursor instead of re-slicing the shrinking remainder.
|
||||
|
||||
Measures peak allocation during the drain rather than wall-clock time:
|
||||
a cursor drain only allocates per-value decode scratch, so its peak
|
||||
stays far below the payload, while every re-slice copies the remainder,
|
||||
so the peak reaches the payload size. Allocation is deterministic where
|
||||
a timing ratio trips on a busy CI runner.
|
||||
"""
|
||||
count: Final = 40_000
|
||||
payload: Final = '{"a": 1}' * count
|
||||
def _holding(value: str, count: int) -> JSONFragmentAccumulator:
|
||||
accumulator: Final = JSONFragmentAccumulator()
|
||||
accumulator.append(payload)
|
||||
accumulator.append(value * count)
|
||||
return accumulator
|
||||
|
||||
|
||||
def _drain_peak_bytes(value: str, count: int) -> int:
|
||||
accumulator: Final = _holding(value, count)
|
||||
already_tracing: Final = tracemalloc.is_tracing()
|
||||
if not already_tracing:
|
||||
tracemalloc.start()
|
||||
|
|
@ -214,15 +206,69 @@ def test_draining_many_concatenated_values_is_not_quadratic():
|
|||
finally:
|
||||
if not already_tracing:
|
||||
tracemalloc.stop()
|
||||
|
||||
assert drained == count
|
||||
peak_bytes: Final = peak - baseline
|
||||
assert peak_bytes < len(payload) // 4, (
|
||||
f"draining {count} values allocated a peak of {peak_bytes} bytes over a {len(payload)} byte payload; "
|
||||
return peak - baseline
|
||||
|
||||
|
||||
def test_draining_many_concatenated_values_never_copies_the_remainder():
|
||||
"""
|
||||
Regression guard: peeling N JSON values already sitting in one buffer,
|
||||
one pop_next_value() call per value with no new fragments in between,
|
||||
must advance a cursor instead of re-slicing the shrinking remainder.
|
||||
|
||||
Measures peak allocation during the drain rather than wall-clock time:
|
||||
a cursor drain only allocates per-value decode scratch, a few KB however
|
||||
large the payload, while every re-slice copies the remainder, so the peak
|
||||
reaches the payload size. Runner load cannot move either number, and the
|
||||
1.28 MB payload leaves room for other threads' allocations under the bound.
|
||||
"""
|
||||
value: Final = '{"text": "' + "x" * 20 + '"}'
|
||||
count: Final = 40_000
|
||||
payload_bytes: Final = len(value) * count
|
||||
peak_bytes: Final = _drain_peak_bytes(value, count)
|
||||
assert peak_bytes < payload_bytes // 4, (
|
||||
f"draining {count} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload; "
|
||||
"a cursor drain allocates per-value scratch only; re-slicing the remainder on every pop?"
|
||||
)
|
||||
|
||||
|
||||
def _drain_cpu_seconds(count: int) -> float:
|
||||
accumulator: Final = _holding('{"a": 1}', count)
|
||||
gc_was_enabled: Final = gc.isenabled()
|
||||
gc.disable()
|
||||
try:
|
||||
started: Final = time.thread_time()
|
||||
drained: Final = sum(1 for _ in _popped_values(accumulator))
|
||||
elapsed: Final = time.thread_time() - started
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
assert drained == count
|
||||
return elapsed
|
||||
|
||||
|
||||
def test_draining_many_concatenated_values_scales_linearly_in_cpu_time():
|
||||
"""
|
||||
Regression guard for a drain that stays quadratic without copying, such as
|
||||
a pop that re-scans the consumed prefix with a C string method. Sixteen
|
||||
times the values costs about 16x the CPU time when the drain is linear and
|
||||
256x when it is quadratic, so the 64x bound sits 4x from both.
|
||||
|
||||
Times the draining thread's own CPU time, the fastest of three runs, with
|
||||
the cyclic GC paused, so neither other processes, other threads holding
|
||||
the GIL, nor a collection over earlier tests' heap moves the ratio the way
|
||||
they moved the wall-clock ratio this test replaced.
|
||||
"""
|
||||
small_count: Final = 2_500
|
||||
small: Final = min(_drain_cpu_seconds(small_count) for _ in range(3))
|
||||
large: Final = min(_drain_cpu_seconds(small_count * 16) for _ in range(3))
|
||||
ratio: Final = large / small
|
||||
assert ratio < 64, (
|
||||
f"draining 16x the values took {ratio:.1f}x the CPU time ({small * 1000:.1f} ms -> {large * 1000:.1f} ms); "
|
||||
"a linear drain takes about 16x; re-scanning the consumed prefix on every pop?"
|
||||
)
|
||||
|
||||
|
||||
def test_could_close_json_after_many_blank_fragments_is_not_quadratic():
|
||||
"""
|
||||
Regression test: a hostile upstream can send malformed JSON that never
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue