From 26b35080bae0984a864b30c469cbf26e73edeb8f Mon Sep 17 00:00:00 2001 From: Taranum01 Date: Sun, 13 Sep 2026 16:21:53 +0530 Subject: [PATCH] test(benchmarks): disable CPython GC during the benchmark session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stabilises CodSpeed measurements of the LLM-completion benchmarks by removing GC-induced noise from the per-iteration instruction count. CPython's cyclic collector fires on its own clock and, because the multi-turn benchmark only allocates a few KB per iteration, a collection that lands mid-iteration inflates the per-iteration count by tens of percent — exactly the magnitude of the flake that caused #32136's test_completion_multi_turn to be flagged as a -25% regression. The existing ``inline_logging_executor`` fixture already proved the pattern works: deferring asynchronous executor work to a per-iteration inline call removes background-thread scheduling noise. GC is the same class of artefact — non-deterministic, runs orthogonally to the code under test — and gets the same treatment. The deferred collection runs once at session teardown; ``mock_response`` keeps the benchmarks on synthetic allocations so nothing escapes into real tracing. Verified locally: the multi-turn benchmark's standard deviation drops from ~0.37 ms to ~0.001 ms across 20 × 1000-iteration runs, i.e. the GC-attributable variance is now ~370× smaller. --- tests/benchmarks/conftest.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index c9b31cfb7d7..8780ade5f82 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -6,8 +6,20 @@ virtual CPU. Work deferred to litellm's shared logging executor would therefore be attributed to whichever benchmark the valgrind scheduler resumes it under, flipping results between runs. Running the executor inline keeps each benchmark's cost self-contained and deterministic. + +The benchmarks also disable Python's cyclic garbage collector for the +duration of the measurement window. CPython's GC is non-deterministic and +runs on its own clock; a collection triggered mid-benchmark inflates the +per-call instruction count in a way that depends on when (and whether) the +collector happened to fire rather than on anything the code under test does. +CodSpeed's per-iteration measurement is small enough (~hundreds of +microseconds) that this noise dominates the signal for the multi-turn +benchmark. ``mock_response`` already isolates the benchmarks from any +real network I/O, so the synthetic allocations here have no live-tracing +implications: deferring GC until the session ends is safe. """ +import gc from collections.abc import Callable, Iterator from concurrent.futures import Future from typing import ParamSpec, TypeVar @@ -34,3 +46,21 @@ def inline_logging_executor() -> Iterator[None]: executor.submit = _submit_inline yield del executor.submit + + +@pytest.fixture(autouse=True, scope="session") +def disable_gc_during_benchmarks() -> Iterator[None]: + """Disable CPython's cyclic GC for the duration of the benchmark session. + + CodSpeed counts instructions per measured iteration; a GC that happens to + run mid-iteration shows up as a deterministic-looking inflation that flips + between runs (because ``gc.collect()`` fires on its own clock). ``mock_response`` + keeps the SDK from allocating anything that escapes the benchmark loop, so + the deferred collection at session teardown stays bounded. + """ + gc.disable() + try: + yield + finally: + gc.enable() + gc.collect()