From 955f0183437eb7657953fc962dec275600d6571c Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 15 Aug 2026 04:36:49 +0000 Subject: [PATCH] feat(tests): rotate the daily vacuous-test batch across one area per day Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/vacuous_tests/README.md | 6 ++- tests/vacuous_tests/inventory.py | 50 +++++++++++++++++++-- tests/vacuous_tests/test_vacuous_tooling.py | 17 +++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/tests/vacuous_tests/README.md b/tests/vacuous_tests/README.md index 831ff1fb1c4..49060f028c7 100644 --- a/tests/vacuous_tests/README.md +++ b/tests/vacuous_tests/README.md @@ -24,9 +24,13 @@ Commands: python tests/vacuous_tests/inventory.py --report python tests/vacuous_tests/inventory.py --check # CI ratchet python tests/vacuous_tests/inventory.py --update-baseline # after a cleanup +python tests/vacuous_tests/inventory.py --areas # candidates per area +python tests/vacuous_tests/inventory.py --queue 15 --todays-area # the daily batch python tests/vacuous_tests/inventory.py --queue 15 --area tests/test_litellm/proxy ``` +`--todays-area` picks one area per day from the ranked list, rotating by date. That keeps every PR inside one owner's area and needs no state file, so two runs on the same day cannot disagree about where they are working + `inventory_baseline.json` records per-file candidate counts. `--check` fails when any count grows, so the number can only go down. If you are adding a deliberate assert-by-not-raising test, say so in the test's docstring and regenerate the baseline ## Stage B: does the test actually have teeth @@ -72,6 +76,6 @@ Each removal needs its own entry, keyed by the removed test id: ## Daily automation -The scheduled run pulls the next batch from `--queue`, probes each candidate, fixes only the confirmed ones, clears the rest into `verified_not_vacuous.json`, runs the flake gate and the guardrails, then opens a single PR capped at 15 tests in one area. Anything it cannot fix honestly is reported rather than patched +The scheduled run pulls the next batch from `--queue 15 --todays-area`, probes each candidate, fixes only the confirmed ones, clears the rest into `verified_not_vacuous.json`, runs the flake gate and the guardrails, then opens a single PR capped at 15 tests in one area. Anything it cannot fix honestly is reported rather than patched It stops rather than lowering the bar: if fewer than three candidates survive probing it opens no PR that day, and if three or more of its own PRs are still open it skips the run entirely diff --git a/tests/vacuous_tests/inventory.py b/tests/vacuous_tests/inventory.py index 627f232b34c..9c590dddbea 100644 --- a/tests/vacuous_tests/inventory.py +++ b/tests/vacuous_tests/inventory.py @@ -13,12 +13,16 @@ Two jobs: tests cannot land. Regenerate with `--update-baseline` after a cleanup. 2. Queue (automation). `--queue N` prints the next N candidates for the daily run, skipping anything Stage B has already cleared in - `verified_not_vacuous.json`. + `verified_not_vacuous.json`. `--todays-area` keeps a run inside one area, + rotating by date so each PR stays reviewable by one owner and no state file + is needed. Usage: python tests/vacuous_tests/inventory.py --report python tests/vacuous_tests/inventory.py --check python tests/vacuous_tests/inventory.py --update-baseline + python tests/vacuous_tests/inventory.py --areas + python tests/vacuous_tests/inventory.py --queue 15 --todays-area python tests/vacuous_tests/inventory.py --queue 15 --area tests/litellm_utils_tests """ @@ -30,8 +34,10 @@ import json import os import sys import warnings +from collections import Counter from dataclasses import dataclass -from typing import Dict, Iterable, List, Optional, Set, Tuple, Union +from datetime import date +from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) TOOL_DIR = os.path.join(REPO_ROOT, "tests", "vacuous_tests") @@ -498,6 +504,28 @@ def print_report(candidates: List[Candidate]) -> None: print(f" {bucket:<20} {totals[bucket]}") +def area_of(path: str) -> str: + parts = path.split("/") + return "/".join(parts[:3]) if len(parts) > 3 else os.path.dirname(path) + + +def areas(candidates: Sequence[Candidate]) -> Tuple[Tuple[str, int], ...]: + cleared = cleared_ids() + open_candidates = tuple(c for c in candidates if c.test_id not in cleared) + return tuple( + sorted( + Counter(area_of(c.path) for c in open_candidates).items(), + key=lambda item: (-item[1], item[0]), + ) + ) + + +def rotated_area(candidates: Sequence[Candidate], day: date) -> Optional[str]: + """Pick one area per day without storing state, so reviewers get one area per PR.""" + ranked = areas(candidates) + return ranked[day.toordinal() % len(ranked)][0] if ranked else None + + def print_queue(candidates: List[Candidate], limit: int, area: Optional[str]) -> None: cleared = cleared_ids() queue = [c for c in candidates if c.test_id not in cleared and (area is None or c.path.startswith(area))] @@ -516,6 +544,12 @@ def main() -> int: parser.add_argument("--json", metavar="PATH", help="write the full candidate list") parser.add_argument("--queue", type=int, metavar="N", help="print the next N candidates") parser.add_argument("--area", help="restrict --queue to a path prefix") + parser.add_argument("--areas", action="store_true", help="print candidate counts per area") + parser.add_argument( + "--todays-area", + action="store_true", + help="print the area this day's run should take, rotating by date", + ) parser.add_argument("--root", default=TESTS_ROOT, help="tests root to scan") args = parser.parse_args() @@ -529,9 +563,17 @@ def main() -> int: if args.update_baseline: write_baseline(counts) print(f"wrote {os.path.relpath(BASELINE_PATH, REPO_ROOT)}") + today = rotated_area(candidates, date.today()) + if args.areas: + for area, count in areas(candidates): + print(f" {area:<50} {count}") + if args.todays_area and not args.queue: + print(today or "") if args.queue: - print_queue(candidates, args.queue, args.area) - if args.report or not (args.check or args.update_baseline or args.queue or args.json): + print_queue(candidates, args.queue, args.area or (today if args.todays_area else None)) + if args.report or not ( + args.check or args.update_baseline or args.queue or args.json or args.areas or args.todays_area + ): print_report(candidates) if args.check: return check_against_baseline(counts) diff --git a/tests/vacuous_tests/test_vacuous_tooling.py b/tests/vacuous_tests/test_vacuous_tooling.py index d9ef5545d4f..5ea546ff565 100644 --- a/tests/vacuous_tests/test_vacuous_tooling.py +++ b/tests/vacuous_tests/test_vacuous_tooling.py @@ -11,6 +11,7 @@ import ast import os import sys import textwrap +from datetime import date from typing import List, Optional sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -238,6 +239,22 @@ def test_module_under_test_is_recognised_from_imports() -> None: assert not mutation_probe._is_under_test("litellm/caching/dual_cache.py", imports) +def test_area_rotation_moves_on_each_day_and_is_stable_within_one(monkeypatch) -> None: + monkeypatch.setattr(inventory, "cleared_ids", lambda: frozenset()) + candidates = [ + inventory.Candidate(path=path, lineno=index, name=f"test_{index}", bucket="no_assert", evidence="e") + for index, path in enumerate( + ["tests/a/one.py"] * 3 + ["tests/b/two.py"] * 2 + ["tests/c/three.py"], + ) + ] + assert inventory.areas(candidates) == (("tests/a", 3), ("tests/b", 2), ("tests/c", 1)) + picks = [inventory.rotated_area(candidates, date(2026, 8, day)) for day in (15, 16, 17, 18)] + assert len(set(picks[:3])) == 3 + assert picks[3] == picks[0] + assert inventory.rotated_area(candidates, date(2026, 8, 15)) == picks[0] + assert inventory.rotated_area([], date(2026, 8, 15)) is None + + def test_budget_reaches_the_module_under_test() -> None: def mutant(path: str, lineno: int) -> mutation_probe.Mutant: return mutation_probe.Mutant(path=path, lineno=lineno, description=f"{path}:{lineno}", source="")