From 2fd511c71f81d76d3e830776853296727dfaa903 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 20 May 2026 16:44:31 -0700 Subject: [PATCH] chore(proxy_behavior): drop empty mutmut triage stub, fold protocol into README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mutmut_triage/pr1.md file was a placeholder for numbers and classifications that don't exist yet — the first mutmut run is a manual follow-up. Empty stubs aren't evidence; deleting it. The G5 protocol (run the workflow, triage survivors in the six Tier-1 handler functions, kill-or-accept-with-reason, zero unreviewed) moves into the suite README's "Gate evidence" block. The real triage file will land alongside the first mutmut follow-up. pyproject.toml's [tool.mutmut].tests_dir entry stays — that's the one-line wiring that makes the existing (manual-trigger) mutation-test workflow include our suite next time someone runs it. Comment updated to drop the dead file reference. --- pyproject.toml | 9 +-- tests/proxy_behavior/management/README.md | 15 +++-- .../management/mutmut_triage/pr1.md | 66 ------------------- 3 files changed, 15 insertions(+), 75 deletions(-) delete mode 100644 tests/proxy_behavior/management/mutmut_triage/pr1.md diff --git a/pyproject.toml b/pyproject.toml index 20cf8b9c115..b7bae873a46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -287,10 +287,11 @@ paths_to_mutate = [ ] tests_dir = [ "tests/test_litellm/proxy/management_endpoints/", - # PR1 (key Tier-1) behavior-pinning suite — see - # tests/proxy_behavior/management/regression_replay/README.md for the - # G4 evidence and tests/proxy_behavior/management/mutmut_triage/pr1.md - # for the G5 survivor-classification protocol. + # PR1 (key Tier-1) behavior-pinning suite. Manual mutmut runs + # (.github/workflows/mutation-test.yml) include this directory so the + # behavior matrix contributes to mutation-score signal alongside the + # legacy mock suite. See tests/proxy_behavior/management/README.md + # for the G5 triage protocol. "tests/proxy_behavior/management/", ] also_copy = [ diff --git a/tests/proxy_behavior/management/README.md b/tests/proxy_behavior/management/README.md index 20ddc86b4a7..0c2f9cb99d3 100644 --- a/tests/proxy_behavior/management/README.md +++ b/tests/proxy_behavior/management/README.md @@ -61,8 +61,7 @@ tests/proxy_behavior/management/ ├── test_key_update.py # Slice 10 ├── test_key_regenerate.py # Slice 11 ├── test_key_delete.py # Slice 12 -├── regression_replay/README.md # G4 — fix-PR → catching-scenario mapping -└── mutmut_triage/pr1.md # G5 — survivor classification protocol +└── regression_replay/README.md # G4 — fix-PR → catching-scenario mapping ``` ## Conventions @@ -95,8 +94,14 @@ PR1's evidence for each G1–G5 + PR1.M1–M3 gate lives in: - **G2** — `pytest --durations=…` summary in the PR description (≤ 10 min). - **G3** — `test_no_management_imports.py` is part of the suite itself. - **G4** — `regression_replay/README.md`. -- **G5** — `mutmut_triage/pr1.md`, filled in after the first manually-triggered - `mutation-test.yml` run. +- **G5** — Trigger `.github/workflows/mutation-test.yml` manually + (`gh workflow run mutation-test.yml`); the workflow uses the + `[tool.mutmut].tests_dir` entries in `pyproject.toml` to include this + suite. Triage every surviving mutant inside the six Tier-1 handler + functions (kill or accept-with-reason) — zero unreviewed survivors is + the binding gate. Record the survivor count + kill rate in a follow-up + PR comment or a triage doc filed alongside that follow-up (we don't + commit empty stubs). - **PR1.M1** — total scenario count, this README's "Layout" section. - **PR1.M2** — this README + the workflow YAML are the local-repro contract. -- **PR1.M3** — `mutmut_triage/pr1.md` "Baseline metrics" table. +- **PR1.M3** — Recorded in the manual mutmut follow-up (see G5 above). diff --git a/tests/proxy_behavior/management/mutmut_triage/pr1.md b/tests/proxy_behavior/management/mutmut_triage/pr1.md deleted file mode 100644 index fe31eea0498..00000000000 --- a/tests/proxy_behavior/management/mutmut_triage/pr1.md +++ /dev/null @@ -1,66 +0,0 @@ -# G5 — Mutmut survivor triage for PR1 (Key Tier-1) - -PR1 wires the new behavior suite into `[tool.mutmut].tests_dir` in -`pyproject.toml`. This file records the **first manually-triggered mutmut -run** that exercises `litellm/proxy/management_endpoints/key_management_endpoints.py` -against the new suite, and classifies every surviving mutant inside the 6 -Tier-1 handler functions: - - * `generate_key_fn` (and helpers `_common_key_generation_helper`, - `key_generation_check`, `_team_key_generation_check`, - `_team_key_operation_team_member_check`, `_get_user_in_team`) - * `update_key_fn` (and `_check_key_admin_access` helper) - * `regenerate_key_fn` (and `_get_and_validate_existing_key` helper) - * `info_key_fn` - * `list_keys` - * `delete_key_fn` - -## How to run - -```bash -# Manual trigger of the workflow: -gh workflow run mutation-test.yml --ref litellm_/silly-wright-1b8559 -gh run watch -``` - -Or locally (~hours, plan around it): - -```bash -uv run --with mutmut==3.5.0 mutmut run -``` - -Once the run completes, download `mutation-report.md` from the workflow -artifact and: - - 1. Filter the survivors to the 6 handler functions listed above. - 2. For each surviving mutant in those functions, add a row to the table - below classifying it **killed** (write a new test → mutant dies → - rerun confirms) or **accepted** with a one-line reason (equivalent - mutation, defensive-only branch, unreachable under any realistic - world-seed). - -G5 is the binding gate for the mutation signal: **zero unreviewed -survivors** in the Tier-1 handler set. The aggregate kill rate is recorded -as telemetry under "Baseline metrics" below — never as a goalpost. - -## Baseline metrics (filled after the first mutmut run lands) - -| Metric | Value | Notes | -|--------|-------|-------| -| Surviving mutants (all paths_to_mutate) | _TBD_ | | -| Surviving mutants in Tier-1 handler funcs | _TBD_ | G5 gate target: 0 unreviewed | -| Killed mutants | _TBD_ | | -| Kill rate (killed / (killed + survived)) | _TBD_ | Telemetry only | -| Wall-clock minutes | _TBD_ | | - -## Survivor triage - -| # | Mutant location (file:line, op) | Classification | Reason / new test | -|---|---------------------------------|----------------|-------------------| -| _none yet — first mutmut run pending_ | | | | - -## PR2 / PR3 delta - -When PR2 (Team Tier-1) ships, append a new "Baseline metrics" block here -keyed on the PR — never overwrite. The headline number to report is the -**delta** vs. PR1's baseline, not the absolute kill rate.