mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
798577d8ea
The single mutation-test job hit GitHub's hard 360-min per-job cap and died with no results. Drop the broken mutate_only_covered_lines gate (it generated zero mutants under mutmut's mutants/ sandbox due to a coverage path mismatch) and fan out one matrix job per source file, each with its own 350-min budget, a matching-test-file scope, and a resumable cache so files that still overflow continue next run.
242 lines
10 KiB
YAML
242 lines
10 KiB
YAML
name: "Mutation Test (manual)"
|
|
|
|
# Manually-triggered mutation testing against litellm/proxy/management_endpoints/.
|
|
# Intended cadence is roughly weekly — clicked from the Actions tab.
|
|
#
|
|
# Why this is sharded: the folder is ~34k LOC / 24 files. mutmut runs the
|
|
# matching test suite per surviving mutant, and `mutate_only_covered_lines`
|
|
# is disabled (it produced zero mutants under mutmut's mutants/ sandbox — see
|
|
# the note in [tool.mutmut] in pyproject.toml). One job cannot finish the
|
|
# folder inside GitHub's hard 360-min runner cap, and that cap cannot be
|
|
# raised. So instead of one long job we fan out: one matrix job per source
|
|
# file (each with its own 350-min budget, running in parallel) and a
|
|
# per-file resumable cache so a file that still overflows its budget
|
|
# continues from where it stopped on the next weekly run.
|
|
#
|
|
# Each shard uploads its own structured `mutation-report.md` (Meta ACH-style:
|
|
# original + mutated function with `# MUTANT START`/`# MUTANT END` delimiters
|
|
# + the existing tests + a task instruction) as a workflow artifact. Failures
|
|
# do not block anything because nothing depends on this workflow.
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
only:
|
|
description: "Optional: limit to a single source file stem (e.g. mcp_management_endpoints). Empty = all files."
|
|
required: false
|
|
default: ""
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: mutation-test-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
discover:
|
|
name: Discover source files
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
matrix: ${{ steps.list.outputs.matrix }}
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
# Build the shard matrix: one entry per mutatable source file. Each
|
|
# entry pins the per-shard test scope to the file's matching
|
|
# test_<stem>.py (so a mutant only re-runs its own file's tests, not
|
|
# all 45k LOC of tests in the folder); files with no matching test
|
|
# file fall back to the whole folder.
|
|
- name: List files
|
|
id: list
|
|
env:
|
|
ONLY: ${{ github.event.inputs.only }}
|
|
run: |
|
|
matrix=$(python3 - <<'PY'
|
|
import json, os, glob
|
|
only = os.environ.get("ONLY", "").strip()
|
|
out = []
|
|
for src in sorted(glob.glob("litellm/proxy/management_endpoints/*.py")):
|
|
stem = os.path.basename(src)[:-3]
|
|
if stem == "__init__":
|
|
continue
|
|
if only and stem != only:
|
|
continue
|
|
tf = f"tests/test_litellm/proxy/management_endpoints/test_{stem}.py"
|
|
tests = tf if os.path.exists(tf) else "tests/test_litellm/proxy/management_endpoints/"
|
|
out.append({"name": stem, "src": src, "tests": tests})
|
|
print(json.dumps(out))
|
|
PY
|
|
)
|
|
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
|
echo "Shards: $matrix"
|
|
|
|
mutation:
|
|
name: "mutmut: ${{ matrix.shard.name }}"
|
|
needs: discover
|
|
runs-on: ubuntu-latest
|
|
# Per-shard budget, just under the GitHub-hosted 360-min job cap. The
|
|
# mutmut step itself is wall-clock-bounded below to a value lower than
|
|
# this so it exits cleanly (persisting progress + cache) instead of
|
|
# being hard-killed on job timeout.
|
|
timeout-minutes: 350
|
|
strategy:
|
|
fail-fast: false
|
|
# Cap parallelism so we don't exhaust the org's runner pool with one
|
|
# workflow; remaining shards queue and start as runners free up.
|
|
max-parallel: 8
|
|
matrix:
|
|
shard: ${{ fromJSON(needs.discover.outputs.matrix) }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Cache uv dependencies
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: |
|
|
~/.cache/uv
|
|
.venv
|
|
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-uv-
|
|
|
|
# Per-shard resumable mutmut state. mutmut writes progress incrementally
|
|
# into mutants/; restoring last run's state lets a shard that exhausted
|
|
# its budget continue instead of restarting. Saved under a run-unique
|
|
# key; restore-keys pulls the most recent prior state for this file.
|
|
- name: Restore mutmut state
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: mutants
|
|
key: mutmut-${{ matrix.shard.name }}-${{ github.run_id }}
|
|
restore-keys: |
|
|
mutmut-${{ matrix.shard.name }}-
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
|
|
|
|
- name: Generate Prisma client
|
|
env:
|
|
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
|
|
run: |
|
|
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
|
|
|
# Narrow [tool.mutmut] to this shard's single source file and its
|
|
# matching test file. Done in-place in CI only; the committed config
|
|
# keeps the whole-folder default for local/manual runs.
|
|
- name: Scope config to shard
|
|
env:
|
|
SHARD_SRC: ${{ matrix.shard.src }}
|
|
SHARD_TESTS: ${{ matrix.shard.tests }}
|
|
run: |
|
|
python3 - <<'PY'
|
|
import os, re
|
|
src = os.environ["SHARD_SRC"]
|
|
tests = os.environ["SHARD_TESTS"]
|
|
p = "pyproject.toml"
|
|
s = open(p).read()
|
|
s = re.sub(r'paths_to_mutate = \[.*?\]',
|
|
'paths_to_mutate = [\n "%s",\n]' % src, s, flags=re.S)
|
|
s = re.sub(r'tests_dir = \[.*?\]',
|
|
'tests_dir = [\n "%s",\n]' % tests, s, flags=re.S)
|
|
open(p, "w").write(s)
|
|
PY
|
|
echo "--- [tool.mutmut] now ---"
|
|
sed -n '/\[tool.mutmut\]/,/^\[/p' pyproject.toml
|
|
|
|
# mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
|
|
# mutation trampolines. uv installs the project as editable by default,
|
|
# which puts the original source dir on sys.path via a .pth file and
|
|
# shadows the sandbox copy — so tests would never exercise the mutated
|
|
# code. Reinstalling non-editable removes the .pth shadow.
|
|
- name: Reinstall litellm non-editable (so mutants/ is not shadowed)
|
|
run: |
|
|
uv pip uninstall litellm
|
|
uv pip install . --no-deps
|
|
|
|
# pytest-retry's pytest_configure hook crashes with
|
|
# `INTERNALERROR: no option named 'filtered_exceptions'` when invoked
|
|
# via mutmut's in-process pytest.main() call. The entry-point name
|
|
# doesn't normalize cleanly with `-p no:<name>`, so just remove the
|
|
# package outright. Reruns are wrong for mutation testing anyway —
|
|
# rerunning a "failed" mutant test would mask which mutants are killed.
|
|
- name: Remove pytest plugins that conflict with mutmut
|
|
run: |
|
|
uv pip uninstall pytest-retry || true
|
|
|
|
- name: Run mutmut
|
|
env:
|
|
# Make the mutants/ sandbox win over site-packages on sys.path so the
|
|
# trampolined files are imported instead of the installed copy.
|
|
PYTHONPATH: ${{ github.workspace }}/mutants
|
|
run: |
|
|
set -o pipefail
|
|
mkdir -p mutants
|
|
# Wall-clock bound below the 350-min job timeout so mutmut receives
|
|
# SIGTERM and exits cleanly — persisting incremental progress so the
|
|
# cache + report steps still run and the next weekly run resumes.
|
|
# Exit 124 (timeout) is expected and non-fatal for this report job.
|
|
timeout --signal=TERM 330m \
|
|
uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log \
|
|
|| echo "mutmut exited non-zero (timeout/survivors) — continuing to report"
|
|
|
|
# Generate the structured report. The script embeds the enclosing
|
|
# function source for each survivor (via Python AST) and includes the
|
|
# existing test files, so an LLM agent has enough context to write
|
|
# killing tests without further file lookups. Modeled on Meta's ACH
|
|
# prompt template (arXiv 2501.12862).
|
|
- name: Generate detailed mutation report
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1
|
|
uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1
|
|
uv run --no-sync python scripts/mutation_report.py
|
|
{
|
|
echo "## Mutation shard: ${{ matrix.shard.name }}"
|
|
echo ""
|
|
head -c 900000 mutation-report.md
|
|
echo ""
|
|
echo ""
|
|
echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# Explicit save so a shard that hit its wall-clock bound (job did NOT
|
|
# fail, mutmut just exited early) still persists state for resume.
|
|
- name: Save mutmut state
|
|
if: always()
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: mutants
|
|
key: mutmut-${{ matrix.shard.name }}-${{ github.run_id }}
|
|
|
|
- name: Upload mutmut artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
|
with:
|
|
name: mutmut-${{ matrix.shard.name }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
mutation-report.md
|
|
mutmut-results.txt
|
|
mutmut-run.log
|
|
mutants/mutmut-stats.json
|
|
mutants/mutmut-cicd-stats.json
|
|
mutants/litellm/proxy/management_endpoints/**/*.py
|
|
if-no-files-found: warn
|
|
retention-days: 14
|