ci: add manually-triggered mutation testing smoke workflow

Adds a workflow_dispatch-only GitHub Actions workflow that runs mutmut
against a single source/test pair (router_settings_endpoints) to validate
the tooling end-to-end before scaling.

The workflow reinstalls litellm non-editable so the mutants/ sandbox is
not shadowed by the editable .pth on sys.path, and sets PYTHONPATH so
the trampolined sandbox copy wins over site-packages.

mutmut itself is pulled in via uv run --with so it does not appear in
uv.lock or affect the shared dev environment.

Includes a temporary push: trigger scoped to this branch so we can
iterate before the workflow file lands on the default branch — to be
removed before merging (workflow_dispatch only requires the file on the
default branch to surface the manual trigger button).
This commit is contained in:
Ryan Crabbe 2026-05-09 17:21:44 -07:00 committed by Cursor Agent
parent 12e59c8798
commit 94e491b554
No known key found for this signature in database
2 changed files with 118 additions and 0 deletions

102
.github/workflows/mutation-test.yml vendored Normal file
View file

@ -0,0 +1,102 @@
name: "Mutation Test (manual)"
# Smoke test for mutation testing. Manually triggered only — never on PR/push.
# Runs mutmut against the scope configured in [tool.mutmut] in pyproject.toml
# (currently a single file, ~186 LOC). Expected runtime: ~5-15 minutes.
#
# Uploads the mutmut cache + a text summary as a workflow artifact. Failures
# do not block anything because nothing depends on this workflow.
on:
workflow_dispatch:
# TEMPORARY: lets us iterate on this workflow before it lands on the default
# branch (workflow_dispatch only works once the file is on default). Remove
# this push: block before merging — final state should be workflow_dispatch
# only.
push:
branches:
- litellm_mutation_test_smoke
permissions:
contents: read
concurrency:
group: mutation-test-${{ github.ref }}
cancel-in-progress: true
jobs:
mutation:
name: Run mutmut
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
# mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
# mutation trampolines. uv installs the project as editable by default,
# which puts the original source dir on sys.path via a .pth file and
# shadows the sandbox copy — so tests would never exercise the mutated
# code. Reinstalling non-editable removes the .pth shadow.
- name: Reinstall litellm non-editable (so mutants/ is not shadowed)
run: |
uv pip uninstall litellm
uv pip install . --no-deps
- name: Run mutmut
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
run: |
mkdir -p mutants
uv run --no-sync --with mutmut==3.5.0 mutmut run
- name: Show mutmut results
if: always()
run: |
uv run --no-sync --with mutmut==3.5.0 mutmut results | tee mutmut-results.txt
- name: Upload mutmut artifacts
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: mutmut-${{ github.run_id }}-${{ github.run_attempt }}
path: |
mutmut-results.txt
mutmut-stats.json
mutmut-cache.json
if-no-files-found: warn
retention-days: 14

View file

@ -275,6 +275,22 @@ filterwarnings = [
"ignore::DeprecationWarning:pytest_asyncio.plugin",
]
[tool.mutmut]
# Mutation-testing scope. Driven by the manually-triggered workflow at
# .github/workflows/mutation-test.yml. mutmut is not part of the project's
# default install; it is pulled in via `uv run --with mutmut==<version>` in CI.
# `also_copy = ["litellm/"]` is required because mutmut runs in a `mutants/`
# sandbox and the test conftest imports from across the litellm package.
paths_to_mutate = [
"litellm/proxy/management_endpoints/router_settings_endpoints.py",
]
tests_dir = [
"tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py",
]
also_copy = [
"litellm/",
]
[tool.coverage.run]
source = ["litellm"]
relative_files = true