diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml new file mode 100644 index 00000000000..a713d2045be --- /dev/null +++ b/.github/workflows/mutation-test.yml @@ -0,0 +1,102 @@ +name: "Mutation Test (manual)" + +# Smoke test for mutation testing. Manually triggered only — never on PR/push. +# Runs mutmut against the scope configured in [tool.mutmut] in pyproject.toml +# (currently a single file, ~186 LOC). Expected runtime: ~5-15 minutes. +# +# Uploads the mutmut cache + a text summary as a workflow artifact. Failures +# do not block anything because nothing depends on this workflow. + +on: + workflow_dispatch: + # TEMPORARY: lets us iterate on this workflow before it lands on the default + # branch (workflow_dispatch only works once the file is on default). Remove + # this push: block before merging — final state should be workflow_dispatch + # only. + push: + branches: + - litellm_mutation_test_smoke + +permissions: + contents: read + +concurrency: + group: mutation-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + mutation: + name: Run mutmut + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: | + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + # mutmut 3.x runs tests inside a `mutants/` sandbox where it injects + # mutation trampolines. uv installs the project as editable by default, + # which puts the original source dir on sys.path via a .pth file and + # shadows the sandbox copy — so tests would never exercise the mutated + # code. Reinstalling non-editable removes the .pth shadow. + - name: Reinstall litellm non-editable (so mutants/ is not shadowed) + run: | + uv pip uninstall litellm + uv pip install . --no-deps + + - name: Run mutmut + env: + # Make the mutants/ sandbox win over site-packages on sys.path so the + # trampolined files are imported instead of the installed copy. + PYTHONPATH: ${{ github.workspace }}/mutants + run: | + mkdir -p mutants + uv run --no-sync --with mutmut==3.5.0 mutmut run + + - name: Show mutmut results + if: always() + run: | + uv run --no-sync --with mutmut==3.5.0 mutmut results | tee mutmut-results.txt + + - name: Upload mutmut artifacts + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: mutmut-${{ github.run_id }}-${{ github.run_attempt }} + path: | + mutmut-results.txt + mutmut-stats.json + mutmut-cache.json + if-no-files-found: warn + retention-days: 14 diff --git a/pyproject.toml b/pyproject.toml index 5cd83148d37..1a5a3e8fd2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -275,6 +275,22 @@ filterwarnings = [ "ignore::DeprecationWarning:pytest_asyncio.plugin", ] +[tool.mutmut] +# Mutation-testing scope. Driven by the manually-triggered workflow at +# .github/workflows/mutation-test.yml. mutmut is not part of the project's +# default install; it is pulled in via `uv run --with mutmut==` in CI. +# `also_copy = ["litellm/"]` is required because mutmut runs in a `mutants/` +# sandbox and the test conftest imports from across the litellm package. +paths_to_mutate = [ + "litellm/proxy/management_endpoints/router_settings_endpoints.py", +] +tests_dir = [ + "tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py", +] +also_copy = [ + "litellm/", +] + [tool.coverage.run] source = ["litellm"] relative_files = true