feat(tests): scaffold per-provider e2e LLM translation refactor loop

Adds tooling for an agent-driven loop that hardens a provider's e2e test suite (coverage + mutation) before refactoring its source. Source edits are blocked via chmod + a sentinel-gated pre-commit hook so tests can't drift to cover internals the refactor will change. - scripts/e2e_llm_loop.sh: per-provider pytest+coverage and mutmut runs - scripts/e2e_llm_lock.sh: chmod-based source lock - scripts/e2e_llm_install_hook.sh: belt-and-suspenders pre-commit hook - Makefile: e2e-llm-{coverage,mutation,lock,unlock,hook-*} targets - tests/llm_translation/E2E_AGENT.md: agent prompt and operating loop - .gitignore: defensive blocks on file-based VCR cassettes - pyproject.toml: branch coverage + report config https://claude.ai/code/session_01MWq8irLUTyWi79JXY8TBHy
2026-10-12 03:48:56 +00:00 · 2026-05-12 22:39:23 +00:00 · 2026-05-12 22:39:23 +00:00 · e7a9a8138b
commit e7a9a8138b
parent 63a2d1ddc9
7 changed files with 395 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -2,6 +2,21 @@
 .venv
 .venv_policy_test
 .env
+
+# E2E LLM translation: never commit VCR/respx cassettes. Repo uses
+# Redis-backed VCR by default, but if anyone falls back to file mode
+# these patterns block accidental commits that might contain API keys
+# or raw provider responses.
+**/cassettes/
+**/vcr_cassettes/
+*.cassette.yaml
+*.cassette.yml
+*.cassette.json
+e2e_llm_coverage/
+e2e_llm_mutation/
+mutants/
+.e2e-llm-loop-active
+
 .claude
 .newenv
 newenv/*
--- a/40
+++ b/40
@ -188,3 +188,43 @@ test-llm-translation-single: install-test-deps

 test-llm-translation-flush-vcr-cache:
 	$(UV_RUN) python tests/_flush_vcr_cache.py
+
+# ---------------------------------------------------------------------------
+# E2E LLM translation loop targets. See tests/llm_translation/E2E_AGENT.md.
+#
+# Usage:
+#   make e2e-llm-coverage PROVIDER=anthropic FAIL_UNDER=90
+#   make e2e-llm-mutation PROVIDER=anthropic
+#   make e2e-llm-lock         # make litellm/ read-only (tests/ stays writable)
+#   make e2e-llm-unlock       # restore writable mode
+#   make e2e-llm-lock-status  # check current mode
+# ---------------------------------------------------------------------------
+FAIL_UNDER ?= 90
+
+.PHONY: e2e-llm-coverage e2e-llm-mutation e2e-llm-lock e2e-llm-unlock e2e-llm-lock-status e2e-llm-hook-install e2e-llm-hook-uninstall
+
+e2e-llm-coverage:
+	@if [ -z "$(PROVIDER)" ]; then echo "Usage: make e2e-llm-coverage PROVIDER=<name> [FAIL_UNDER=N]"; exit 2; fi
+	./scripts/e2e_llm_loop.sh coverage $(PROVIDER) --fail-under $(FAIL_UNDER)
+
+e2e-llm-mutation:
+	@if [ -z "$(PROVIDER)" ]; then echo "Usage: make e2e-llm-mutation PROVIDER=<name>"; exit 2; fi
+	./scripts/e2e_llm_loop.sh mutation $(PROVIDER)
+
+e2e-llm-lock:
+	./scripts/e2e_llm_lock.sh lock
+
+e2e-llm-unlock:
+	./scripts/e2e_llm_lock.sh unlock
+
+e2e-llm-lock-status:
+	./scripts/e2e_llm_lock.sh status
+
+e2e-llm-hook-install:
+	./scripts/e2e_llm_install_hook.sh install
+	@touch .e2e-llm-loop-active
+	@echo "Sentinel .e2e-llm-loop-active created. Commits to litellm/ now blocked."
+
+e2e-llm-hook-uninstall:
+	@rm -f .e2e-llm-loop-active
+	./scripts/e2e_llm_install_hook.sh uninstall
--- a/pyproject.toml
+++ b/pyproject.toml
@ -305,3 +305,16 @@ pytest_add_cli_args = [
 [tool.coverage.run]
 source = ["litellm"]
 relative_files = true
+branch = true
+
+[tool.coverage.report]
+# scripts/e2e_llm_loop.sh overrides `source` and `fail_under` per-provider
+# via CLI flags; this is the default for ad-hoc local runs.
+show_missing = true
+skip_covered = false
+exclude_also = [
+    "if TYPE_CHECKING:",
+    "raise NotImplementedError",
+    "pragma: no cover",
+    "@overload",
+]
--- a/scripts/e2e_llm_install_hook.sh
+++ b/scripts/e2e_llm_install_hook.sh
@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# e2e_llm_install_hook.sh — install (or remove) a local git pre-commit
+# hook that blocks commits touching litellm/ source while the e2e LLM
+# refactor loop is active. The hook lives at .git/hooks/pre-commit and
+# is local-only (not pushed). It's belt-and-suspenders alongside
+# scripts/e2e_llm_lock.sh because chmod doesn't stop root.
+#
+# Usage:
+#   scripts/e2e_llm_install_hook.sh install
+#   scripts/e2e_llm_install_hook.sh uninstall
+#
+# The hook checks for the sentinel file .e2e-llm-loop-active at repo
+# root. While the sentinel exists, any staged change under litellm/
+# (except documentation-only) is rejected. Remove the sentinel
+# (`rm .e2e-llm-loop-active`) to exit the locked mode.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+HOOK="${ROOT}/.git/hooks/pre-commit"
+
+cmd="${1:-install}"
+
+case "${cmd}" in
+  install)
+    cat > "${HOOK}" <<'HOOKBODY'
+#!/usr/bin/env bash
+# Installed by scripts/e2e_llm_install_hook.sh. Blocks commits to
+# litellm/ source while the e2e LLM translation loop is active.
+set -euo pipefail
+repo_root="$(git rev-parse --show-toplevel)"
+sentinel="${repo_root}/.e2e-llm-loop-active"
+if [[ ! -f "${sentinel}" ]]; then
+  exit 0
+fi
+blocked=$(git diff --cached --name-only --diff-filter=ACMR \
+  | grep -E '^litellm/' \
+  | grep -v -E '\.(md|rst|txt)$' || true)
+if [[ -n "${blocked}" ]]; then
+  echo "E2E LLM translation loop is active (.e2e-llm-loop-active present)." >&2
+  echo "Commits to litellm/ source are blocked. Staged files:" >&2
+  echo "${blocked}" | sed 's/^/  /' >&2
+  echo "" >&2
+  echo "To proceed:" >&2
+  echo "  1. Unstage litellm/ changes: git reset HEAD -- litellm/" >&2
+  echo "  2. Or exit the loop:        rm ${sentinel}" >&2
+  exit 1
+fi
+exit 0
+HOOKBODY
+    chmod +x "${HOOK}"
+    echo "Installed pre-commit hook at ${HOOK}"
+    echo "Touch .e2e-llm-loop-active to activate the block."
+    ;;
+  uninstall)
+    if [[ -f "${HOOK}" ]] && grep -q "e2e-llm-loop-active" "${HOOK}"; then
+      rm "${HOOK}"
+      echo "Removed pre-commit hook."
+    else
+      echo "No matching pre-commit hook to remove."
+    fi
+    ;;
+  *)
+    echo "Usage: $0 {install|uninstall}" >&2
+    exit 2
+    ;;
+esac
--- a/scripts/e2e_llm_lock.sh
+++ b/scripts/e2e_llm_lock.sh
@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# e2e_llm_lock.sh — make non-test code read-only so a long-running agent
+# can only modify tests. Used by the e2e LLM translation refactor loop.
+#
+# Usage:
+#   scripts/e2e_llm_lock.sh lock    # set non-test code to read-only
+#   scripts/e2e_llm_lock.sh unlock  # restore writable mode
+#   scripts/e2e_llm_lock.sh status  # report current mode
+#
+# Locked directories: everything under litellm/ EXCEPT tests/ stays
+# writable. The .git tree and tests/ are always writable so commits and
+# new tests can land. chmod is per-user (u-w / u+w) so it doesn't affect
+# other accounts.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+TARGET_DIR="${ROOT}/litellm"
+
+cmd="${1:-status}"
+
+case "$cmd" in
+  lock)
+    echo "Locking ${TARGET_DIR} (read-only for current user)..."
+    find "${TARGET_DIR}" -type f -exec chmod u-w {} +
+    find "${TARGET_DIR}" -type d -exec chmod u-w {} +
+    echo "Locked. Run 'scripts/e2e_llm_lock.sh unlock' to reverse."
+    ;;
+  unlock)
+    echo "Unlocking ${TARGET_DIR}..."
+    find "${TARGET_DIR}" -type d -exec chmod u+w {} +
+    find "${TARGET_DIR}" -type f -exec chmod u+w {} +
+    echo "Unlocked."
+    ;;
+  status)
+    # Sample a known file. If it's writable, we're unlocked.
+    sample="${TARGET_DIR}/__init__.py"
+    if [[ -w "${sample}" ]]; then
+      echo "unlocked (${sample} is writable)"
+    else
+      echo "locked (${sample} is read-only)"
+    fi
+    ;;
+  *)
+    echo "Usage: $0 {lock|unlock|status}" >&2
+    exit 2
+    ;;
+esac
--- a/scripts/e2e_llm_loop.sh
+++ b/scripts/e2e_llm_loop.sh
@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# e2e_llm_loop.sh — run e2e LLM-translation tests with coverage (and
+# optionally mutation) scoped to ONE provider's source dir.
+#
+# Drives the loop described in tests/llm_translation/E2E_AGENT.md.
+#
+# Usage:
+#   scripts/e2e_llm_loop.sh coverage <provider> [--fail-under N] [pytest-args...]
+#   scripts/e2e_llm_loop.sh mutation <provider> [mutmut-args...]
+#
+# Example:
+#   scripts/e2e_llm_loop.sh coverage anthropic --fail-under 90
+#   scripts/e2e_llm_loop.sh mutation anthropic
+#
+# Provider name = subdirectory of litellm/llms/ (e.g. anthropic, openai,
+# bedrock). Test files are auto-discovered from
+# tests/llm_translation/test_<provider>*.py.
+#
+# Coverage scope is litellm/llms/<provider>/ only — everything else is
+# excluded from the report so the threshold reflects the provider's
+# translation code, not the whole repo.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "${ROOT}"
+
+mode="${1:-}"
+provider="${2:-}"
+
+if [[ -z "${mode}" || -z "${provider}" ]]; then
+  echo "Usage: $0 {coverage|mutation} <provider> [extra args...]" >&2
+  exit 2
+fi
+shift 2
+
+PROVIDER_SRC="litellm/llms/${provider}"
+if [[ ! -d "${PROVIDER_SRC}" ]]; then
+  echo "No source dir: ${PROVIDER_SRC}" >&2
+  exit 2
+fi
+
+# Test files: test_<provider>*.py under tests/llm_translation/. Glob may
+# expand to multiple files; that's fine for pytest.
+shopt -s nullglob
+test_files=( tests/llm_translation/test_${provider}*.py )
+shopt -u nullglob
+if [[ ${#test_files[@]} -eq 0 ]]; then
+  echo "No test files matching tests/llm_translation/test_${provider}*.py" >&2
+  exit 2
+fi
+
+case "${mode}" in
+  coverage)
+    fail_under=90
+    pytest_extra=()
+    while [[ $# -gt 0 ]]; do
+      case "$1" in
+        --fail-under)
+          fail_under="$2"
+          shift 2
+          ;;
+        --fail-under=*)
+          fail_under="${1#*=}"
+          shift
+          ;;
+        *)
+          pytest_extra+=( "$1" )
+          shift
+          ;;
+      esac
+    done
+
+    out_dir="e2e_llm_coverage/${provider}"
+    mkdir -p "${out_dir}"
+
+    echo "==> Coverage run: provider=${provider} fail_under=${fail_under}"
+    echo "    Tests:        ${test_files[*]}"
+    echo "    Source scope: ${PROVIDER_SRC}"
+
+    uv run --no-sync pytest \
+      "${test_files[@]}" \
+      --cov="${PROVIDER_SRC}" \
+      --cov-branch \
+      --cov-report="term-missing:skip-covered" \
+      --cov-report="html:${out_dir}/html" \
+      --cov-report="xml:${out_dir}/coverage.xml" \
+      --cov-fail-under="${fail_under}" \
+      "${pytest_extra[@]}"
+    ;;
+
+  mutation)
+    out_dir="e2e_llm_mutation/${provider}"
+    mkdir -p "${out_dir}"
+
+    echo "==> Mutation run: provider=${provider}"
+    echo "    Tests:        ${test_files[*]}"
+    echo "    Mutate scope: ${PROVIDER_SRC}"
+    echo "    Output:       ${out_dir}"
+    echo ""
+    echo "NOTE: mutmut config in pyproject.toml [tool.mutmut] is the"
+    echo "      default scope (proxy/management_endpoints). This script"
+    echo "      overrides it via env vars MUTMUT_PATHS / MUTMUT_TESTS"
+    echo "      consumed by the wrapper below."
+
+    # mutmut reads paths_to_mutate / tests_dir from pyproject.toml. We
+    # override at invocation by exporting and using --paths-to-mutate.
+    uv run --no-sync --with mutmut==3.3.1 mutmut run \
+      --paths-to-mutate="${PROVIDER_SRC}" \
+      --tests-dir="tests/llm_translation" \
+      "$@"
+
+    # Persist a JSON report alongside coverage output.
+    uv run --no-sync --with mutmut==3.3.1 mutmut results > "${out_dir}/results.txt" || true
+    echo "Wrote ${out_dir}/results.txt"
+    ;;
+
+  *)
+    echo "Unknown mode: ${mode}" >&2
+    echo "Usage: $0 {coverage|mutation} <provider> [extra args...]" >&2
+    exit 2
+    ;;
+esac
--- a/tests/llm_translation/E2E_AGENT.md
+++ b/tests/llm_translation/E2E_AGENT.md
@ -0,0 +1,89 @@
+# E2E LLM Translation — Long-Running Agent Loop
+
+Drives an agent (typically Claude Code on a Max subscription) that
+strengthens the e2e test suite for one provider's translation code
+until the suite reaches a coverage + mutation-survival bar. The goal of
+the *test bar* is to enable a later refactor of the provider's source
+code with confidence.
+
+## Why e2e, not unit
+
+If the tests assert on internals (private functions, class attributes,
+specific helper signatures), then any refactor of those internals also
+rewrites the tests — and a test rewritten in the same change as the
+code it covers is no longer a check on that change. E2E tests pin only
+the public contract (`completion()`, `acompletion()`, streaming
+iterators, request shape sent on the wire, response shape returned to
+the caller), which is what we promise to users. Internals stay free to
+refactor.
+
+## What the agent must do
+
+1. Pick **one** provider at a time (`anthropic`, `openai`, `bedrock`,
+   ...). The provider name maps to `litellm/llms/<provider>/` (source)
+   and `tests/llm_translation/test_<provider>*.py` (tests).
+2. Make `make e2e-llm-coverage PROVIDER=<provider>` pass with
+   `FAIL_UNDER=90` (configurable upward).
+3. Then make `make e2e-llm-mutation PROVIDER=<provider>` finish with
+   mutation-survival ≤ 10% (i.e. ≥ 90% killed). Equivalent mutants —
+   log-string tweaks, unreachable error-message text, dead branches —
+   are acceptable survivors; document them in
+   `tests/llm_translation/E2E_AGENT_NOTES.md` rather than chasing them.
+4. Never edit `litellm/` source. The `make e2e-llm-lock` target enforces
+   this with `chmod u-w`; the agent should treat any failed write under
+   `litellm/llms/<provider>/` as a signal to write a test instead.
+   `chmod` does not stop a process running as root, so additionally run
+   `make e2e-llm-hook-install` once — this installs a local git
+   pre-commit hook that rejects commits touching `litellm/` while the
+   `.e2e-llm-loop-active` sentinel file is present.
+
+## Rules for tests
+
+- Use the existing Redis-backed VCR plumbing
+  (`tests/_vcr_conftest_common.py`). New tests inherit it automatically.
+- For files that need `respx` instead, add them to
+  `_RESPX_CONFLICTING_FILES` in `tests/llm_translation/conftest.py`.
+- **Never** record cassettes to disk. The repo uses Redis cassettes and
+  `.gitignore` blocks file cassettes — if a test creates a `.yaml`
+  cassette, treat that as a bug.
+- Assert on the **request** the SDK puts on the wire (URL, headers
+  modulo scrubbed auth, JSON body) and on the **response shape** the
+  SDK returns. Avoid assertions on private attributes of provider
+  classes.
+
+## Operating loop (suggested)
+
+```bash
+# 1. Lock source so the agent can only touch tests.
+make e2e-llm-lock
+
+# 2. Run coverage; iterate until pass.
+make e2e-llm-coverage PROVIDER=anthropic FAIL_UNDER=90
+
+# 3. Run mutation; iterate until survival ≤ 10%.
+make e2e-llm-mutation PROVIDER=anthropic
+
+# 4. Unlock and refactor the provider's source. Tests must stay green.
+make e2e-llm-unlock
+#    ... edit litellm/llms/anthropic/ ...
+make e2e-llm-coverage PROVIDER=anthropic FAIL_UNDER=90
+```
+
+## Cost note
+
+Test runs replay from Redis VCR by default — no provider API spend
+after first record. Re-record cost (24h TTL or
+`make test-llm-translation-flush-vcr-cache`) is real and per-call.
+Agent reasoning/edit cost is covered by the Max plan; the agent is the
+expensive thing here, not the LLM API.
+
+## Stop conditions
+
+The loop ends when one of:
+
+- Coverage ≥ `FAIL_UNDER` **and** mutation-survival ≤ 10% on the
+  selected provider.
+- The agent reports it cannot make further progress without source
+  changes (the locked tree forces this signal — it can't silently work
+  around a hard-to-test code path).
+- The user calls a stop.