diff --git a/.gitignore b/.gitignore index 20355a8e4ef..87de72a8766 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,21 @@ .venv .venv_policy_test .env + +# E2E LLM translation: never commit VCR/respx cassettes. Repo uses +# Redis-backed VCR by default, but if anyone falls back to file mode +# these patterns block accidental commits that might contain API keys +# or raw provider responses. +**/cassettes/ +**/vcr_cassettes/ +*.cassette.yaml +*.cassette.yml +*.cassette.json +e2e_llm_coverage/ +e2e_llm_mutation/ +mutants/ +.e2e-llm-loop-active + .claude .newenv newenv/* diff --git a/Makefile b/Makefile index 5dbd308a3e2..5aed537b9ea 100644 --- a/Makefile +++ b/Makefile @@ -188,3 +188,43 @@ test-llm-translation-single: install-test-deps test-llm-translation-flush-vcr-cache: $(UV_RUN) python tests/_flush_vcr_cache.py + +# --------------------------------------------------------------------------- +# E2E LLM translation loop targets. See tests/llm_translation/E2E_AGENT.md. +# +# Usage: +# make e2e-llm-coverage PROVIDER=anthropic FAIL_UNDER=90 +# make e2e-llm-mutation PROVIDER=anthropic +# make e2e-llm-lock # make litellm/ read-only (tests/ stays writable) +# make e2e-llm-unlock # restore writable mode +# make e2e-llm-lock-status # check current mode +# --------------------------------------------------------------------------- +FAIL_UNDER ?= 90 + +.PHONY: e2e-llm-coverage e2e-llm-mutation e2e-llm-lock e2e-llm-unlock e2e-llm-lock-status e2e-llm-hook-install e2e-llm-hook-uninstall + +e2e-llm-coverage: + @if [ -z "$(PROVIDER)" ]; then echo "Usage: make e2e-llm-coverage PROVIDER= [FAIL_UNDER=N]"; exit 2; fi + ./scripts/e2e_llm_loop.sh coverage $(PROVIDER) --fail-under $(FAIL_UNDER) + +e2e-llm-mutation: + @if [ -z "$(PROVIDER)" ]; then echo "Usage: make e2e-llm-mutation PROVIDER="; exit 2; fi + ./scripts/e2e_llm_loop.sh mutation $(PROVIDER) + +e2e-llm-lock: + ./scripts/e2e_llm_lock.sh lock + +e2e-llm-unlock: + ./scripts/e2e_llm_lock.sh unlock + +e2e-llm-lock-status: + ./scripts/e2e_llm_lock.sh status + +e2e-llm-hook-install: + ./scripts/e2e_llm_install_hook.sh install + @touch .e2e-llm-loop-active + @echo "Sentinel .e2e-llm-loop-active created. Commits to litellm/ now blocked." + +e2e-llm-hook-uninstall: + @rm -f .e2e-llm-loop-active + ./scripts/e2e_llm_install_hook.sh uninstall diff --git a/pyproject.toml b/pyproject.toml index 65557d8a9ee..0bfac0b9f13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -305,3 +305,16 @@ pytest_add_cli_args = [ [tool.coverage.run] source = ["litellm"] relative_files = true +branch = true + +[tool.coverage.report] +# scripts/e2e_llm_loop.sh overrides `source` and `fail_under` per-provider +# via CLI flags; this is the default for ad-hoc local runs. +show_missing = true +skip_covered = false +exclude_also = [ + "if TYPE_CHECKING:", + "raise NotImplementedError", + "pragma: no cover", + "@overload", +] diff --git a/scripts/e2e_llm_install_hook.sh b/scripts/e2e_llm_install_hook.sh new file mode 100755 index 00000000000..dcd24ba3b71 --- /dev/null +++ b/scripts/e2e_llm_install_hook.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# e2e_llm_install_hook.sh — install (or remove) a local git pre-commit +# hook that blocks commits touching litellm/ source while the e2e LLM +# refactor loop is active. The hook lives at .git/hooks/pre-commit and +# is local-only (not pushed). It's belt-and-suspenders alongside +# scripts/e2e_llm_lock.sh because chmod doesn't stop root. +# +# Usage: +# scripts/e2e_llm_install_hook.sh install +# scripts/e2e_llm_install_hook.sh uninstall +# +# The hook checks for the sentinel file .e2e-llm-loop-active at repo +# root. While the sentinel exists, any staged change under litellm/ +# (except documentation-only) is rejected. Remove the sentinel +# (`rm .e2e-llm-loop-active`) to exit the locked mode. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +HOOK="${ROOT}/.git/hooks/pre-commit" + +cmd="${1:-install}" + +case "${cmd}" in + install) + cat > "${HOOK}" <<'HOOKBODY' +#!/usr/bin/env bash +# Installed by scripts/e2e_llm_install_hook.sh. Blocks commits to +# litellm/ source while the e2e LLM translation loop is active. +set -euo pipefail +repo_root="$(git rev-parse --show-toplevel)" +sentinel="${repo_root}/.e2e-llm-loop-active" +if [[ ! -f "${sentinel}" ]]; then + exit 0 +fi +blocked=$(git diff --cached --name-only --diff-filter=ACMR \ + | grep -E '^litellm/' \ + | grep -v -E '\.(md|rst|txt)$' || true) +if [[ -n "${blocked}" ]]; then + echo "E2E LLM translation loop is active (.e2e-llm-loop-active present)." >&2 + echo "Commits to litellm/ source are blocked. Staged files:" >&2 + echo "${blocked}" | sed 's/^/ /' >&2 + echo "" >&2 + echo "To proceed:" >&2 + echo " 1. Unstage litellm/ changes: git reset HEAD -- litellm/" >&2 + echo " 2. Or exit the loop: rm ${sentinel}" >&2 + exit 1 +fi +exit 0 +HOOKBODY + chmod +x "${HOOK}" + echo "Installed pre-commit hook at ${HOOK}" + echo "Touch .e2e-llm-loop-active to activate the block." + ;; + uninstall) + if [[ -f "${HOOK}" ]] && grep -q "e2e-llm-loop-active" "${HOOK}"; then + rm "${HOOK}" + echo "Removed pre-commit hook." + else + echo "No matching pre-commit hook to remove." + fi + ;; + *) + echo "Usage: $0 {install|uninstall}" >&2 + exit 2 + ;; +esac diff --git a/scripts/e2e_llm_lock.sh b/scripts/e2e_llm_lock.sh new file mode 100755 index 00000000000..c052d252965 --- /dev/null +++ b/scripts/e2e_llm_lock.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# e2e_llm_lock.sh — make non-test code read-only so a long-running agent +# can only modify tests. Used by the e2e LLM translation refactor loop. +# +# Usage: +# scripts/e2e_llm_lock.sh lock # set non-test code to read-only +# scripts/e2e_llm_lock.sh unlock # restore writable mode +# scripts/e2e_llm_lock.sh status # report current mode +# +# Locked directories: everything under litellm/ EXCEPT tests/ stays +# writable. The .git tree and tests/ are always writable so commits and +# new tests can land. chmod is per-user (u-w / u+w) so it doesn't affect +# other accounts. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TARGET_DIR="${ROOT}/litellm" + +cmd="${1:-status}" + +case "$cmd" in + lock) + echo "Locking ${TARGET_DIR} (read-only for current user)..." + find "${TARGET_DIR}" -type f -exec chmod u-w {} + + find "${TARGET_DIR}" -type d -exec chmod u-w {} + + echo "Locked. Run 'scripts/e2e_llm_lock.sh unlock' to reverse." + ;; + unlock) + echo "Unlocking ${TARGET_DIR}..." + find "${TARGET_DIR}" -type d -exec chmod u+w {} + + find "${TARGET_DIR}" -type f -exec chmod u+w {} + + echo "Unlocked." + ;; + status) + # Sample a known file. If it's writable, we're unlocked. + sample="${TARGET_DIR}/__init__.py" + if [[ -w "${sample}" ]]; then + echo "unlocked (${sample} is writable)" + else + echo "locked (${sample} is read-only)" + fi + ;; + *) + echo "Usage: $0 {lock|unlock|status}" >&2 + exit 2 + ;; +esac diff --git a/scripts/e2e_llm_loop.sh b/scripts/e2e_llm_loop.sh new file mode 100755 index 00000000000..d869f560861 --- /dev/null +++ b/scripts/e2e_llm_loop.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# e2e_llm_loop.sh — run e2e LLM-translation tests with coverage (and +# optionally mutation) scoped to ONE provider's source dir. +# +# Drives the loop described in tests/llm_translation/E2E_AGENT.md. +# +# Usage: +# scripts/e2e_llm_loop.sh coverage [--fail-under N] [pytest-args...] +# scripts/e2e_llm_loop.sh mutation [mutmut-args...] +# +# Example: +# scripts/e2e_llm_loop.sh coverage anthropic --fail-under 90 +# scripts/e2e_llm_loop.sh mutation anthropic +# +# Provider name = subdirectory of litellm/llms/ (e.g. anthropic, openai, +# bedrock). Test files are auto-discovered from +# tests/llm_translation/test_*.py. +# +# Coverage scope is litellm/llms// only — everything else is +# excluded from the report so the threshold reflects the provider's +# translation code, not the whole repo. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "${ROOT}" + +mode="${1:-}" +provider="${2:-}" + +if [[ -z "${mode}" || -z "${provider}" ]]; then + echo "Usage: $0 {coverage|mutation} [extra args...]" >&2 + exit 2 +fi +shift 2 + +PROVIDER_SRC="litellm/llms/${provider}" +if [[ ! -d "${PROVIDER_SRC}" ]]; then + echo "No source dir: ${PROVIDER_SRC}" >&2 + exit 2 +fi + +# Test files: test_*.py under tests/llm_translation/. Glob may +# expand to multiple files; that's fine for pytest. +shopt -s nullglob +test_files=( tests/llm_translation/test_${provider}*.py ) +shopt -u nullglob +if [[ ${#test_files[@]} -eq 0 ]]; then + echo "No test files matching tests/llm_translation/test_${provider}*.py" >&2 + exit 2 +fi + +case "${mode}" in + coverage) + fail_under=90 + pytest_extra=() + while [[ $# -gt 0 ]]; do + case "$1" in + --fail-under) + fail_under="$2" + shift 2 + ;; + --fail-under=*) + fail_under="${1#*=}" + shift + ;; + *) + pytest_extra+=( "$1" ) + shift + ;; + esac + done + + out_dir="e2e_llm_coverage/${provider}" + mkdir -p "${out_dir}" + + echo "==> Coverage run: provider=${provider} fail_under=${fail_under}" + echo " Tests: ${test_files[*]}" + echo " Source scope: ${PROVIDER_SRC}" + + uv run --no-sync pytest \ + "${test_files[@]}" \ + --cov="${PROVIDER_SRC}" \ + --cov-branch \ + --cov-report="term-missing:skip-covered" \ + --cov-report="html:${out_dir}/html" \ + --cov-report="xml:${out_dir}/coverage.xml" \ + --cov-fail-under="${fail_under}" \ + "${pytest_extra[@]}" + ;; + + mutation) + out_dir="e2e_llm_mutation/${provider}" + mkdir -p "${out_dir}" + + echo "==> Mutation run: provider=${provider}" + echo " Tests: ${test_files[*]}" + echo " Mutate scope: ${PROVIDER_SRC}" + echo " Output: ${out_dir}" + echo "" + echo "NOTE: mutmut config in pyproject.toml [tool.mutmut] is the" + echo " default scope (proxy/management_endpoints). This script" + echo " overrides it via env vars MUTMUT_PATHS / MUTMUT_TESTS" + echo " consumed by the wrapper below." + + # mutmut reads paths_to_mutate / tests_dir from pyproject.toml. We + # override at invocation by exporting and using --paths-to-mutate. + uv run --no-sync --with mutmut==3.3.1 mutmut run \ + --paths-to-mutate="${PROVIDER_SRC}" \ + --tests-dir="tests/llm_translation" \ + "$@" + + # Persist a JSON report alongside coverage output. + uv run --no-sync --with mutmut==3.3.1 mutmut results > "${out_dir}/results.txt" || true + echo "Wrote ${out_dir}/results.txt" + ;; + + *) + echo "Unknown mode: ${mode}" >&2 + echo "Usage: $0 {coverage|mutation} [extra args...]" >&2 + exit 2 + ;; +esac diff --git a/tests/llm_translation/E2E_AGENT.md b/tests/llm_translation/E2E_AGENT.md new file mode 100644 index 00000000000..866bd8e8c8b --- /dev/null +++ b/tests/llm_translation/E2E_AGENT.md @@ -0,0 +1,89 @@ +# E2E LLM Translation — Long-Running Agent Loop + +Drives an agent (typically Claude Code on a Max subscription) that +strengthens the e2e test suite for one provider's translation code +until the suite reaches a coverage + mutation-survival bar. The goal of +the *test bar* is to enable a later refactor of the provider's source +code with confidence. + +## Why e2e, not unit + +If the tests assert on internals (private functions, class attributes, +specific helper signatures), then any refactor of those internals also +rewrites the tests — and a test rewritten in the same change as the +code it covers is no longer a check on that change. E2E tests pin only +the public contract (`completion()`, `acompletion()`, streaming +iterators, request shape sent on the wire, response shape returned to +the caller), which is what we promise to users. Internals stay free to +refactor. + +## What the agent must do + +1. Pick **one** provider at a time (`anthropic`, `openai`, `bedrock`, + ...). The provider name maps to `litellm/llms//` (source) + and `tests/llm_translation/test_*.py` (tests). +2. Make `make e2e-llm-coverage PROVIDER=` pass with + `FAIL_UNDER=90` (configurable upward). +3. Then make `make e2e-llm-mutation PROVIDER=` finish with + mutation-survival ≤ 10% (i.e. ≥ 90% killed). Equivalent mutants — + log-string tweaks, unreachable error-message text, dead branches — + are acceptable survivors; document them in + `tests/llm_translation/E2E_AGENT_NOTES.md` rather than chasing them. +4. Never edit `litellm/` source. The `make e2e-llm-lock` target enforces + this with `chmod u-w`; the agent should treat any failed write under + `litellm/llms//` as a signal to write a test instead. + `chmod` does not stop a process running as root, so additionally run + `make e2e-llm-hook-install` once — this installs a local git + pre-commit hook that rejects commits touching `litellm/` while the + `.e2e-llm-loop-active` sentinel file is present. + +## Rules for tests + +- Use the existing Redis-backed VCR plumbing + (`tests/_vcr_conftest_common.py`). New tests inherit it automatically. +- For files that need `respx` instead, add them to + `_RESPX_CONFLICTING_FILES` in `tests/llm_translation/conftest.py`. +- **Never** record cassettes to disk. The repo uses Redis cassettes and + `.gitignore` blocks file cassettes — if a test creates a `.yaml` + cassette, treat that as a bug. +- Assert on the **request** the SDK puts on the wire (URL, headers + modulo scrubbed auth, JSON body) and on the **response shape** the + SDK returns. Avoid assertions on private attributes of provider + classes. + +## Operating loop (suggested) + +```bash +# 1. Lock source so the agent can only touch tests. +make e2e-llm-lock + +# 2. Run coverage; iterate until pass. +make e2e-llm-coverage PROVIDER=anthropic FAIL_UNDER=90 + +# 3. Run mutation; iterate until survival ≤ 10%. +make e2e-llm-mutation PROVIDER=anthropic + +# 4. Unlock and refactor the provider's source. Tests must stay green. +make e2e-llm-unlock +# ... edit litellm/llms/anthropic/ ... +make e2e-llm-coverage PROVIDER=anthropic FAIL_UNDER=90 +``` + +## Cost note + +Test runs replay from Redis VCR by default — no provider API spend +after first record. Re-record cost (24h TTL or +`make test-llm-translation-flush-vcr-cache`) is real and per-call. +Agent reasoning/edit cost is covered by the Max plan; the agent is the +expensive thing here, not the LLM API. + +## Stop conditions + +The loop ends when one of: + +- Coverage ≥ `FAIL_UNDER` **and** mutation-survival ≤ 10% on the + selected provider. +- The agent reports it cannot make further progress without source + changes (the locked tree forces this signal — it can't silently work + around a hard-to-test code path). +- The user calls a stop.