From a9e918577bb9cc5c35bd4fcf90948aa4522ea059 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 18:46:51 -0700
Subject: [PATCH] ci(e2e): run the access_control canary on harness changes and
 name failed tests

A harness-only change (proxy_client.py, conftest.py, pytest.ini, the gateway
config, .github/e2e-stack, or the workflow) selected nothing, so the stack was
never exercised by the change that touched it. select_tests.py keeps the
changed-file rule and adds the access_control suite whenever a harness file
changes. The run step now reports the pytest exit code before the evidence
check, prints pytest's summary line per pass so the rerun count is visible,
and assert_tests_ran.py names each failed or errored test as classname::name
---
 .github/e2e-stack/assert_tests_ran.py         |  4 +
 .github/e2e-stack/select_tests.py             | 31 +++++++
 .github/workflows/test-e2e-changed.yml        | 29 +++++--
 .../test_e2e_changed_gate.py                  | 85 +++++++++++++++++++
 tests/e2e/CONTRIBUTING.md                     |  6 +-
 5 files changed, 143 insertions(+), 12 deletions(-)
 create mode 100644 .github/e2e-stack/select_tests.py

diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py
index 7fd3f7c7c33..c4348c20873 100644
--- a/.github/e2e-stack/assert_tests_ran.py
+++ b/.github/e2e-stack/assert_tests_ran.py
@@ -20,6 +20,10 @@ def main() -> int:
         collected: Final = sum(case.get("file") == path for case in cases)
         skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
         _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
     if (
         selected
         and not missing
diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py
new file mode 100644
index 00000000000..cbe582b5b42
--- /dev/null
+++ b/.github/e2e-stack/select_tests.py
@@ -0,0 +1,31 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+OWN_LANE: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if SELECTABLE.match(path) and not OWN_LANE.match(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and SELECTABLE.match(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, tuple(sys.argv[1:]))) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index 16d84beab2d..23ab6dfcfe4 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -15,11 +15,21 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
+      contents: read
       pull-requests: read
     outputs:
       tests: ${{ steps.changed.outputs.tests }}
       any: ${{ steps.changed.outputs.any }}
     steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
       - name: List the e2e test files this PR added or modified
         id: changed
         env:
@@ -27,7 +37,6 @@ jobs:
           REPO: ${{ github.repository }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$'
         run: |
           gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
             --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
@@ -36,9 +45,7 @@ jobs:
             --jq '.[] | select(.status != "removed") | .filename')"
           gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
           tests="$(printf '%s\n' "${files}" \
-            | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \
-            | grep -vE "${OWN_LANE}" \
-            | sort -u | tr '\n' ' ' | sed 's/ $//')" || true
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
           echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
           if [ -n "${tests}" ]; then
             echo "any=true" >> "${GITHUB_OUTPUT}"
@@ -170,25 +177,29 @@ jobs:
           read -r -a test_files <<< "${TESTS}"
           for pass in 1 2 3; do
             report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
             echo "::group::pass ${pass} of 3"
             set +e
             uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
-              -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
             status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
             set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
             echo "::endgroup::"
             if [ "${status}" = "5" ]; then
               echo "::error::the selected files collected no runnable tests, so nothing was verified"
               exit 1
             fi
-            if ! uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"; then
-              echo "::error::pass ${pass} of 3 did not verify every selected file"
-              exit 1
-            fi
             if [ "${status}" != "0" ]; then
               echo "::error::pass ${pass} of 3 failed with exit code ${status}"
               exit "${status}"
             fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
             echo "pass ${pass} of 3 passed"
           done
 
diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py
index 2e95c33cf20..f7c4a5e2527 100644
--- a/tests/code_coverage_tests/test_e2e_changed_gate.py
+++ b/tests/code_coverage_tests/test_e2e_changed_gate.py
@@ -8,6 +8,8 @@ import pytest
 
 GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
 SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py")
+SELECT_TESTS: Final = GATE.with_name("select_tests.py")
+CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 
 
@@ -46,6 +48,29 @@ def test_passing_case_does_not_hide_a_failure_in_the_same_file(tmp_path: Path, o
     assert result.returncode == 1
 
 
+def test_failed_cases_are_named_per_selected_file(tmp_path: Path) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_ok")
+    failed: Final = ET.SubElement(
+        suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_boom"
+    )
+    _ = ET.SubElement(failed, "failure", message="secret-bearing message")
+    errored: Final = ET.SubElement(
+        suite, "testcase", file=SELECTED[1], classname="tests.e2e.access_control.test_b", name="test_setup"
+    )
+    _ = ET.SubElement(errored, "error")
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == 1
+    assert "  failed: tests.e2e.access_control.test_a::test_boom\n" in result.stdout
+    assert "  failed: tests.e2e.access_control.test_b::test_setup\n" in result.stdout
+    assert "test_ok" not in result.stdout
+    assert "secret-bearing message" not in result.stdout
+
+
 @pytest.mark.parametrize("contents", ("<testsuite/>", "<testsuite", '<testsuite><testcase name="a"/></testsuite>'))
 def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None:
     report: Final = tmp_path / "report.xml"
@@ -69,3 +94,63 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat
     assert result.returncode == 0, result.stderr
     assert result.stdout == "::add-mask::sk-0123456789abcdef\n"
     assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n"
+
+
+def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
+    result: Final = subprocess.run(
+        [sys.executable, str(SELECT_TESTS), *CANARY],
+        input="".join(f"{path}\n" for path in changed),
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stderr
+    return tuple(result.stdout.split())
+
+
+@pytest.mark.parametrize(
+    ("changed", "expected"),
+    (
+        (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)),
+        (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()),
+        (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()),
+        (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/CLAUDE.md"), ()),
+        (
+            ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"),
+            ("tests/e2e/logging/test_datadog_e2e.py",),
+        ),
+    ),
+)
+def test_changed_suite_files_are_selected_outside_the_own_lane(
+    changed: tuple[str, ...], expected: tuple[str, ...]
+) -> None:
+    assert select_tests(changed) == expected
+
+
+@pytest.mark.parametrize(
+    "harness_file",
+    (
+        "tests/e2e/proxy_client.py",
+        "tests/e2e/conftest.py",
+        "tests/e2e/pytest.ini",
+        "tests/e2e/gateway/stage_mirror_ci_config.yml",
+        ".github/e2e-stack/up.sh",
+        ".github/workflows/test-e2e-changed.yml",
+    ),
+)
+def test_harness_changes_run_the_canary_suite(harness_file: str) -> None:
+    assert select_tests((harness_file, "litellm/router.py")) == CANARY
+
+
+def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() -> None:
+    assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY
+
+
+def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None:
+    assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == (
+        *CANARY,
+        "tests/e2e/logging/test_datadog_e2e.py",
+    )
+
+
+def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None:
+    assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py")
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 0b8859d9260..be282e2a77e 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -56,15 +56,15 @@ A couple of logging destinations are configured on the proxy rather than by the
 
 ### The pull request check
 
-Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
 
-Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
+Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
 
 Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code
 
 Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need
 
-Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
+Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, pytest's summary line, failed test ids, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
 
 To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use