From 0915e87f08f8152c9b4ee28b6b8517dbb4ff6d54 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 17 May 2026 21:39:48 +0000 Subject: [PATCH] =?UTF-8?q?fix(ci):=20bugbot=20=E2=80=94=20persist=20compa?= =?UTF-8?q?t-results=20artifacts=20from=20PR=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conftest's pytest_sessionfinish writes per-cell tagged-union JSON (compat-results.json) and the per-provider rate-limit summary to paths controlled by COMPAT_RESULTS_PATH / COMPAT_RATE_LIMIT_SUMMARY_PATH. The PR gate never set either env var, so the artifacts were written to the working directory — but store_test_results only collects test-results/junit.xml, leaving the per-cell JSON unreachable from the CircleCI artifact browser. Reviewers triaging a red PR gate couldn't pull the cell-level breakdown without re-running. Point both env vars at a dedicated compat-artifacts/ directory and add a store_artifacts step so the JSON blobs become downloadable. Also extend the existing PR-gate wiring test to pin all three pieces: COMPAT_RESULTS_PATH override, COMPAT_RATE_LIMIT_SUMMARY_PATH override, and at least one store_artifacts step whose path matches the directory the exports point at. Without the cross-check, a future refactor could break the chain (e.g. only export the env vars, or only add the store_artifacts step) and the wiring would silently drop the artifacts again. Co-authored-by: Mateo Wang --- .circleci/config.yml | 20 ++++++++- .../test_circleci_pr_gate_wiring.py | 43 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4b6d2b6b47..c03736a378a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2370,7 +2370,15 @@ jobs: command: | export LITELLM_PROXY_BASE_URL="http://localhost:4000" export LITELLM_PROXY_API_KEY="sk-1234" - mkdir -p test-results + mkdir -p test-results compat-artifacts + # Point the conftest's tagged-union artifacts at a directory + # we explicitly persist below via store_artifacts. Without + # this override, the conftest defaults to compat-results.json + # / compat-rate-limit-summary.json in the working directory, + # which no CI step collects -- so reviewers can't pull the + # per-cell pass/fail breakdown when a red gate needs triage. + export COMPAT_RESULTS_PATH="$(pwd)/compat-artifacts/compat-results.json" + export COMPAT_RATE_LIMIT_SUMMARY_PATH="$(pwd)/compat-artifacts/compat-rate-limit-summary.json" uv run --no-sync python -m pytest -vv tests/claude_code/ \ --ignore=tests/claude_code/_driver_unit_tests \ --ignore=tests/claude_code/_builder_unit_tests \ @@ -2380,9 +2388,17 @@ jobs: --durations=10 no_output_timeout: 30m - # Store test results + # Store test results (JUnit) for CircleCI's test-summary view. - store_test_results: path: test-results + # Persist the compat-matrix artifacts (per-cell tagged-union JSON + # + per-provider rate-limit summary) so a red gate can be triaged + # from the CircleCI UI without re-running. `store_test_results` + # only handles JUnit-shaped XML, so these JSON blobs need their + # own store_artifacts step. + - store_artifacts: + path: compat-artifacts + destination: compat-artifacts upload-coverage: docker: diff --git a/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py b/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py index e238bc01554..3140a0bcb48 100644 --- a/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py +++ b/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py @@ -113,6 +113,49 @@ def test_pr_gate_job_exports_proxy_env_used_by_tests( assert "LITELLM_PROXY_API_KEY" in commands +def test_pr_gate_job_persists_compat_result_artifacts( + circleci_config: dict, +) -> None: + """The conftest writes per-cell tagged-union JSON + the per-provider + rate-limit summary to paths controlled by COMPAT_RESULTS_PATH / + COMPAT_RATE_LIMIT_SUMMARY_PATH. The PR gate must (1) point both env + vars at a known directory, and (2) `store_artifacts` that directory + so reviewers can pull the breakdown when a red gate needs triage. + + Without this, the conftest silently writes the artifacts to the + working directory and no CI step persists them. + """ + job = circleci_config["jobs"][JOB_NAME] + commands = "\n".join(_job_step_runs(job)) + assert "COMPAT_RESULTS_PATH" in commands, ( + "PR gate must override COMPAT_RESULTS_PATH so the compat JSON " + "lands in a directory we explicitly persist below." + ) + assert "COMPAT_RATE_LIMIT_SUMMARY_PATH" in commands, ( + "PR gate must override COMPAT_RATE_LIMIT_SUMMARY_PATH so the " + "per-provider rate-limit summary lands in a persisted directory." + ) + # The chosen directory must be wired into a store_artifacts step. + store_artifacts_paths: list[str] = [] + for step in job.get("steps", []): + if not isinstance(step, dict): + continue + sa = step.get("store_artifacts") + if isinstance(sa, dict) and isinstance(sa.get("path"), str): + store_artifacts_paths.append(sa["path"]) + assert store_artifacts_paths, ( + "PR gate must declare at least one store_artifacts step so the " + "compat-results.json / compat-rate-limit-summary.json can be " + "downloaded from the CircleCI artifact browser." + ) + # And the export must point at that directory (any of them), so the + # conftest actually writes inside the persisted tree. + assert any(path in commands for path in store_artifacts_paths), ( + "PR gate exports COMPAT_RESULTS_PATH but not into any directory " + f"declared by store_artifacts (declared: {store_artifacts_paths})." + ) + + def test_existing_proxy_e2e_anthropic_job_unchanged(circleci_config: dict) -> None: """No regression to the existing `proxy_e2e_anthropic_messages_tests` job (acceptance criterion). We don't lock its full body, but we do