From a9dfcd92d1ca9f1516a7fe04d13f926704b5cc21 Mon Sep 17 00:00:00 2001 From: yuneng Date: Sat, 3 Oct 2026 23:45:52 +0000 Subject: [PATCH] ci: move the roi-database Postgres shard to CircleCI integration --- .circleci/config.yml | 6 ++++++ .github/scripts/assert_ci_coverage.py | 7 +++++-- tests/integration/README.md | 4 ++-- tests/unit/test_assert_ci_coverage.py | 8 +++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9e7024e4ab..7d4d4c870b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3569,6 +3569,12 @@ workflows: test_path: tests/proxy_migration_tests seed: false timeout_minutes: 20 + - postgres_suite: + name: roi-database + test_path: tests/integration/database/test_roi_observed.py + seed: false + coverage_flag: roi-postgres + timeout_minutes: 10 - mcp_integration: name: mcp-integration - redis_compat: diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 575bb92731c..59511c7dc7b 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -568,6 +568,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes) circle_path: Final = repo_root / ".circleci/config.yml" circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {} + circle_test_path_tokens: Final = _invoked_test_tokens( + scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path" + ) steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ()) invoked: Final = any( ".circleci/scripts/run_integration.sh" in scalar.value @@ -609,9 +612,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens if any(_token_covers(token, path) for token in gha_tokens) ) + tuple( - Finding(path, "GitHub-owned integration contract has no invoking workflow") + Finding(path, "GitHub-owned integration contract has no invoking job") for path in sorted(github_files) - if not any(_token_covers(token, path) for token in gha_tokens) + if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens) ) + tuple( Finding(path, "GitHub-owned integration file is missing") diff --git a/tests/integration/README.md b/tests/integration/README.md index ef418c50759..7edd0eb2579 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The ROI database contracts in `database/test_roi_observed.py` run in the GitHub Actions `roi-database` Postgres shard and upload coverage on each PR. They own temporary databases and script only the external provider transport. `GITHUB_FILES` in `run.py` assigns these files to GitHub Actions and excludes them from the CircleCI selection +The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` @@ -14,7 +14,7 @@ The generated lifecycle models use 20 examples, eight steps, generation and shri Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change -The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. GitHub Actions runs only the explicit `GITHUB_FILES` set in `run.py` +The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection There is no per-node manifest. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 5f3903a5feb..64476a447ea 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -84,8 +84,14 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: workflow.write_text(yaml.safe_dump({"jobs": {}})) _, missing_invocation = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in missing_invocation] == [ - (github_path, "GitHub-owned integration contract has no invoking workflow") + (github_path, "GitHub-owned integration contract has no invoking job") ] + circle_config: Final = yaml.safe_load(circle.read_text()) + circle_config["jobs"]["postgres_suite"] = {"parameters": {"test_path": {"type": "string"}}} + circle_config["workflows"]["integration"]["jobs"].append({"postgres_suite": {"test_path": github_path}}) + circle.write_text(yaml.safe_dump(circle_config)) + _, circle_invocation = coverage._integration_ownership(tmp_path) + assert circle_invocation == () def test_an_ancestor_directory_covers_a_file_but_does_not_name_it():