mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
ci: move the roi-database Postgres shard to CircleCI integration
This commit is contained in:
parent
ebeb59d9be
commit
a9dfcd92d1
4 changed files with 20 additions and 5 deletions
|
|
@ -3569,6 +3569,12 @@ workflows:
|
|||
test_path: tests/proxy_migration_tests
|
||||
seed: false
|
||||
timeout_minutes: 20
|
||||
- postgres_suite:
|
||||
name: roi-database
|
||||
test_path: tests/integration/database/test_roi_observed.py
|
||||
seed: false
|
||||
coverage_flag: roi-postgres
|
||||
timeout_minutes: 10
|
||||
- mcp_integration:
|
||||
name: mcp-integration
|
||||
- redis_compat:
|
||||
|
|
|
|||
7
.github/scripts/assert_ci_coverage.py
vendored
7
.github/scripts/assert_ci_coverage.py
vendored
|
|
@ -568,6 +568,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes)
|
||||
circle_path: Final = repo_root / ".circleci/config.yml"
|
||||
circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
|
||||
circle_test_path_tokens: Final = _invoked_test_tokens(
|
||||
scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path"
|
||||
)
|
||||
steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
|
||||
invoked: Final = any(
|
||||
".circleci/scripts/run_integration.sh" in scalar.value
|
||||
|
|
@ -609,9 +612,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
if any(_token_covers(token, path) for token in gha_tokens)
|
||||
)
|
||||
+ tuple(
|
||||
Finding(path, "GitHub-owned integration contract has no invoking workflow")
|
||||
Finding(path, "GitHub-owned integration contract has no invoking job")
|
||||
for path in sorted(github_files)
|
||||
if not any(_token_covers(token, path) for token in gha_tokens)
|
||||
if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens)
|
||||
)
|
||||
+ tuple(
|
||||
Finding(path, "GitHub-owned integration file is missing")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
|
||||
|
||||
The ROI database contracts in `database/test_roi_observed.py` run in the GitHub Actions `roi-database` Postgres shard and upload coverage on each PR. They own temporary databases and script only the external provider transport. `GITHUB_FILES` in `run.py` assigns these files to GitHub Actions and excludes them from the CircleCI selection
|
||||
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport
|
||||
|
||||
The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ The generated lifecycle models use 20 examples, eight steps, generation and shri
|
|||
|
||||
Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change
|
||||
|
||||
The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. GitHub Actions runs only the explicit `GITHUB_FILES` set in `run.py`
|
||||
The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection
|
||||
|
||||
There is no per-node manifest. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration/<directory>/test_*.py` file in a scheduled group as owned by CircleCI
|
||||
|
||||
|
|
|
|||
|
|
@ -84,8 +84,14 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path:
|
|||
workflow.write_text(yaml.safe_dump({"jobs": {}}))
|
||||
_, missing_invocation = coverage._integration_ownership(tmp_path)
|
||||
assert [(finding.subject, finding.detail) for finding in missing_invocation] == [
|
||||
(github_path, "GitHub-owned integration contract has no invoking workflow")
|
||||
(github_path, "GitHub-owned integration contract has no invoking job")
|
||||
]
|
||||
circle_config: Final = yaml.safe_load(circle.read_text())
|
||||
circle_config["jobs"]["postgres_suite"] = {"parameters": {"test_path": {"type": "string"}}}
|
||||
circle_config["workflows"]["integration"]["jobs"].append({"postgres_suite": {"test_path": github_path}})
|
||||
circle.write_text(yaml.safe_dump(circle_config))
|
||||
_, circle_invocation = coverage._integration_ownership(tmp_path)
|
||||
assert circle_invocation == ()
|
||||
|
||||
|
||||
def test_an_ancestor_directory_covers_a_file_but_does_not_name_it():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue