diff --git a/.github/actions/cache-prisma-binaries/action.yml b/.github/actions/cache-prisma-binaries/action.yml new file mode 100644 index 00000000000..68615e94c08 --- /dev/null +++ b/.github/actions/cache-prisma-binaries/action.yml @@ -0,0 +1,40 @@ +name: "Cache Prisma binaries" +description: >- + Cache the Prisma CLI and engine binaries that `prisma generate` downloads, so + only the first job on a given prisma-client-py version pays for the download. + + prisma-client-py shells out to `npm install prisma@` whenever its + binary cache directory has no CLI entrypoint, which pulls ~85 MB of query and + schema engines over the network. That normally takes a few seconds, but it is + unbounded: one shard of a proxy-db run took 5m18s on that single step versus + 3.8s on its eleven siblings, which pushed the job past its timeout and got a + fully passing test run cancelled. + + Callers must not set PRISMA_BINARY_CACHE_DIR. The prisma-client-py default + (~/.cache/prisma-python/binaries//) is already + keyed by both versions, so a cache entry can never be served to a run that + expects different binaries. + +runs: + using: composite + steps: + - name: Resolve prisma-client-py version + id: version + shell: bash + run: | + version="$(grep -A1 '^name = "prisma"$' uv.lock | sed -n 's/^version = "\(.*\)"$/\1/p' | head -1)" + if [ -z "${version}" ]; then + echo "could not resolve the prisma package version from uv.lock" >&2 + exit 1 + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Restore Prisma binaries + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + # ~/.cache/prisma-python holds the npm install tree prisma-client-py + # drives; ~/.cache/prisma is where @prisma/engines stages its downloads. + path: | + ~/.cache/prisma-python + ~/.cache/prisma + key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }} diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index cee93bde7f2..1de1faefd49 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -18,10 +18,18 @@ on: type: number default: 2 timeout-minutes: - description: "Job timeout in minutes" + description: >- + Timeout for the test step alone. Setup (checkout, dependency install, + Prisma client generation) gets its own allowance on top, so a slow + runner or a cold binary download can never cancel passing tests. required: false type: number default: 20 + setup-timeout-minutes: + description: "Timeout allowance for everything before the test step" + required: false + type: number + default: 12 max-failures: description: "Stop after this many failures" required: false @@ -44,7 +52,7 @@ jobs: run: name: Run tests runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.timeout-minutes }} + timeout-minutes: ${{ inputs.timeout-minutes + inputs.setup-timeout-minutes }} outputs: decision: ${{ steps.changes.outputs.decision }} @@ -82,15 +90,18 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.decision != 'skip' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests if: steps.changes.outputs.decision != 'skip' + timeout-minutes: ${{ inputs.timeout-minutes }} env: TEST_PATH: ${{ inputs.test-path }} MAX_FAILURES: ${{ inputs.max-failures }} diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml index 02543d67a82..dbd663a2efa 100644 --- a/.github/workflows/check-ui-api-types.yml +++ b/.github/workflows/check-ui-api-types.yml @@ -71,10 +71,12 @@ jobs: if: steps.changes.outputs.relevant == 'true' run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + if: steps.changes.outputs.relevant == 'true' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.relevant == 'true' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Set up Node.js diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index da4fe073a6a..68317d5dd12 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -57,9 +57,10 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/publish-basedpyright-base-counts.yml b/.github/workflows/publish-basedpyright-base-counts.yml index c85d30df0ce..71e196d8361 100644 --- a/.github/workflows/publish-basedpyright-base-counts.yml +++ b/.github/workflows/publish-basedpyright-base-counts.yml @@ -43,12 +43,13 @@ jobs: with: version: "0.10.9" + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + # The gate provisions its own measurement env (.venv-typecheck: a frozen # uv sync of its canonical dependency groups plus a generated Prisma # client), so no install step here can drift from what local runs measure. - name: Emit basedpyright counts for HEAD - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts" counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json) diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index fab05fc2bbb..57847eae01b 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -65,6 +65,9 @@ jobs: - name: check_provider_folders_documented run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py + - name: check_prisma_binary_cache + run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py + - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 3db3fb07a94..280ec476cdf 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -71,12 +71,13 @@ jobs: run: | uv sync --frozen --group proxy-dev --group e2e-dev + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma) # only after `prisma generate` writes prisma/client.py et al. Without this the # DB wrappers typed against the generated client would degrade to Unknown. - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma @@ -119,7 +120,6 @@ jobs: - name: Check basedpyright budget (delta vs base) env: GH_TOKEN: ${{ github.token }} - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA" diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 058a2538c15..7ea22825f4f 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -92,9 +92,10 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index 50589cb5926..c93779c177f 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -65,10 +65,12 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.decision != 'skip' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 60d2e471862..df212a85885 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -28,6 +28,10 @@ concurrency: # Most of a shard's time is pytest plugin load + xdist worker imports + # pytest-cov instrumentation, not the tests themselves. Keeping per-shard # work low and matching worker count to runner cores is what controls it. +# * `timeout` bounds the pytest step only. Checkout, dependency install, and +# Prisma client generation draw on a separate allowance in the base +# workflow, so slow setup shows up as a slow job rather than as a +# cancelled shard whose tests were passing. # * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores # oversubscribes 2x and workers fight for CPU during their cold-start # imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective). diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index 49aa5f9f51d..e8ca36fb30d 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -82,10 +82,12 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.decision != 'skip' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 4c2103f026d..2dffc889d0e 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -51,9 +51,10 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/tests/code_coverage_tests/check_prisma_binary_cache.py b/tests/code_coverage_tests/check_prisma_binary_cache.py new file mode 100644 index 00000000000..2544ec9708f --- /dev/null +++ b/tests/code_coverage_tests/check_prisma_binary_cache.py @@ -0,0 +1,125 @@ +"""Guard the CI cache for Prisma's CLI and engine binaries. + +``prisma generate`` shells out to ``npm install prisma@`` whenever the +prisma-client-py binary cache directory has no CLI entrypoint, pulling ~85 MB of +engines over the network. The download is normally seconds and occasionally +minutes, and a job timeout cannot tell the difference from a hung test, so an +uncached job is one slow npm response away from cancelling a passing test run. + +Three invariants keep that download off the critical path: + +1. No workflow sets ``PRISMA_BINARY_CACHE_DIR``. The prisma-client-py default is + ``~/.cache/prisma-python/binaries//``, already + keyed by both versions and the only path the cache action restores. Pointing + it elsewhere (``runner.temp`` especially, which is wiped every job) silently + guarantees a cold download. +2. Every job that generates the client also restores the cache. +3. The cache key resolves to a real version from ``uv.lock``. The action fails + the job when it cannot, so a lock format change must break here instead. +""" + +import re +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import yaml + +REPO_ROOT: Final = Path(__file__).resolve().parent.parent.parent +WORKFLOWS_DIR: Final = REPO_ROOT / ".github" / "workflows" +UV_LOCK: Final = REPO_ROOT / "uv.lock" +CACHE_ACTION: Final = "./.github/actions/cache-prisma-binaries" + +# Commands that reach the prisma binary cache: a direct generate, or a script +# that runs one on the caller's behalf. +PRISMA_GENERATE_MARKERS: Final = ("prisma generate", "type_check_gate.py") + + +class PrismaBinaryCacheError(Exception): + pass + + +def resolve_prisma_version(lock_text: str) -> str | None: + """Mirror of the shell lookup in the cache action's version step.""" + match: Final = re.search( + r'^name = "prisma"\n^version = "(?P[^"]+)"$', + lock_text, + re.MULTILINE, + ) + return match.group("version") if match else None + + +def iter_jobs(workflow: object) -> Iterator[tuple[str, dict]]: + jobs: Final = workflow.get("jobs") if isinstance(workflow, dict) else None + if not isinstance(jobs, dict): + return + yield from ((name, job) for name, job in jobs.items() if isinstance(job, dict)) + + +def job_steps(job: dict) -> tuple[dict, ...]: + steps: Final = job.get("steps") + return tuple(s for s in steps if isinstance(s, dict)) if isinstance(steps, list) else () + + +def step_generates_prisma_client(step: dict) -> bool: + run: Final = step.get("run") + return isinstance(run, str) and any(m in run for m in PRISMA_GENERATE_MARKERS) + + +def step_restores_cache(step: dict) -> bool: + return step.get("uses") == CACHE_ACTION + + +def lock_errors(lock_text: str) -> Iterator[str]: + if not resolve_prisma_version(lock_text): + yield ( + "uv.lock has no resolvable `prisma` package version. The version step " + f"in {CACHE_ACTION} greps the same shape and will fail every job that " + "generates the Prisma client." + ) + + +def workflow_errors(rel: Path, text: str) -> Iterator[str]: + if "PRISMA_BINARY_CACHE_DIR" in text: + yield ( + f"{rel}: sets PRISMA_BINARY_CACHE_DIR. Leave it unset so the binaries " + f"land in the version-keyed default path the {CACHE_ACTION} action restores." + ) + + for job_name, job in iter_jobs(yaml.safe_load(text)): + steps: Final = job_steps(job) + if any(map(step_generates_prisma_client, steps)) and not any( + map(step_restores_cache, steps) + ): + yield ( + f"{rel}: job `{job_name}` generates the Prisma client without a " + f"`uses: {CACHE_ACTION}` step, so it downloads ~85 MB of engines " + "on every run." + ) + + +def main() -> None: + errors: Final = ( + *lock_errors(UV_LOCK.read_text()), + *( + error + for path in sorted(WORKFLOWS_DIR.glob("*.y*ml")) + for error in workflow_errors(path.relative_to(REPO_ROOT), path.read_text()) + ), + ) + + if errors: + raise PrismaBinaryCacheError( + "Prisma binary cache invariants violated:\n - " + "\n - ".join(errors) + ) + + print("Prisma binary cache invariants hold across .github/workflows/") + + +if __name__ == "__main__": + try: + main() + except PrismaBinaryCacheError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1)