diff --git a/.github/actions/cache-prisma-binaries/action.yml b/.github/actions/cache-prisma-binaries/action.yml new file mode 100644 index 00000000000..68615e94c08 --- /dev/null +++ b/.github/actions/cache-prisma-binaries/action.yml @@ -0,0 +1,40 @@ +name: "Cache Prisma binaries" +description: >- + Cache the Prisma CLI and engine binaries that `prisma generate` downloads, so + only the first job on a given prisma-client-py version pays for the download. + + prisma-client-py shells out to `npm install prisma@` whenever its + binary cache directory has no CLI entrypoint, which pulls ~85 MB of query and + schema engines over the network. That normally takes a few seconds, but it is + unbounded: one shard of a proxy-db run took 5m18s on that single step versus + 3.8s on its eleven siblings, which pushed the job past its timeout and got a + fully passing test run cancelled. + + Callers must not set PRISMA_BINARY_CACHE_DIR. The prisma-client-py default + (~/.cache/prisma-python/binaries//) is already + keyed by both versions, so a cache entry can never be served to a run that + expects different binaries. + +runs: + using: composite + steps: + - name: Resolve prisma-client-py version + id: version + shell: bash + run: | + version="$(grep -A1 '^name = "prisma"$' uv.lock | sed -n 's/^version = "\(.*\)"$/\1/p' | head -1)" + if [ -z "${version}" ]; then + echo "could not resolve the prisma package version from uv.lock" >&2 + exit 1 + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Restore Prisma binaries + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + # ~/.cache/prisma-python holds the npm install tree prisma-client-py + # drives; ~/.cache/prisma is where @prisma/engines stages its downloads. + path: | + ~/.cache/prisma-python + ~/.cache/prisma + key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }} diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index cee93bde7f2..58208988fca 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -18,10 +18,25 @@ on: type: number default: 2 timeout-minutes: - description: "Job timeout in minutes" + description: >- + Timeout for the test step alone. Setup (checkout, dependency install, + Prisma client generation) gets its own allowance on top, so a slow + runner or a cold binary download can never cancel passing tests. required: false type: number default: 20 + job-timeout-minutes: + description: >- + Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for + the per-step ceilings on the setup steps below, and 5 for the runner + overhead the job clock charges but no step owns (job init, step + transitions, post-job cleanup). That headroom is what makes the test + budget a floor rather than a hope, since setup cannot overrun into it + without failing its own step first. GitHub expressions have no + arithmetic, so the sum is passed in rather than computed. + required: false + type: number + default: 55 max-failures: description: "Stop after this many failures" required: false @@ -44,30 +59,35 @@ jobs: run: name: Run tests runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.timeout-minutes }} + timeout-minutes: ${{ inputs.job-timeout-minutes }} outputs: decision: ${{ steps.changes.outputs.decision }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + timeout-minutes: 3 with: persist-credentials: false - name: Detect backend-relevant changes id: changes + timeout-minutes: 2 uses: ./.github/actions/detect-backend-changes - name: Set up Python + timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + timeout-minutes: 3 uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + timeout-minutes: 5 uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | @@ -79,18 +99,24 @@ jobs: - name: Install dependencies if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 8 run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 3 + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.decision != 'skip' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + timeout-minutes: 3 run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests if: steps.changes.outputs.decision != 'skip' + timeout-minutes: ${{ inputs.timeout-minutes }} env: TEST_PATH: ${{ inputs.test-path }} MAX_FAILURES: ${{ inputs.max-failures }} diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml index 02543d67a82..dbd663a2efa 100644 --- a/.github/workflows/check-ui-api-types.yml +++ b/.github/workflows/check-ui-api-types.yml @@ -71,10 +71,12 @@ jobs: if: steps.changes.outputs.relevant == 'true' run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + if: steps.changes.outputs.relevant == 'true' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.relevant == 'true' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Set up Node.js diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index da4fe073a6a..68317d5dd12 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -57,9 +57,10 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/publish-basedpyright-base-counts.yml b/.github/workflows/publish-basedpyright-base-counts.yml index c85d30df0ce..71e196d8361 100644 --- a/.github/workflows/publish-basedpyright-base-counts.yml +++ b/.github/workflows/publish-basedpyright-base-counts.yml @@ -43,12 +43,13 @@ jobs: with: version: "0.10.9" + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + # The gate provisions its own measurement env (.venv-typecheck: a frozen # uv sync of its canonical dependency groups plus a generated Prisma # client), so no install step here can drift from what local runs measure. - name: Emit basedpyright counts for HEAD - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts" counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json) diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index fab05fc2bbb..8f62837d29a 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -65,6 +65,12 @@ jobs: - name: check_provider_folders_documented run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py + - name: check_prisma_binary_cache + run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py + + - name: check_workflow_startup_safety + run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py + - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 3db3fb07a94..280ec476cdf 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -71,12 +71,13 @@ jobs: run: | uv sync --frozen --group proxy-dev --group e2e-dev + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma) # only after `prisma generate` writes prisma/client.py et al. Without this the # DB wrappers typed against the generated client would degrade to Unknown. - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma @@ -119,7 +120,6 @@ jobs: - name: Check basedpyright budget (delta vs base) env: GH_TOKEN: ${{ github.token }} - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA" diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 058a2538c15..7ea22825f4f 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -92,9 +92,10 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index 50589cb5926..c93779c177f 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -65,10 +65,12 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.decision != 'skip' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 60d2e471862..df212a85885 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -28,6 +28,10 @@ concurrency: # Most of a shard's time is pytest plugin load + xdist worker imports + # pytest-cov instrumentation, not the tests themselves. Keeping per-shard # work low and matching worker count to runner cores is what controls it. +# * `timeout` bounds the pytest step only. Checkout, dependency install, and +# Prisma client generation draw on a separate allowance in the base +# workflow, so slow setup shows up as a slow job rather than as a +# cancelled shard whose tests were passing. # * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores # oversubscribes 2x and workers fight for CPU during their cold-start # imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective). diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index 2ea3c521e8b..64b92f7d847 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -76,4 +76,5 @@ jobs: workers: 4 reruns: 2 timeout-minutes: 60 + job-timeout-minutes: 95 artifact-name: proxy-server diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index 49aa5f9f51d..e8ca36fb30d 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -82,10 +82,12 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client if: steps.changes.outputs.decision != 'skip' - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 4c2103f026d..2dffc889d0e 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -51,9 +51,10 @@ jobs: run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma diff --git a/tests/code_coverage_tests/check_prisma_binary_cache.py b/tests/code_coverage_tests/check_prisma_binary_cache.py new file mode 100644 index 00000000000..501688385ff --- /dev/null +++ b/tests/code_coverage_tests/check_prisma_binary_cache.py @@ -0,0 +1,143 @@ +"""Guard the CI cache for Prisma's CLI and engine binaries. + +``prisma generate`` shells out to ``npm install prisma@`` whenever the +prisma-client-py binary cache directory has no CLI entrypoint, pulling ~85 MB of +engines over the network. The download is normally seconds and occasionally +minutes, and a job timeout cannot tell the difference from a hung test, so an +uncached job is one slow npm response away from cancelling a passing test run. + +Three invariants keep that download off the critical path: + +1. No workflow sets ``PRISMA_BINARY_CACHE_DIR``. The prisma-client-py default is + ``~/.cache/prisma-python/binaries//``, already + keyed by both versions and the only path the cache action restores. Pointing + it elsewhere (``runner.temp`` especially, which is wiped every job) silently + guarantees a cold download. +2. Every job that generates the client also restores the cache. +3. The cache key resolves to a real version from ``uv.lock``. The action fails + the job when it cannot, so a lock format change must break here instead. +""" + +import re +import sys +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Final + +import yaml +from pydantic import BaseModel, Field, ValidationError + +REPO_ROOT: Final = Path(__file__).resolve().parent.parent.parent +WORKFLOWS_DIR: Final = REPO_ROOT / ".github" / "workflows" +UV_LOCK: Final = REPO_ROOT / "uv.lock" +CACHE_ACTION: Final = "./.github/actions/cache-prisma-binaries" + +# Commands that reach the prisma binary cache: a direct generate, or a script +# that runs one on the caller's behalf. +PRISMA_GENERATE_MARKERS: Final = ("prisma generate", "type_check_gate.py") + + +class PrismaBinaryCacheError(Exception): + pass + + +def resolve_prisma_version(lock_text: str) -> str | None: + """Mirror of the shell lookup in the cache action's version step.""" + match: Final = re.search( + r'^name = "prisma"\n^version = "(?P[^"]+)"$', + lock_text, + re.MULTILINE, + ) + return match.group("version") if match else None + + +class WorkflowStep(BaseModel): + """The two step fields this guard reads; every other key is ignored.""" + + run: str | None = None + uses: str | None = None + + def generates_prisma_client(self) -> bool: + return self.run is not None and any(m in self.run for m in PRISMA_GENERATE_MARKERS) + + def restores_cache(self) -> bool: + return self.uses == CACHE_ACTION + + +class WorkflowJob(BaseModel): + # Absent for jobs that delegate to a reusable workflow via a job-level `uses`. + steps: tuple[WorkflowStep, ...] = () + + +class Workflow(BaseModel): + jobs: Mapping[str, WorkflowJob] = Field(default_factory=dict) + + +def parse_workflow(text: str) -> Workflow | str: + """Validate untyped YAML at the boundary so the checks below stay typed. + + Returns the parsed workflow, or a description of why it could not be read. + """ + parsed: Final = yaml.safe_load(text) + try: + return Workflow.model_validate(parsed if isinstance(parsed, dict) else {}) + except ValidationError as exc: + return f"does not parse as a workflow: {exc.error_count()} schema error(s)" + + +def lock_errors(lock_text: str) -> Iterator[str]: + if not resolve_prisma_version(lock_text): + yield ( + "uv.lock has no resolvable `prisma` package version. The version step " + f"in {CACHE_ACTION} greps the same shape and will fail every job that " + "generates the Prisma client." + ) + + +def workflow_errors(rel: Path, text: str) -> Iterator[str]: + if "PRISMA_BINARY_CACHE_DIR" in text: + yield ( + f"{rel}: sets PRISMA_BINARY_CACHE_DIR. Leave it unset so the binaries " + f"land in the version-keyed default path the {CACHE_ACTION} action restores." + ) + + workflow: Final = parse_workflow(text) + if isinstance(workflow, str): + yield f"{rel}: {workflow}" + return + + for job_name, job in workflow.jobs.items(): + if any(s.generates_prisma_client() for s in job.steps) and not any( + s.restores_cache() for s in job.steps + ): + yield ( + f"{rel}: job `{job_name}` generates the Prisma client without a " + f"`uses: {CACHE_ACTION}` step, so it downloads ~85 MB of engines " + "on every run." + ) + + +def main() -> None: + errors: Final = ( + *lock_errors(UV_LOCK.read_text()), + *( + error + for path in sorted(WORKFLOWS_DIR.glob("*.y*ml")) + for error in workflow_errors(path.relative_to(REPO_ROOT), path.read_text()) + ), + ) + + if errors: + raise PrismaBinaryCacheError( + "Prisma binary cache invariants violated:\n - " + "\n - ".join(errors) + ) + + print("Prisma binary cache invariants hold across .github/workflows/") + + +if __name__ == "__main__": + try: + main() + except PrismaBinaryCacheError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/tests/code_coverage_tests/check_workflow_startup_safety.py b/tests/code_coverage_tests/check_workflow_startup_safety.py new file mode 100644 index 00000000000..cf150daef4c --- /dev/null +++ b/tests/code_coverage_tests/check_workflow_startup_safety.py @@ -0,0 +1,239 @@ +"""Catch workflow mistakes that GitHub reports as nothing at all. + +A workflow whose YAML is valid but whose expressions are not fails at *startup*: +the run is marked failed, no jobs are created, and no check run is ever posted. +Nothing turns red on the PR, so an entire test suite can silently stop running +while the checks list stays green. These invariants have to be enforced here +because CI cannot enforce them on itself. + +1. No arithmetic inside ``${{ }}``. GitHub expressions support grouping, index, + dereference, ``!``, the comparisons, ``&&`` and ``||``, and nothing else. A + ``${{ a + b }}`` is a startup failure, not a value. Only ``+`` and ``*`` are + flagged: ``-`` appears in hyphenated input names like ``inputs.timeout-minutes`` + and ``/`` inside ref strings, so neither can be told apart from arithmetic by + inspection alone. +2. Callers of the reusable unit-test workflow keep the job timeout at or above + the test budget plus the setup ceilings plus the runner overhead below. + Otherwise the job deadline preempts pytest inside its own advertised budget, + which is the failure the split timeouts exist to prevent, and it shows up as + a cancelled shard whose tests were passing. A budget this check cannot resolve + is reported rather than skipped, so a mistyped input or matrix column surfaces + here instead of leaving the pair silently unchecked. +""" + +import re +import sys +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import yaml +from pydantic import BaseModel, Field, ValidationError + +REPO_ROOT: Final = Path(__file__).resolve().parent.parent.parent +WORKFLOWS_DIR: Final = REPO_ROOT / ".github" / "workflows" +BASE_WORKFLOW: Final = "./.github/workflows/_test-unit-base.yml" +BASE_WORKFLOW_PATH: Final = WORKFLOWS_DIR / "_test-unit-base.yml" + +# Runner time the job clock charges but no step owns: job init, the gaps between +# steps, and post-job cleanup. Without it a job capped at exactly test + setup +# would still preempt pytest inside its own budget. +JOB_OVERHEAD_MINUTES: Final = 5 + +EXPRESSION: Final = re.compile(r"\$\{\{(?P.*?)\}\}", re.DOTALL) +QUOTED: Final = re.compile(r"'[^']*'") +ARITHMETIC: Final = re.compile(r"[+*]") +MATRIX_REF: Final = re.compile(r"^\$\{\{\s*matrix\.(?P[\w-]+)\s*\}\}$") + + +class WorkflowStartupError(Exception): + pass + + +class ReusableCall(BaseModel): + uses: str | None = None + with_: Mapping[str, object] = Field(default_factory=dict, alias="with") + strategy: Mapping[str, object] = Field(default_factory=dict) + steps: tuple[Mapping[str, object], ...] = () + + model_config = {"populate_by_name": True} + + +class WorkflowFile(BaseModel): + jobs: Mapping[str, ReusableCall] = Field(default_factory=dict) + + +def parse_workflow(text: str) -> WorkflowFile | str: + parsed: Final = yaml.safe_load(text) + try: + return WorkflowFile.model_validate(parsed if isinstance(parsed, dict) else {}) + except ValidationError as exc: + return f"does not parse as a workflow: {exc.error_count()} schema error(s)" + + +def arithmetic_expressions(text: str) -> Iterator[str]: + for match in EXPRESSION.finditer(text): + body: Final = match.group("body") + if ARITHMETIC.search(QUOTED.sub("", body)): + yield body.strip() + + +def setup_ceiling_minutes(base_text: str) -> int: + """Sum the per-step timeouts on everything the base workflow runs before pytest.""" + base: Final = yaml.safe_load(base_text) + steps: Final = base["jobs"]["run"]["steps"] + return sum( + s["timeout-minutes"] + for s in steps + if s.get("name") != "Run tests" and isinstance(s.get("timeout-minutes"), int) + ) + + +def base_default(base_text: str, name: str) -> int: + base: Final = yaml.safe_load(base_text) + return base[True]["workflow_call"]["inputs"][name]["default"] + + +@dataclass(frozen=True, slots=True) +class Column: + """A budget the caller reads from one column of its own matrix.""" + + name: str + + +def budget_source(job: ReusableCall, key: str, fallback: int) -> int | Column | str: + """A caller passes a literal, or `${{ matrix.x }}` naming a column of its matrix. + + Anything else comes back as the reason it could not be read, since a budget + nothing can resolve has to be reported rather than passed over. + """ + value: Final = job.with_.get(key) + if value is None: + return fallback + if isinstance(value, int): + return value + + matrix_ref: Final = MATRIX_REF.match(str(value)) + if not matrix_ref: + return f"passes `{key}: {value}`, which is neither a number nor a `matrix` reference." + return Column(matrix_ref.group("key")) + + +def matrix_rows(job: ReusableCall) -> Sequence[Mapping[str, object]]: + matrix: Final = job.strategy.get("matrix", {}) + entries: Final = matrix.get("include", ()) if isinstance(matrix, dict) else () + return tuple(e for e in entries if isinstance(e, dict)) + + +def budget_pairs(job: ReusableCall, test_source: int | Column, job_source: int | Column) -> Iterator[tuple[int, int]]: + """Pair each shard's test budget with the job budget of that same shard. + + Matrix-sourced budgets resolve per `include` row, so two matrix columns are + read off the same row rather than cross-producted across rows. + """ + if isinstance(test_source, int) and isinstance(job_source, int): + yield test_source, job_source + return + + for row in matrix_rows(job): + test_budget = row.get(test_source.name) if isinstance(test_source, Column) else test_source + job_budget = row.get(job_source.name) if isinstance(job_source, Column) else job_source + if isinstance(test_budget, int) and isinstance(job_budget, int): + yield test_budget, job_budget + + +def unresolved_message(where: str, job: ReusableCall, sources: Sequence[int | Column]) -> str: + """Why no shard yielded a pair of budgets to compare. + + Naming only the columns that resolve nowhere keeps the message honest: a + column every row supplies is not what left the pair unchecked. + """ + rows: Final = matrix_rows(job) + missing: Final = tuple( + f"`matrix.{s.name}`" + for s in sources + if isinstance(s, Column) and not any(isinstance(row.get(s.name), int) for row in rows) + ) + if missing: + return ( + f"{where} reads a budget from {', '.join(missing)}, which no `include` row supplies " + "as a number, so the pair would go unchecked." + ) + return ( + f"{where} reads both budgets from its matrix, but no single `include` row supplies both " + "as numbers, so the pair would go unchecked." + ) + + +def job_errors(rel: Path, job_name: str, job: ReusableCall, ceiling: int, base_text: str) -> Iterator[str]: + where: Final = f"{rel}: job `{job_name}`" + test_source: Final = budget_source(job, "timeout-minutes", base_default(base_text, "timeout-minutes")) + job_source: Final = budget_source(job, "job-timeout-minutes", base_default(base_text, "job-timeout-minutes")) + sources: Final = (test_source, job_source) + + unreadable: Final = tuple(f"{where} {reason}" for reason in sources if isinstance(reason, str)) + if unreadable: + yield from unreadable + return + + pairs: Final = tuple(budget_pairs(job, test_source, job_source)) + if not pairs: + yield unresolved_message(where, job, sources) + return + + for test_budget, job_budget in pairs: + required = test_budget + ceiling + JOB_OVERHEAD_MINUTES + if job_budget < required: + yield ( + f"{where} gives pytest {test_budget}m but caps the job at " + f"{job_budget}m. Setup can use up to {ceiling}m plus {JOB_OVERHEAD_MINUTES}m of " + f"runner overhead, so the job deadline would preempt pytest; raise " + f"job-timeout-minutes to at least {required}." + ) + + +def timeout_contract_errors(rel: Path, workflow: WorkflowFile, ceiling: int, base_text: str) -> Iterator[str]: + for job_name, job in workflow.jobs.items(): + if job.uses == BASE_WORKFLOW: + yield from job_errors(rel, job_name, job, ceiling, base_text) + + +def workflow_errors(rel: Path, text: str, ceiling: int, base_text: str) -> Iterator[str]: + for expression in arithmetic_expressions(text): + yield ( + f"{rel}: `${{{{ {expression} }}}}` uses arithmetic, which GitHub expressions do not " + "support. The workflow will fail at startup with no jobs and no check run." + ) + + workflow: Final = parse_workflow(text) + if isinstance(workflow, str): + yield f"{rel}: {workflow}" + return + + yield from timeout_contract_errors(rel, workflow, ceiling, base_text) + + +def main() -> None: + base_text: Final = BASE_WORKFLOW_PATH.read_text() + ceiling: Final = setup_ceiling_minutes(base_text) + errors: Final = tuple( + error + for path in sorted(WORKFLOWS_DIR.glob("*.y*ml")) + for error in workflow_errors(path.relative_to(REPO_ROOT), path.read_text(), ceiling, base_text) + ) + + if errors: + raise WorkflowStartupError( + "Workflow startup invariants violated:\n - " + "\n - ".join(errors) + ) + + print(f"Workflow startup invariants hold (setup ceiling {ceiling}m)") + + +if __name__ == "__main__": + try: + main() + except WorkflowStartupError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1)