Merge pull request #36417 from BerriAI/litellm_cache_prisma_ci_binaries

ci: cache Prisma CLI and engine binaries, split test timeout from setup

.github/actions/cache-prisma-binaries/action.yml

@@ -0,0 +1,40 @@
+name: "Cache Prisma binaries"
+description: >-
+  Cache the Prisma CLI and engine binaries that `prisma generate` downloads, so
+  only the first job on a given prisma-client-py version pays for the download.
+
+  prisma-client-py shells out to `npm install prisma@<version>` whenever its
+  binary cache directory has no CLI entrypoint, which pulls ~85 MB of query and
+  schema engines over the network. That normally takes a few seconds, but it is
+  unbounded: one shard of a proxy-db run took 5m18s on that single step versus
+  3.8s on its eleven siblings, which pushed the job past its timeout and got a
+  fully passing test run cancelled.
+
+  Callers must not set PRISMA_BINARY_CACHE_DIR. The prisma-client-py default
+  (~/.cache/prisma-python/binaries/<prisma-version>/<engine-version>) is already
+  keyed by both versions, so a cache entry can never be served to a run that
+  expects different binaries.
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve prisma-client-py version
+      id: version
+      shell: bash
+      run: |
+        version="$(grep -A1 '^name = "prisma"$' uv.lock | sed -n 's/^version = "\(.*\)"$/\1/p' | head -1)"
+        if [ -z "${version}" ]; then
+          echo "could not resolve the prisma package version from uv.lock" >&2
+          exit 1
+        fi
+        echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+    - name: Restore Prisma binaries
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        # ~/.cache/prisma-python holds the npm install tree prisma-client-py
+        # drives; ~/.cache/prisma is where @prisma/engines stages its downloads.
+        path: |
+          ~/.cache/prisma-python
+          ~/.cache/prisma
+        key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}

.github/workflows/_test-unit-base.yml

@@ -18,10 +18,25 @@ on:
         type: number
         default: 2
       timeout-minutes:
-        description: "Job timeout in minutes"
+        description: >-
+          Timeout for the test step alone. Setup (checkout, dependency install,
+          Prisma client generation) gets its own allowance on top, so a slow
+          runner or a cold binary download can never cancel passing tests.
         required: false
         type: number
         default: 20
+      job-timeout-minutes:
+        description: >-
+          Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
+          the per-step ceilings on the setup steps below, and 5 for the runner
+          overhead the job clock charges but no step owns (job init, step
+          transitions, post-job cleanup). That headroom is what makes the test
+          budget a floor rather than a hope, since setup cannot overrun into it
+          without failing its own step first. GitHub expressions have no
+          arithmetic, so the sum is passed in rather than computed.
+        required: false
+        type: number
+        default: 55
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -44,30 +59,35 @@ jobs:
   run:
     name: Run tests
     runs-on: ubuntu-latest
-    timeout-minutes: ${{ inputs.timeout-minutes }}
+    timeout-minutes: ${{ inputs.job-timeout-minutes }}
     outputs:
       decision: ${{ steps.changes.outputs.decision }}
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
         with:
           persist-credentials: false
 
       - name: Detect backend-relevant changes
         id: changes
+        timeout-minutes: 2
         uses: ./.github/actions/detect-backend-changes
 
       - name: Set up Python
+        timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        timeout-minutes: 3
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -79,18 +99,24 @@ jobs:
 
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 8
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        timeout-minutes: 3
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests
         if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ inputs.timeout-minutes }}
         env:
           TEST_PATH: ${{ inputs.test-path }}
           MAX_FAILURES: ${{ inputs.max-failures }}

.github/workflows/check-ui-api-types.yml

@@ -71,10 +71,12 @@ jobs:
         if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.relevant == 'true'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.relevant == 'true'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Set up Node.js

.github/workflows/mutation-test.yml

@@ -57,9 +57,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/publish-basedpyright-base-counts.yml

@@ -43,12 +43,13 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       # The gate provisions its own measurement env (.venv-typecheck: a frozen
       # uv sync of its canonical dependency groups plus a generated Prisma
       # client), so no install step here can drift from what local runs measure.
       - name: Emit basedpyright counts for HEAD
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
           counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)

.github/workflows/test-code-quality.yml

@@ -65,6 +65,12 @@ jobs:
       - name: check_provider_folders_documented
         run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
 
+      - name: check_prisma_binary_cache
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
+
+      - name: check_workflow_startup_safety
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-linting.yml

@@ -71,12 +71,13 @@ jobs:
         run: |
           uv sync --frozen --group proxy-dev --group e2e-dev
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
       # only after `prisma generate` writes prisma/client.py et al. Without this the
       # DB wrappers typed against the generated client would degrade to Unknown.
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
@@ -119,7 +120,6 @@ jobs:
       - name: Check basedpyright budget (delta vs base)
         env:
           GH_TOKEN: ${{ github.token }}
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
 

.github/workflows/test-terraform-provider.yml

@@ -92,9 +92,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-documentation.yml

@@ -65,10 +65,12 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-proxy-db.yml

@@ -28,6 +28,10 @@ concurrency:
 #     Most of a shard's time is pytest plugin load + xdist worker imports +
 #     pytest-cov instrumentation, not the tests themselves. Keeping per-shard
 #     work low and matching worker count to runner cores is what controls it.
+#   * `timeout` bounds the pytest step only. Checkout, dependency install, and
+#     Prisma client generation draw on a separate allowance in the base
+#     workflow, so slow setup shows up as a slow job rather than as a
+#     cancelled shard whose tests were passing.
 #   * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
 #     oversubscribes 2x and workers fight for CPU during their cold-start
 #     imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).

.github/workflows/test-unit-proxy-endpoints.yml

@@ -76,4 +76,5 @@ jobs:
       workers: 4
       reruns: 2
       timeout-minutes: 60
+      job-timeout-minutes: 95
       artifact-name: proxy-server

.github/workflows/test-unit-proxy-legacy.yml

@@ -82,10 +82,12 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/weekly_load_anomaly.yml

@@ -51,9 +51,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

tests/code_coverage_tests/check_prisma_binary_cache.py

@@ -0,0 +1,143 @@
+"""Guard the CI cache for Prisma's CLI and engine binaries.
+
+``prisma generate`` shells out to ``npm install prisma@<version>`` whenever the
+prisma-client-py binary cache directory has no CLI entrypoint, pulling ~85 MB of
+engines over the network. The download is normally seconds and occasionally
+minutes, and a job timeout cannot tell the difference from a hung test, so an
+uncached job is one slow npm response away from cancelling a passing test run.
+
+Three invariants keep that download off the critical path:
+
+1. No workflow sets ``PRISMA_BINARY_CACHE_DIR``. The prisma-client-py default is
+   ``~/.cache/prisma-python/binaries/<prisma-version>/<engine-version>``, already
+   keyed by both versions and the only path the cache action restores. Pointing
+   it elsewhere (``runner.temp`` especially, which is wiped every job) silently
+   guarantees a cold download.
+2. Every job that generates the client also restores the cache.
+3. The cache key resolves to a real version from ``uv.lock``. The action fails
+   the job when it cannot, so a lock format change must break here instead.
+"""
+
+import re
+import sys
+from collections.abc import Iterator, Mapping
+from pathlib import Path
+from typing import Final
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError
+
+REPO_ROOT: Final = Path(__file__).resolve().parent.parent.parent
+WORKFLOWS_DIR: Final = REPO_ROOT / ".github" / "workflows"
+UV_LOCK: Final = REPO_ROOT / "uv.lock"
+CACHE_ACTION: Final = "./.github/actions/cache-prisma-binaries"
+
+# Commands that reach the prisma binary cache: a direct generate, or a script
+# that runs one on the caller's behalf.
+PRISMA_GENERATE_MARKERS: Final = ("prisma generate", "type_check_gate.py")
+
+
+class PrismaBinaryCacheError(Exception):
+    pass
+
+
+def resolve_prisma_version(lock_text: str) -> str | None:
+    """Mirror of the shell lookup in the cache action's version step."""
+    match: Final = re.search(
+        r'^name = "prisma"\n^version = "(?P<version>[^"]+)"$',
+        lock_text,
+        re.MULTILINE,
+    )
+    return match.group("version") if match else None
+
+
+class WorkflowStep(BaseModel):
+    """The two step fields this guard reads; every other key is ignored."""
+
+    run: str | None = None
+    uses: str | None = None
+
+    def generates_prisma_client(self) -> bool:
+        return self.run is not None and any(m in self.run for m in PRISMA_GENERATE_MARKERS)
+
+    def restores_cache(self) -> bool:
+        return self.uses == CACHE_ACTION
+
+
+class WorkflowJob(BaseModel):
+    # Absent for jobs that delegate to a reusable workflow via a job-level `uses`.
+    steps: tuple[WorkflowStep, ...] = ()
+
+
+class Workflow(BaseModel):
+    jobs: Mapping[str, WorkflowJob] = Field(default_factory=dict)
+
+
+def parse_workflow(text: str) -> Workflow | str:
+    """Validate untyped YAML at the boundary so the checks below stay typed.
+
+    Returns the parsed workflow, or a description of why it could not be read.
+    """
+    parsed: Final = yaml.safe_load(text)
+    try:
+        return Workflow.model_validate(parsed if isinstance(parsed, dict) else {})
+    except ValidationError as exc:
+        return f"does not parse as a workflow: {exc.error_count()} schema error(s)"
+
+
+def lock_errors(lock_text: str) -> Iterator[str]:
+    if not resolve_prisma_version(lock_text):
+        yield (
+            "uv.lock has no resolvable `prisma` package version. The version step "
+            f"in {CACHE_ACTION} greps the same shape and will fail every job that "
+            "generates the Prisma client."
+        )
+
+
+def workflow_errors(rel: Path, text: str) -> Iterator[str]:
+    if "PRISMA_BINARY_CACHE_DIR" in text:
+        yield (
+            f"{rel}: sets PRISMA_BINARY_CACHE_DIR. Leave it unset so the binaries "
+            f"land in the version-keyed default path the {CACHE_ACTION} action restores."
+        )
+
+    workflow: Final = parse_workflow(text)
+    if isinstance(workflow, str):
+        yield f"{rel}: {workflow}"
+        return
+
+    for job_name, job in workflow.jobs.items():
+        if any(s.generates_prisma_client() for s in job.steps) and not any(
+            s.restores_cache() for s in job.steps
+        ):
+            yield (
+                f"{rel}: job `{job_name}` generates the Prisma client without a "
+                f"`uses: {CACHE_ACTION}` step, so it downloads ~85 MB of engines "
+                "on every run."
+            )
+
+
+def main() -> None:
+    errors: Final = (
+        *lock_errors(UV_LOCK.read_text()),
+        *(
+            error
+            for path in sorted(WORKFLOWS_DIR.glob("*.y*ml"))
+            for error in workflow_errors(path.relative_to(REPO_ROOT), path.read_text())
+        ),
+    )
+
+    if errors:
+        raise PrismaBinaryCacheError(
+            "Prisma binary cache invariants violated:\n  - " + "\n  - ".join(errors)
+        )
+
+    print("Prisma binary cache invariants hold across .github/workflows/")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except PrismaBinaryCacheError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)

tests/code_coverage_tests/check_workflow_startup_safety.py

@@ -0,0 +1,239 @@
+"""Catch workflow mistakes that GitHub reports as nothing at all.
+
+A workflow whose YAML is valid but whose expressions are not fails at *startup*:
+the run is marked failed, no jobs are created, and no check run is ever posted.
+Nothing turns red on the PR, so an entire test suite can silently stop running
+while the checks list stays green. These invariants have to be enforced here
+because CI cannot enforce them on itself.
+
+1. No arithmetic inside ``${{ }}``. GitHub expressions support grouping, index,
+   dereference, ``!``, the comparisons, ``&&`` and ``||``, and nothing else. A
+   ``${{ a + b }}`` is a startup failure, not a value. Only ``+`` and ``*`` are
+   flagged: ``-`` appears in hyphenated input names like ``inputs.timeout-minutes``
+   and ``/`` inside ref strings, so neither can be told apart from arithmetic by
+   inspection alone.
+2. Callers of the reusable unit-test workflow keep the job timeout at or above
+   the test budget plus the setup ceilings plus the runner overhead below.
+   Otherwise the job deadline preempts pytest inside its own advertised budget,
+   which is the failure the split timeouts exist to prevent, and it shows up as
+   a cancelled shard whose tests were passing. A budget this check cannot resolve
+   is reported rather than skipped, so a mistyped input or matrix column surfaces
+   here instead of leaving the pair silently unchecked.
+"""
+
+import re
+import sys
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError
+
+REPO_ROOT: Final = Path(__file__).resolve().parent.parent.parent
+WORKFLOWS_DIR: Final = REPO_ROOT / ".github" / "workflows"
+BASE_WORKFLOW: Final = "./.github/workflows/_test-unit-base.yml"
+BASE_WORKFLOW_PATH: Final = WORKFLOWS_DIR / "_test-unit-base.yml"
+
+# Runner time the job clock charges but no step owns: job init, the gaps between
+# steps, and post-job cleanup. Without it a job capped at exactly test + setup
+# would still preempt pytest inside its own budget.
+JOB_OVERHEAD_MINUTES: Final = 5
+
+EXPRESSION: Final = re.compile(r"\$\{\{(?P<body>.*?)\}\}", re.DOTALL)
+QUOTED: Final = re.compile(r"'[^']*'")
+ARITHMETIC: Final = re.compile(r"[+*]")
+MATRIX_REF: Final = re.compile(r"^\$\{\{\s*matrix\.(?P<key>[\w-]+)\s*\}\}$")
+
+
+class WorkflowStartupError(Exception):
+    pass
+
+
+class ReusableCall(BaseModel):
+    uses: str | None = None
+    with_: Mapping[str, object] = Field(default_factory=dict, alias="with")
+    strategy: Mapping[str, object] = Field(default_factory=dict)
+    steps: tuple[Mapping[str, object], ...] = ()
+
+    model_config = {"populate_by_name": True}
+
+
+class WorkflowFile(BaseModel):
+    jobs: Mapping[str, ReusableCall] = Field(default_factory=dict)
+
+
+def parse_workflow(text: str) -> WorkflowFile | str:
+    parsed: Final = yaml.safe_load(text)
+    try:
+        return WorkflowFile.model_validate(parsed if isinstance(parsed, dict) else {})
+    except ValidationError as exc:
+        return f"does not parse as a workflow: {exc.error_count()} schema error(s)"
+
+
+def arithmetic_expressions(text: str) -> Iterator[str]:
+    for match in EXPRESSION.finditer(text):
+        body: Final = match.group("body")
+        if ARITHMETIC.search(QUOTED.sub("", body)):
+            yield body.strip()
+
+
+def setup_ceiling_minutes(base_text: str) -> int:
+    """Sum the per-step timeouts on everything the base workflow runs before pytest."""
+    base: Final = yaml.safe_load(base_text)
+    steps: Final = base["jobs"]["run"]["steps"]
+    return sum(
+        s["timeout-minutes"]
+        for s in steps
+        if s.get("name") != "Run tests" and isinstance(s.get("timeout-minutes"), int)
+    )
+
+
+def base_default(base_text: str, name: str) -> int:
+    base: Final = yaml.safe_load(base_text)
+    return base[True]["workflow_call"]["inputs"][name]["default"]
+
+
+@dataclass(frozen=True, slots=True)
+class Column:
+    """A budget the caller reads from one column of its own matrix."""
+
+    name: str
+
+
+def budget_source(job: ReusableCall, key: str, fallback: int) -> int | Column | str:
+    """A caller passes a literal, or `${{ matrix.x }}` naming a column of its matrix.
+
+    Anything else comes back as the reason it could not be read, since a budget
+    nothing can resolve has to be reported rather than passed over.
+    """
+    value: Final = job.with_.get(key)
+    if value is None:
+        return fallback
+    if isinstance(value, int):
+        return value
+
+    matrix_ref: Final = MATRIX_REF.match(str(value))
+    if not matrix_ref:
+        return f"passes `{key}: {value}`, which is neither a number nor a `matrix` reference."
+    return Column(matrix_ref.group("key"))
+
+
+def matrix_rows(job: ReusableCall) -> Sequence[Mapping[str, object]]:
+    matrix: Final = job.strategy.get("matrix", {})
+    entries: Final = matrix.get("include", ()) if isinstance(matrix, dict) else ()
+    return tuple(e for e in entries if isinstance(e, dict))
+
+
+def budget_pairs(job: ReusableCall, test_source: int | Column, job_source: int | Column) -> Iterator[tuple[int, int]]:
+    """Pair each shard's test budget with the job budget of that same shard.
+
+    Matrix-sourced budgets resolve per `include` row, so two matrix columns are
+    read off the same row rather than cross-producted across rows.
+    """
+    if isinstance(test_source, int) and isinstance(job_source, int):
+        yield test_source, job_source
+        return
+
+    for row in matrix_rows(job):
+        test_budget = row.get(test_source.name) if isinstance(test_source, Column) else test_source
+        job_budget = row.get(job_source.name) if isinstance(job_source, Column) else job_source
+        if isinstance(test_budget, int) and isinstance(job_budget, int):
+            yield test_budget, job_budget
+
+
+def unresolved_message(where: str, job: ReusableCall, sources: Sequence[int | Column]) -> str:
+    """Why no shard yielded a pair of budgets to compare.
+
+    Naming only the columns that resolve nowhere keeps the message honest: a
+    column every row supplies is not what left the pair unchecked.
+    """
+    rows: Final = matrix_rows(job)
+    missing: Final = tuple(
+        f"`matrix.{s.name}`"
+        for s in sources
+        if isinstance(s, Column) and not any(isinstance(row.get(s.name), int) for row in rows)
+    )
+    if missing:
+        return (
+            f"{where} reads a budget from {', '.join(missing)}, which no `include` row supplies "
+            "as a number, so the pair would go unchecked."
+        )
+    return (
+        f"{where} reads both budgets from its matrix, but no single `include` row supplies both "
+        "as numbers, so the pair would go unchecked."
+    )
+
+
+def job_errors(rel: Path, job_name: str, job: ReusableCall, ceiling: int, base_text: str) -> Iterator[str]:
+    where: Final = f"{rel}: job `{job_name}`"
+    test_source: Final = budget_source(job, "timeout-minutes", base_default(base_text, "timeout-minutes"))
+    job_source: Final = budget_source(job, "job-timeout-minutes", base_default(base_text, "job-timeout-minutes"))
+    sources: Final = (test_source, job_source)
+
+    unreadable: Final = tuple(f"{where} {reason}" for reason in sources if isinstance(reason, str))
+    if unreadable:
+        yield from unreadable
+        return
+
+    pairs: Final = tuple(budget_pairs(job, test_source, job_source))
+    if not pairs:
+        yield unresolved_message(where, job, sources)
+        return
+
+    for test_budget, job_budget in pairs:
+        required = test_budget + ceiling + JOB_OVERHEAD_MINUTES
+        if job_budget < required:
+            yield (
+                f"{where} gives pytest {test_budget}m but caps the job at "
+                f"{job_budget}m. Setup can use up to {ceiling}m plus {JOB_OVERHEAD_MINUTES}m of "
+                f"runner overhead, so the job deadline would preempt pytest; raise "
+                f"job-timeout-minutes to at least {required}."
+            )
+
+
+def timeout_contract_errors(rel: Path, workflow: WorkflowFile, ceiling: int, base_text: str) -> Iterator[str]:
+    for job_name, job in workflow.jobs.items():
+        if job.uses == BASE_WORKFLOW:
+            yield from job_errors(rel, job_name, job, ceiling, base_text)
+
+
+def workflow_errors(rel: Path, text: str, ceiling: int, base_text: str) -> Iterator[str]:
+    for expression in arithmetic_expressions(text):
+        yield (
+            f"{rel}: `${{{{ {expression} }}}}` uses arithmetic, which GitHub expressions do not "
+            "support. The workflow will fail at startup with no jobs and no check run."
+        )
+
+    workflow: Final = parse_workflow(text)
+    if isinstance(workflow, str):
+        yield f"{rel}: {workflow}"
+        return
+
+    yield from timeout_contract_errors(rel, workflow, ceiling, base_text)
+
+
+def main() -> None:
+    base_text: Final = BASE_WORKFLOW_PATH.read_text()
+    ceiling: Final = setup_ceiling_minutes(base_text)
+    errors: Final = tuple(
+        error
+        for path in sorted(WORKFLOWS_DIR.glob("*.y*ml"))
+        for error in workflow_errors(path.relative_to(REPO_ROOT), path.read_text(), ceiling, base_text)
+    )
+
+    if errors:
+        raise WorkflowStartupError(
+            "Workflow startup invariants violated:\n  - " + "\n  - ".join(errors)
+        )
+
+    print(f"Workflow startup invariants hold (setup ceiling {ceiling}m)")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except WorkflowStartupError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)