test(interactions): read the spec from a vendored snapshot and canary the live one daily (#48002)

* test(interactions): assert the live status enum is parsable instead of pinning it

* test(interactions): read the spec from a vendored snapshot and canary the live one daily

* test(interactions): run the canary on PRs touching its inputs, file an issue on a red run, pin the regenerate command

* test(interactions): take the live spec URL as a pytest option, assert its source, split the canary's tracking-issue job

* test(interactions): file the canary issue whenever its compliance step did not pass, match the issue title exactly, cover UrlContextResult

.github/workflows/interactions-spec-canary.yml

@@ -0,0 +1,91 @@
+name: Interactions spec canary
+
+on:
+  schedule:
+    - cron: "17 7 * * *"
+  pull_request:
+    paths:
+      - .github/workflows/interactions-spec-canary.yml
+      - tests/unit/interactions/**
+      - litellm/types/interactions/**
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency: interactions-spec-canary
+
+jobs:
+  interactions-spec-canary:
+    name: interactions-spec-canary
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    outputs:
+      compliance: ${{ steps.compliance.outcome }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
+
+      - name: Run the compliance tests against Google's live spec
+        id: compliance
+        run: |
+          uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
+            --interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
+            --tb=short -vv -p no:rerunfailures -p no:pytest-retry \
+            --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
+
+  tracking-issue:
+    name: tracking-issue
+    needs: interactions-spec-canary
+    if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      issues: write
+    steps:
+      - name: File the tracking issue unless the compliance step passed, close it when it did
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        with:
+          script: |
+            const title = "Interactions spec canary is red";
+            const outcome = process.env.COMPLIANCE_OUTCOME;
+            const runUrl = process.env.RUN_URL;
+            const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
+            const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
+            const open = found.items.find((issue) => issue.title === title);
+            if (outcome === "success") {
+              if (open === undefined) return;
+              await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
+              await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
+              return;
+            }
+            const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
+            if (open !== undefined) {
+              await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
+              return;
+            }
+            await github.rest.issues.create({
+              ...context.repo,
+              title,
+              assignees: ["mateo-berri"],
+              body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
+            });

litellm/types/interactions/README.md

@@ -9,16 +9,38 @@ https://ai.google.dev/static/api/interactions.openapi.json
 
 ### How to Regenerate
 
-When the API spec changes, regenerate the types with:
+The compliance tests in `tests/unit/interactions/test_openapi_compliance.py` read the snapshot at
+`tests/unit/interactions/interactions.openapi.json`, so a PR never depends on Google's live spec. The scheduled
+`interactions-spec-canary` workflow runs those tests against the live spec once a day (`--interactions-spec-url`),
+files the issue "Interactions spec canary is red" on a red run and closes it once a run is green again. A red run
+means the spec moved in a way these types no longer cover
+
+When the red run names one new status, the small fix is a hand edit adding that member to the enum the failing
+schema's model carries, the way `queued` was added in #34135. When the spec moved further than that, refresh the
+snapshot and regenerate the types from it:
 
 ```bash
-uv tool run --from datamodel-code-generator datamodel-codegen \
-    --url "https://ai.google.dev/static/api/interactions.openapi.json" \
+curl -fsSL https://ai.google.dev/static/api/interactions.openapi.json \
+    -o tests/unit/interactions/interactions.openapi.json
+uv tool run --from datamodel-code-generator==0.66.0 datamodel-codegen \
+    --input tests/unit/interactions/interactions.openapi.json \
+    --input-file-type openapi \
     --output litellm/types/interactions/generated.py \
     --output-model-type pydantic_v2.BaseModel \
-    --target-python-version 3.9
+    --target-python-version 3.10 \
+    --base-class litellm.types.llms.base.LiteLLMBaseModel
 ```
 
+The pin is the same `datamodel-code-generator` version `scripts/generate_trace_types.py` uses. The generator names
+inline enums by order of appearance (`Status`, `Status1`, `Status2`, ...), so a regeneration can rename them. The
+compliance tests resolve each status enum through its model's `status` field, but
+`tests/unit/interactions/test_background_cost_polling.py` and
+`tests/unit/proxy/hooks/test_proxy_track_cost_callback.py` import `Status1` by name and need the new name, and
+`litellm/types/interactions/__init__.py` exports the interaction status enum as `InteractionStatus`, which has to
+point at the enum the regenerated `Interaction.status` carries (a compliance test checks their members match). The
+spec has grown a lot since the 2025-12-16 generation (agents, triggers, credentials, environments), so a
+regeneration today is a review-sized change rather than a refresh
+
 Then add the LiteLLM-specific types at the bottom of the generated file:
 - `InteractionsAPIResponse`
 - `InteractionsAPIStreamingResponse`

tests/unit/AGENTS.md

@@ -1,6 +1,9 @@
 # tests/unit
 
-In-process. No network or subprocess
+In-process. No network or subprocess. `interactions/test_openapi_compliance.py` reads the spec snapshot beside it,
+and only the `interactions-spec-canary` workflow passes `--interactions-spec-url` (registered in
+`interactions/conftest.py`, so name that file or directory on the command line) with `ai.google.dev` allowed, to run
+the same tests against Google's live spec
 
 Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures
 and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed

tests/unit/interactions/conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--interactions-spec-url",
+        default=None,
+        help="Run the Interactions OpenAPI compliance tests against this live spec instead of the vendored snapshot",
+    )

tests/unit/interactions/test_openapi_compliance.py

@@ -1,41 +1,56 @@
 """
 OpenAPI compliance tests for Google Interactions API.
 
-Validates that our SDK requests/responses match the OpenAPI spec at:
-https://ai.google.dev/static/api/interactions.openapi.json
+Validates that our SDK requests/responses match the OpenAPI spec snapshot in
+interactions.openapi.json, taken from https://ai.google.dev/static/api/interactions.openapi.json
+(litellm/types/interactions/README.md says how to refresh it). The scheduled
+interactions-spec-canary workflow runs the same tests against the live spec by passing
+--interactions-spec-url, so a change Google makes shows up there, not in every PR.
 
 Run with: pytest tests/unit/interactions/test_openapi_compliance.py -v
 """
 
 import json
-import os
 import re
-from typing import Any, Dict, Final
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Final, get_args
 from unittest.mock import MagicMock, patch
 
 import httpx
 import pytest
 from openapi_core import OpenAPI
+from pydantic import BaseModel, TypeAdapter
 
 OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json"
+OPENAPI_SPEC_SNAPSHOT: Final = Path(__file__).with_name("interactions.openapi.json")
+SPEC_URL_OPTION: Final = TypeAdapter(str | None)
 
 
-def _load_openapi_spec_dict() -> Dict[str, Any]:
-    """
-    Load the OpenAPI spec JSON.
+@dataclass(frozen=True, slots=True)
+class LoadedSpec:
+    source: str
+    spec: Dict[str, Any]
 
-    In CI or offline environments, network access may not be available.
-    In that case, gracefully skip these tests instead of erroring.
-    """
-    try:
-        response = httpx.get(OPENAPI_SPEC_URL, timeout=5.0)
-        response.raise_for_status()
-        return response.json()
-    except Exception as e:  # pragma: no cover - defensive, env-dependent
-        pytest.skip(
-            f"Skipping Google Interactions OpenAPI compliance tests - "
-            f"unable to load spec from {OPENAPI_SPEC_URL}: {e}"
-        )
+
+def _status_enum_of(model: type[BaseModel]) -> type[Enum]:
+    annotation: Final = model.model_fields["status"].annotation
+    candidates: Final = (annotation, *get_args(annotation))
+    return next(candidate for candidate in candidates if isinstance(candidate, type) and issubclass(candidate, Enum))
+
+
+def _requested_spec_url(request: pytest.FixtureRequest) -> str | None:
+    return SPEC_URL_OPTION.validate_python(request.config.getoption("interactions_spec_url"))
+
+
+def _load_openapi_spec(live_url: str | None) -> LoadedSpec:
+    if live_url is None:
+        return LoadedSpec(source=str(OPENAPI_SPEC_SNAPSHOT), spec=json.loads(OPENAPI_SPEC_SNAPSHOT.read_text()))
+    with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0) as client:
+        response: Final = client.get(live_url)
+    response.raise_for_status()
+    return LoadedSpec(source=str(response.url), spec=response.json())
 
 
 def _model_create_request_schema(spec_dict: Dict[str, Any]) -> Dict[str, Any]:
@@ -65,9 +80,13 @@ def _declared_type_value(variant_schema: Dict[str, Any]) -> Any:
 
 
 @pytest.fixture(scope="module")
-def spec_dict() -> Dict[str, Any]:
-    """Load raw spec dict for manual validation."""
-    return _load_openapi_spec_dict()
+def loaded_spec(request: pytest.FixtureRequest) -> LoadedSpec:
+    return _load_openapi_spec(_requested_spec_url(request))
+
+
+@pytest.fixture(scope="module")
+def spec_dict(loaded_spec: LoadedSpec) -> Dict[str, Any]:
+    return loaded_spec.spec
 
 
 @pytest.fixture(scope="module")
@@ -76,6 +95,18 @@ def openapi_spec(spec_dict: Dict[str, Any]) -> OpenAPI:
     return OpenAPI.from_dict(spec_dict)
 
 
+def test_spec_comes_from_the_requested_source(loaded_spec: LoadedSpec, request: pytest.FixtureRequest) -> None:
+    assert loaded_spec.source == (_requested_spec_url(request) or str(OPENAPI_SPEC_SNAPSHOT))
+
+
+def test_interaction_status_export_carries_the_interaction_models_statuses() -> None:
+    from litellm.types.interactions import InteractionStatus, generated
+
+    exported: Final = frozenset(member.value for member in InteractionStatus)
+    modelled: Final = frozenset(member.value for member in _status_enum_of(generated.Interaction))
+    assert exported == modelled, sorted(exported ^ modelled)
+
+
 class TestRequestCompliance:
     """Tests that our request bodies match the OpenAPI spec."""
 
@@ -249,17 +280,24 @@ class TestResponseCompliance:
             assert field in schema["properties"], f"Output field '{field}' not in spec"
             print(f"✓ Output field '{field}' exists in spec")
 
-    def test_status_enum_values(self, spec_dict):
-        from pydantic import TypeAdapter
+    @pytest.mark.parametrize(
+        "schema_name",
+        [
+            "Interaction",
+            "InteractionStatusUpdate",
+            "CreateModelInteractionParams",
+            "CreateAgentInteractionParams",
+            "UrlContextResult",
+        ],
+    )
+    def test_status_enum_values(self, spec_dict, schema_name):
+        from litellm.types.interactions import generated
 
-        from litellm.types.interactions import Interaction, InteractionStatus
-
-        schema: Final = spec_dict["components"]["schemas"]["Interaction"]
-        statuses: Final = TypeAdapter(tuple[str, ...]).validate_python(schema["properties"]["status"]["enum"])
-        assert statuses
-        for status in statuses:
-            assert Interaction.model_validate({"id": "interaction-status", "status": status}).status.value == status
-            assert InteractionStatus(status).value == status
+        schema = spec_dict["components"]["schemas"][schema_name]
+        spec_statuses: Final = frozenset(schema["properties"]["status"]["enum"])
+        parsed_statuses: Final = frozenset(member.value for member in _status_enum_of(getattr(generated, schema_name)))
+        assert spec_statuses
+        assert spec_statuses <= parsed_statuses, sorted(spec_statuses - parsed_statuses)
 
     def test_usage_schema(self, spec_dict):
         """Verify Usage schema fields."""