diff --git a/.github/workflows/check-lazy-openapi-snapshot.yml b/.github/workflows/check-lazy-openapi-snapshot.yml index e3fd21a2145..9d4945fc04d 100644 --- a/.github/workflows/check-lazy-openapi-snapshot.yml +++ b/.github/workflows/check-lazy-openapi-snapshot.yml @@ -53,7 +53,21 @@ jobs: - name: Fail if the snapshot is stale run: | - if ! uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot --check; then - echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::Lazy OpenAPI snapshot is out of sync with the proxy routes." - exit 1 - fi + set +e + uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot --check + status=$? + set -e + + case "$status" in + 0) + exit 0 + ;; + 2) + echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::Could not verify the lazy OpenAPI snapshot: a lazy feature failed to import, so the regenerated snapshot would be incomplete. This is a missing dependency in this job, not snapshot drift. See the log for the feature and error." + exit 1 + ;; + *) + echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::Lazy OpenAPI snapshot is out of sync with the proxy routes. Run 'python -m litellm.proxy._lazy_openapi_snapshot', commit the result, and re-run 'npm run gen:api' if schema.d.ts changes." + exit 1 + ;; + esac diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index e469d0f609c..a687ff2abaf 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -153,6 +153,10 @@ def generate_snapshot(*, strict: bool = False) -> dict[str, dict]: REGEN_COMMAND = "python -m litellm.proxy._lazy_openapi_snapshot" +EXIT_IN_SYNC = 0 +EXIT_DRIFTED = 1 +EXIT_CANNOT_VERIFY = 2 + def serialize_snapshot(fragments: dict[str, dict]) -> str: return json.dumps(fragments, indent=2, sort_keys=True) + "\n" @@ -164,13 +168,30 @@ def drifted_features(committed: dict[str, dict], fresh: dict[str, dict]) -> tupl def check_snapshot() -> int: - """Regenerate in memory and compare against the committed file. 0 when in sync.""" - fresh = generate_snapshot(strict=True) + """Regenerate in memory and compare against the committed file. + + Returns EXIT_IN_SYNC, EXIT_DRIFTED, or EXIT_CANNOT_VERIFY. A feature that will not + import is reported as its own outcome rather than as drift: the fragments it owns + are simply absent from the regen, which is indistinguishable from someone deleting + them, so calling it "stale" would send you off to regenerate a file that is fine. + """ + try: + fresh = generate_snapshot(strict=True) + except RuntimeError as exc: + sys.stderr.write( + f"cannot verify {SNAPSHOT_FILE.name}: a lazy feature failed to import.\n\n" + f"{exc}\n\n" + "This is an environment problem, not snapshot drift. Install the extras the\n" + "feature needs and re-run; do not regenerate the snapshot from here, or the\n" + "missing feature's fragments will be dropped from the committed file.\n" + ) + return EXIT_CANNOT_VERIFY + committed = load_snapshot() if committed is not None and serialize_snapshot(committed) == serialize_snapshot(fresh): sys.stdout.write(f"{SNAPSHOT_FILE.name} is in sync ({len(fresh)} feature fragments).\n") - return 0 + return EXIT_IN_SYNC drifted = drifted_features(committed or {}, fresh) sys.stderr.write( @@ -182,7 +203,7 @@ def check_snapshot() -> int: "then commit the updated litellm/proxy/_lazy_openapi_snapshot.json (and re-run\n" "`npm run gen:api` in ui/litellm-dashboard if schema.d.ts changes).\n" ) - return 1 + return EXIT_DRIFTED def _main(argv: Sequence[str]) -> int: diff --git a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py index 30255e7866c..e402b3bc163 100644 --- a/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py +++ b/tests/test_litellm/proxy/test_lazy_openapi_snapshot.py @@ -1,10 +1,12 @@ import sys from types import ModuleType, SimpleNamespace -import pytest - from litellm.proxy import _lazy_openapi_snapshot as snapshot_module from litellm.proxy._lazy_openapi_snapshot import ( + EXIT_CANNOT_VERIFY, + EXIT_DRIFTED, + EXIT_IN_SYNC, + REGEN_COMMAND, _normalize_operation_ids, _register_lazy_feature, check_snapshot, @@ -170,7 +172,7 @@ class TestCheckSnapshot: monkeypatch.setattr(snapshot_module, "generate_snapshot", lambda *, strict=False: fragments) monkeypatch.setattr(snapshot_module, "load_snapshot", lambda: dict(fragments)) - assert check_snapshot() == 0 + assert check_snapshot() == EXIT_IN_SYNC def test_fails_when_a_fragment_drifted(self, monkeypatch, capsys): monkeypatch.setattr( @@ -184,25 +186,45 @@ class TestCheckSnapshot: lambda: {"guardrails": {"paths": {"/old": {}}, "components": {"schemas": {}}}}, ) - assert check_snapshot() == 1 + assert check_snapshot() == EXIT_DRIFTED assert "guardrails" in capsys.readouterr().err def test_fails_when_the_file_is_absent(self, monkeypatch): monkeypatch.setattr(snapshot_module, "generate_snapshot", lambda *, strict=False: {"a": {"paths": {}}}) monkeypatch.setattr(snapshot_module, "load_snapshot", lambda: None) - assert check_snapshot() == 1 + assert check_snapshot() == EXIT_DRIFTED - def test_verification_refuses_to_run_on_an_incomplete_regen(self, monkeypatch): - """A feature that fails to import would silently look like a deleted fragment.""" + def test_an_unimportable_feature_is_not_reported_as_drift(self, monkeypatch, capsys): + """A skipped feature looks exactly like a deleted fragment, so it gets its own outcome. + + Reporting it as drift would tell you to regenerate a snapshot that is fine, and the + regen would then silently drop the missing feature's fragments. + """ def _boom(*, strict: bool = False): raise RuntimeError("cannot verify the snapshot: mcp_app (ImportError: no module)") monkeypatch.setattr(snapshot_module, "generate_snapshot", _boom) - with pytest.raises(RuntimeError, match="cannot verify the snapshot"): - check_snapshot() + assert check_snapshot() == EXIT_CANNOT_VERIFY + assert check_snapshot() != EXIT_DRIFTED + + stderr = capsys.readouterr().err + assert "mcp_app" in stderr + assert "not snapshot drift" in stderr + assert REGEN_COMMAND not in stderr + + def test_main_reports_the_import_failure_instead_of_raising(self, monkeypatch, capsys): + """The workflow branches on the exit code, so this must not escape as a traceback.""" + + def _boom(*, strict: bool = False): + raise RuntimeError("cannot verify the snapshot: mcp_app (ImportError: no module)") + + monkeypatch.setattr(snapshot_module, "generate_snapshot", _boom) + + assert snapshot_module._main(["--check"]) == EXIT_CANNOT_VERIFY + assert "mcp_app" in capsys.readouterr().err class TestSerializeSnapshot: