diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index e8047c8eb..adf3af883 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -20,5 +20,15 @@ jobs: java-version: 21 cache: maven + - name: Set up Go for integration tests + uses: actions/setup-go@v5 + with: + go-version-file: github-sync-go/go.mod + cache-dependency-path: github-sync-go/go.mod + + - name: Compile Go integration test dependencies + working-directory: github-sync-go + run: go test ./internal/worker -run '^$' + - name: Backend tests run: cd server && ./mvnw -B test diff --git a/.github/workflows/migration-preflight.yml b/.github/workflows/migration-preflight.yml new file mode 100644 index 000000000..f80e3056d --- /dev/null +++ b/.github/workflows/migration-preflight.yml @@ -0,0 +1,45 @@ +name: Migration Preflight + +on: + pull_request_target: + types: [opened, reopened, synchronize, closed, edited, ready_for_review] + schedule: + - cron: '17 */6 * * *' + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +concurrency: + group: migration-preflight + cancel-in-progress: true + +jobs: + preflight: + name: Advisory Flyway version scan + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + steps: + # Never check out a PR head or run SQL. This workflow only runs trusted + # default-branch code, including for pull requests from forks. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + id: checkout + continue-on-error: true + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Summarize open PR migration versions + id: scan + if: steps.checkout.outcome == 'success' + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/migration-preflight.py + + - name: Report unavailable scan + if: ${{ always() && (steps.checkout.outcome != 'success' || steps.scan.outcome != 'success') }} + run: | + printf '%s\n' '## Flyway migration preflight' '' '**Warning: scan unavailable. No clearance issued.** Check this workflow run for checkout, runtime or timeout failures.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr-scripts.yml b/.github/workflows/pr-scripts.yml index 9fbd44ac7..86afde8fe 100644 --- a/.github/workflows/pr-scripts.yml +++ b/.github/workflows/pr-scripts.yml @@ -30,6 +30,7 @@ on: - '.github/workflows/publish-chart.yml' - '.github/workflows/security.yml' - '.github/workflows/pr-scripts.yml' + - '.github/workflows/migration-preflight.yml' - '**/*.py' permissions: @@ -66,3 +67,4 @@ jobs: - run: bash scripts/tests/server-image-compat-test.sh - run: bash scripts/tests/workflow-security-test.sh - run: python3 -m unittest discover -s scripts/tests -p 'test_deepwiki_workflow.py' -v + - run: python3 -m unittest discover -s scripts/tests -p 'test_migration_preflight.py' -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc9ad708a..dfc470b6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,3 +107,33 @@ Do not open public issues for suspected security vulnerabilities. Use GitHub Security Advisories or your internal security process to report them privately to the maintainers. + +## Flyway migration version advisory + +The **Migration Preflight** workflow reads file metadata for all open PRs on PR +changes, every six hours, or via **Actions → Migration Preflight → Run workflow**. +See its Actions run summary for migration numbers introduced by more than one PR. +It checks `server/skillhub-app/src/main/resources/db/migration/V__*.sql` additions, +including renames that introduce a different migration number. Pure renames with +the same number, modified/deleted migrations and conflicts with migrations already +merged into the base branch are outside this advisory's scope. + +This is a snapshot and a non-blocking warning, not a reservation or merge gate. +If a scan is incomplete (for example API permissions, file-list truncation or a +PR changing during collection), its summary says so; it must not be interpreted +as confirmation of no conflict. The script exits nonzero on an incomplete scan; +the workflow keeps this advisory and explicitly adds an unavailable-scan warning. PRs with 3,000 or more changed files are conservatively +reported as incomplete because GitHub caps the files endpoint at 3,000 entries. +Coordinate numbering with the other PR's author, update your migration filename, +and rerun the relevant database/migration tests before merging. The workflow does +not renumber files, post comments, fetch raw SQL files or execute PR code. The +GitHub files API may include diff patches; the script ignores them and only uses +file names/status, PR numbers and revision metadata. It uses only +repository-content and pull-request read permissions and runs the default branch's +script, so workflow/script changes take effect only after merging into that branch. + +Local offline regression checks (Python standard library only): + +```bash +python3 -m unittest discover -s scripts/tests -p 'test_migration_preflight.py' -v +``` diff --git a/scripts/migration-preflight.py b/scripts/migration-preflight.py new file mode 100644 index 000000000..97451cc9d --- /dev/null +++ b/scripts/migration-preflight.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Read GitHub PR file metadata and summarize possible Flyway version collisions.""" + +import html +import json +import os +from pathlib import Path +import re +import urllib.error +import urllib.request + + +MIGRATION = re.compile( + r"server/skillhub-app/src/main/resources/db/migration/V([0-9]+)__[^/]+\.sql\Z" +) + + +class ScanError(Exception): + pass + + +def version(path): + match = MIGRATION.fullmatch(path) + return int(match[1]) if match else None + + +def introduced_version(file): + number = version(file["filename"]) + if file["status"] == "added": + return number + if file["status"] == "renamed" and version(file.get("previous_filename", "")) != number: + return number + return None + + +class GitHub: + def __init__(self, repository, token): + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository): + raise ScanError("Invalid repository name") + self.base = f"https://api.github.com/repos/{repository}" + self.token = token + + def get(self, path): + request = urllib.request.Request(self.base + path, headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + raise ScanError(f"GitHub API HTTP {error.code}; scan incomplete") from error + except (urllib.error.URLError, TimeoutError, ValueError, OSError) as error: + # Do not echo server payloads, tokens, request headers or arbitrary URLs. + raise ScanError(f"GitHub API {type(error).__name__}; scan incomplete") from error + + def pages(self, path, limit=None): + result = [] + for page in range(1, 1001): + separator = "&" if "?" in path else "?" + batch = self.get(f"{path}{separator}per_page=100&page={page}") + if not isinstance(batch, list): + raise ScanError("Unexpected GitHub pagination response") + result.extend(batch) + if len(batch) < 100: + return result + if limit is not None and len(result) >= limit: + raise ScanError("PR files reached GitHub's 3000-file limit") + raise ScanError("Pagination safety limit reached") + + +def scan(api): + versions = {} + errors = [] + prs = api.pages("/pulls?state=open&sort=created&direction=asc") + for pr in prs: + number = pr["number"] + try: + files = api.pages(f"/pulls/{number}/files", limit=3000) + current = api.get(f"/pulls/{number}") + if (current["head"]["sha"] != pr["head"]["sha"] + or current["base"]["sha"] != pr["base"]["sha"] + or current["state"] != "open"): + raise ScanError("PR changed during scan; rerun required") + if len(files) != current["changed_files"]: + raise ScanError("PR file list incomplete; rerun required") + for file in files: + value = introduced_version(file) + if value is not None: + versions.setdefault(value, {}).setdefault(number, []).append(file["filename"]) + except ScanError as error: + errors.append(f"PR #{number}: {error}") + return {v: entries for v, entries in sorted(versions.items()) if len(entries) > 1}, errors, len(prs) + + +def safe(value): + # HTML text in a pre block avoids Markdown/link/table injection from filenames. + return html.escape(str(value)).replace("\r", " ").replace("\n", " ") + + +def summary(collisions, errors, count): + lines = ["## Flyway migration preflight", "", + "Advisory only: no SQL files are fetched or executed; no PR is changed.", "", + f"Open PRs discovered: {count}.", ""] + if errors: + lines += ["**Warning: scan incomplete. Absence of a reported collision is not clearance.**", "", "
",
+                  *(safe(error) for error in errors), "
", ""] + if collisions: + lines += ["**Warning: migration versions introduced by multiple open PRs:**", "", "
"]
+        for value, entries in collisions.items():
+            lines.append(f"V{value}")
+            for number, paths in sorted(entries.items()):
+                lines.extend(f"  PR #{number}: {safe(path)}" for path in sorted(set(paths)))
+        lines += ["
", "", "Authors should coordinate numbering and rerun migration tests before merge."] + elif not errors: + lines.append("No cross-PR version collisions found in this snapshot.") + lines += ["", "Scope: added migrations and renames introducing a different version in the application migration directory. " + "Same-version renames, modified/deleted files and conflicts with already merged migrations are not checked. " + "Concurrent PR changes can make this advisory snapshot stale.", ""] + return "\n".join(lines) + + +def main(): + try: + api = GitHub(os.environ.get("GITHUB_REPOSITORY", ""), os.environ.get("GITHUB_TOKEN", "")) + collisions, errors, count = scan(api) + except ScanError as error: + collisions, errors, count = {}, [str(error)], 0 + except (KeyError, TypeError): + collisions, errors, count = {}, ["Malformed GitHub API response; no clearance issued."], 0 + report = summary(collisions, errors, count) + destination = os.environ.get("GITHUB_STEP_SUMMARY") + if destination: + with Path(destination).open("a", encoding="utf-8") as output: + output.write(report) + else: + print(report) + return 1 if errors else 0 # Workflow treats errors as advisory, never as scan clearance. + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_migration_preflight.py b/scripts/tests/test_migration_preflight.py new file mode 100644 index 000000000..4587de4b6 --- /dev/null +++ b/scripts/tests/test_migration_preflight.py @@ -0,0 +1,200 @@ +"""Offline behavior tests for the read-only migration preflight.""" +import contextlib +import importlib.util +import io +import os +import tempfile +import urllib.error +from unittest.mock import patch +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location("preflight", ROOT / "scripts/migration-preflight.py") +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +PREFIX = "server/skillhub-app/src/main/resources/db/migration/" + + +class PreflightTest(unittest.TestCase): + def test_status_and_rename_boundaries(self): + path = PREFIX + "V001__new.sql" + self.assertEqual(m.introduced_version(dict(filename=path, status="added")), 1) + for status in ("modified", "removed", "copied"): + self.assertIsNone(m.introduced_version(dict(filename=path, status=status))) + self.assertIsNone(m.introduced_version(dict(filename=path, status="renamed", + previous_filename=PREFIX + "V1__old.sql"))) + self.assertEqual(m.introduced_version(dict(filename=path, status="renamed", + previous_filename=PREFIX + "V2__old.sql")), 1) + for path in ("tests/V1__fake.sql", PREFIX + "V1_2__other.sql", PREFIX + "V1__new.sql/no"): + self.assertIsNone(m.introduced_version(dict(filename=path, status="added"))) + + def test_multiple_collisions_sorted_and_single_pr_not_collision(self): + class API: + def pages(self, path, limit=None): + if path.startswith("/pulls?"): + return [dict(number=n, head=dict(sha=str(n)), base=dict(sha="base")) for n in (9, 2)] + return [dict(filename=PREFIX + f"V{v}__a.sql", status="added") for v in (20, 3, 3)] + def get(self, path): + return dict(head=dict(sha=path.split("/")[-1]), state="open", base=dict(sha="base"), changed_files=3) + collisions, errors, count = m.scan(API()) + self.assertEqual(list(collisions), [3, 20]) + self.assertEqual(errors, []) + self.assertEqual(count, 2) + report = m.summary(collisions, errors, count) + self.assertLess(report.index("PR #2"), report.index("PR #9")) + self.assertEqual(report.count("PR #2:"), 2) + + def test_incomplete_never_reports_clearance(self): + report = m.summary({}, ["API HTTP 403"], 3) + self.assertIn("scan incomplete", report) + self.assertNotIn("No cross-PR", report) + + def test_untrusted_filename_is_text(self): + attack = PREFIX + "V1__\n::error::x.sql" + report = m.summary({1: {2: [attack], 3: [PREFIX + "V1__a.sql"]}}, [], 2) + self.assertNotIn("