mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-14 04:08:05 +00:00
22aeeebaf3
Use the existing snapshot capture ceiling for buffered materialization so valid graphs remain usable on filesystems without reflinks. Preflight both release runtimes before paid sessions and exercise the real copy boundary with a 513 MiB regression and the native release smoke.
197 lines
10 KiB
Python
197 lines
10 KiB
Python
"""Both trusted workflows must own a complete, serialized EC2 lifecycle."""
|
|
|
|
from pathlib import Path
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import yaml
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def test_shared_ec2_lock_obeys_repository_concurrency_gate():
|
|
subprocess.run(
|
|
[sys.executable, str(ROOT / ".github/scripts/check-workflow-concurrency.py"), str(ROOT / ".github/workflows")],
|
|
check=True,
|
|
)
|
|
|
|
|
|
def test_shared_lock_exception_cannot_be_reused_by_unrelated_workflows(tmp_path):
|
|
source = "on: workflow_dispatch\nconcurrency:\n group: gitnexus-evolution-runner\n"
|
|
script = ROOT / ".github/scripts/check-workflow-concurrency.py"
|
|
allowed = tmp_path / "release-evaluation.yml"
|
|
allowed.write_text(source)
|
|
assert subprocess.run([sys.executable, str(script), str(tmp_path)], capture_output=True).returncode == 0
|
|
allowed.rename(tmp_path / "unrelated.yml")
|
|
assert subprocess.run([sys.executable, str(script), str(tmp_path)], capture_output=True).returncode == 1
|
|
|
|
|
|
def workflow(name):
|
|
return yaml.safe_load((ROOT / ".github/workflows" / name).read_text())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name,paid", [("gitnexus-skill-evolution.yml", "evolve"), ("release-evaluation.yml", "evaluate")]
|
|
)
|
|
def test_instance_is_serialized_and_cleanup_is_hosted_even_after_failure(name, paid):
|
|
document = workflow(name)
|
|
assert document["concurrency"] == {"group": "gitnexus-evolution-runner", "cancel-in-progress": False}
|
|
jobs = document["jobs"]
|
|
start, stop = jobs["start-runner"], jobs["stop-runner"]
|
|
assert start["runs-on"] == stop["runs-on"] == "ubuntu-latest"
|
|
assert start["environment"] == stop["environment"] == "gitnexus-evolution"
|
|
assert "refs/heads/main" in start["if"] and "refs/heads/main" in stop["if"]
|
|
assert "always()" in stop["if"]
|
|
assert "cleanup_required == 'true'" in stop["if"]
|
|
assert paid in stop["needs"] and "start-runner" in stop["needs"]
|
|
assert start["permissions"] == stop["permissions"] == {"contents": "read", "id-token": "write"}
|
|
assert "self-hosted" in jobs[paid]["runs-on"]
|
|
assert set(jobs[paid]["needs"]) >= {"start-runner", "check-runner", "runner-ready"}
|
|
assert jobs["runner-ready"]["needs"] == ["start-runner"]
|
|
assert jobs["check-runner"]["needs"] == ["start-runner"]
|
|
for job, operation in ((start, "start"), (stop, "stop")):
|
|
action = next(step for step in job["steps"] if step.get("uses") == "./.github/actions/ec2-runner")
|
|
assert action["with"]["operation"] == operation
|
|
assert action["with"]["role-arn"] == "${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}"
|
|
assert action["with"]["instance-id"] == "${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}"
|
|
assert action["with"]["region"] == "${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}"
|
|
assert action["with"]["stop-schedule"] == "${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}"
|
|
|
|
|
|
def test_release_probe_skips_paid_calls_and_paid_job_has_pickup_watchdog():
|
|
jobs = workflow("release-evaluation.yml")["jobs"]
|
|
assert "inputs.runner_only != true" in jobs["evaluate"]["if"]
|
|
watch = jobs["watch-evaluate-pickup"]
|
|
assert watch["needs"] == jobs["evaluate"]["needs"]
|
|
assert "watch-evaluate-pickup" not in jobs["evaluate"]["needs"]
|
|
assert any('--job-name "Evaluate candidate and stable quality"' in step.get("run", "") for step in watch["steps"])
|
|
assert watch["steps"][-1]["if"] == "failure()"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name,paid,watch",
|
|
[
|
|
("gitnexus-skill-evolution.yml", "evolve", "watch-evolve-pickup"),
|
|
("release-evaluation.yml", "evaluate", "watch-evaluate-pickup"),
|
|
],
|
|
)
|
|
def test_pickup_watchdog_names_the_paid_job_it_bounds(name, paid, watch):
|
|
jobs = workflow(name)["jobs"]
|
|
command = f'python3 .github/scripts/evolution-runner-ready.py --job-name "{jobs[paid]["name"]}"'
|
|
assert [step.get("run") for step in jobs[watch]["steps"]].count(command) == 1
|
|
|
|
|
|
def test_release_comparison_is_scheduled_or_manual_and_gates_outcomes():
|
|
document = workflow("release-evaluation.yml")
|
|
# PyYAML parses the bare `on` key as True.
|
|
assert set(document[True]) == {"schedule", "workflow_dispatch"}
|
|
steps = document["jobs"]["evaluate"]["steps"]
|
|
scripts = "\n".join(step.get("run", "") for step in steps)
|
|
assert "releases/latest" in scripts
|
|
assert "workflow_bench.release_gate" in scripts
|
|
assert "--stable-sha" in scripts and "--candidate-sha" in scripts
|
|
|
|
|
|
def test_both_runtimes_grade_tasks_against_one_pinned_task_dependency_checkout():
|
|
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
|
names = [step.get("name", step.get("uses", "")) for step in steps]
|
|
resolve = next(step for step in steps if step.get("id") == "tasks")
|
|
assert "workflow_bench.release_report task-sha" in resolve["run"]
|
|
assert 'echo "sha=$sha" >> "$GITHUB_OUTPUT"' in resolve["run"]
|
|
checkout = next(step for step in steps if step.get("with", {}).get("path") == "tasks-base")
|
|
assert checkout["with"]["ref"] == "${{ steps.tasks.outputs.sha }}"
|
|
assert checkout["with"]["persist-credentials"] is False
|
|
build = next(step for step in steps if "workflow_bench.release_build" in step.get("run", ""))
|
|
prepare = next(step for step in steps if "release_report prepare" in step.get("run", ""))
|
|
sessions = next(step for step in steps if "workflow_bench.runner" in step.get("run", ""))
|
|
assert steps.index(resolve) < steps.index(checkout) < steps.index(build) < steps.index(prepare)
|
|
# The task checkout is built in the same sandbox as both runtimes.
|
|
assert "for checkout in tasks-base stable candidate; do" in build["run"]
|
|
assert '--repo "$GITHUB_WORKSPACE/$checkout"' in build["run"]
|
|
# Task source and dependencies never come from the runtime under test,
|
|
# which reaches the sessions only through --gitnexus-root.
|
|
assert '--repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base"' in prepare["run"]
|
|
assert '--gitnexus-root "$GITHUB_WORKSPACE/$runtime"' in sessions["run"]
|
|
assert "tasks-base" not in sessions["run"]
|
|
assert names.count("Resolve pinned task revision") == 1
|
|
|
|
|
|
def test_both_release_graphs_are_materialized_before_any_paid_sessions():
|
|
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
|
preflight = next(step for step in steps if "workflow_bench.release_preflight" in step.get("run", ""))
|
|
paid = next(step for step in steps if "workflow_bench.runner" in step.get("run", ""))
|
|
assert steps.index(preflight) < steps.index(paid)
|
|
assert "for runtime in stable candidate; do" in preflight["run"]
|
|
assert "set -euo pipefail" in preflight["run"]
|
|
assert '--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml"' in preflight["run"]
|
|
assert '--gitnexus-root "$GITHUB_WORKSPACE/$runtime"' in preflight["run"]
|
|
assert preflight.get("continue-on-error", False) is False
|
|
assert "secrets." not in json.dumps(preflight)
|
|
assert "if" not in paid # A failed preflight must skip the paid step.
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name,paid", [("gitnexus-skill-evolution.yml", "evolve"), ("release-evaluation.yml", "evaluate")]
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"startup_attempt,current_attempt,fresh", [("1", "1", True), ("1", "2", False), ("", "2", False), ("2", "2", True)]
|
|
)
|
|
def test_partial_reruns_fail_on_hosted_runner_before_any_work(name, paid, startup_attempt, current_attempt, fresh):
|
|
jobs = workflow(name)["jobs"]
|
|
assert jobs["start-runner"]["outputs"]["run_attempt"] == "${{ github.run_attempt }}"
|
|
for name in ("runner-ready", paid):
|
|
job = jobs[name]
|
|
# Parse the actual conditional routing expression, including both JSON
|
|
# branches. A job-if skip would make the reusable workflow look green.
|
|
routing = re.fullmatch(
|
|
r"\$\{\{\s*fromJSON\(needs\.start-runner\.outputs\.run_attempt == github\.run_attempt\s*"
|
|
r"&&\s*'([^']+)'\s*\|\|\s*'([^']+)'\)\s*\}\}",
|
|
job["runs-on"],
|
|
)
|
|
assert routing is not None, job["runs-on"]
|
|
labels = json.loads(routing[1 if startup_attempt == current_attempt else 2])
|
|
assert labels == (["self-hosted", "linux", "x64", "gitnexus-evolution"] if fresh else ["ubuntu-latest"])
|
|
assert "run_attempt" not in job["if"]
|
|
guard = job["steps"][0]
|
|
assert not {"if", "uses", "env"}.intersection(guard)
|
|
assert "${{ needs.start-runner.outputs.run_attempt }}" in guard["run"]
|
|
script = guard["run"].replace("${{ needs.start-runner.outputs.run_attempt }}", startup_attempt)
|
|
result = subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-c", script],
|
|
env={**os.environ, "GITHUB_RUN_ATTEMPT": current_attempt},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
assert result.returncode == (0 if fresh else 1), result.stderr
|
|
if not fresh:
|
|
assert "Re-run all jobs" in result.stdout
|
|
|
|
|
|
def test_both_cold_boot_workflows_wait_for_package_lock():
|
|
for name, paid in (("gitnexus-skill-evolution.yml", "evolve"), ("release-evaluation.yml", "evaluate")):
|
|
scripts = "\n".join(step.get("run", "") for step in workflow(name)["jobs"][paid]["steps"])
|
|
apt_commands = [line.strip() for line in scripts.splitlines() if line.strip().startswith("sudo apt-get ")]
|
|
assert len(apt_commands) == 2
|
|
assert "sudo apt-get -o DPkg::Lock::Timeout=600 update" in apt_commands
|
|
assert (
|
|
"sudo apt-get -o DPkg::Lock::Timeout=600 install --yes --no-install-recommends bubblewrap ripgrep socat"
|
|
in apt_commands
|
|
)
|
|
|
|
|
|
def test_shared_lifecycle_jobs_stay_identical_across_both_ec2_workflows():
|
|
# The two workflows duplicate the verified lifecycle instead of sharing a
|
|
# reusable workflow; this pins them together so an edit to one copy fails.
|
|
evaluation = workflow("release-evaluation.yml")["jobs"]
|
|
evolution = workflow("gitnexus-skill-evolution.yml")["jobs"]
|
|
renamed = {"watch-evaluate-pickup": "watch-evolve-pickup", "evaluate": "evolve"}
|
|
for job in ("start-runner", "check-runner", "runner-ready", "stop-runner"):
|
|
expected = {key: value for key, value in evaluation[job].items() if key != "name"}
|
|
expected["needs"] = [renamed.get(need, need) for need in expected.get("needs", [])]
|
|
actual = {key: value for key, value in evolution[job].items() if key != "name"}
|
|
actual["needs"] = actual.get("needs", [])
|
|
assert actual == expected, job
|