mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-17 04:33:45 +00:00
252a64573e
Validate continuation evidence, sum per-turn tokens and turns, retain final cumulative cost, and include background wait in elapsed time. Add an offline pinned-CLI canary before paid evaluation and allow 60-minute sessions within the existing deadline and cleanup reserve.
213 lines
11 KiB
Python
213 lines
11 KiB
Python
"""Both trusted workflows must own a complete, serialized EC2 lifecycle."""
|
|
|
|
from pathlib import Path
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import yaml
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def test_shared_ec2_lock_obeys_repository_concurrency_gate():
|
|
subprocess.run(
|
|
[sys.executable, str(ROOT / ".github/scripts/check-workflow-concurrency.py"), str(ROOT / ".github/workflows")],
|
|
check=True,
|
|
)
|
|
|
|
|
|
def test_shared_lock_exception_cannot_be_reused_by_unrelated_workflows(tmp_path):
|
|
source = "on: workflow_dispatch\nconcurrency:\n group: gitnexus-evolution-runner\n"
|
|
script = ROOT / ".github/scripts/check-workflow-concurrency.py"
|
|
allowed = tmp_path / "release-evaluation.yml"
|
|
allowed.write_text(source)
|
|
assert subprocess.run([sys.executable, str(script), str(tmp_path)], capture_output=True).returncode == 0
|
|
allowed.rename(tmp_path / "unrelated.yml")
|
|
assert subprocess.run([sys.executable, str(script), str(tmp_path)], capture_output=True).returncode == 1
|
|
|
|
|
|
def workflow(name):
|
|
return yaml.safe_load((ROOT / ".github/workflows" / name).read_text())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name,paid", [("gitnexus-skill-evolution.yml", "evolve"), ("release-evaluation.yml", "evaluate")]
|
|
)
|
|
def test_instance_is_serialized_and_cleanup_is_hosted_even_after_failure(name, paid):
|
|
document = workflow(name)
|
|
assert document["concurrency"] == {"group": "gitnexus-evolution-runner", "cancel-in-progress": False}
|
|
jobs = document["jobs"]
|
|
start, stop = jobs["start-runner"], jobs["stop-runner"]
|
|
assert start["runs-on"] == stop["runs-on"] == "ubuntu-latest"
|
|
assert start["environment"] == stop["environment"] == "gitnexus-evolution"
|
|
assert "refs/heads/main" in start["if"] and "refs/heads/main" in stop["if"]
|
|
assert "always()" in stop["if"]
|
|
assert "cleanup_required == 'true'" in stop["if"]
|
|
assert paid in stop["needs"] and "start-runner" in stop["needs"]
|
|
assert start["permissions"] == stop["permissions"] == {"contents": "read", "id-token": "write"}
|
|
assert "self-hosted" in jobs[paid]["runs-on"]
|
|
assert set(jobs[paid]["needs"]) >= {"start-runner", "check-runner", "runner-ready"}
|
|
assert jobs["runner-ready"]["needs"] == ["start-runner"]
|
|
assert jobs["check-runner"]["needs"] == ["start-runner"]
|
|
for job, operation in ((start, "start"), (stop, "stop")):
|
|
action = next(step for step in job["steps"] if step.get("uses") == "./.github/actions/ec2-runner")
|
|
assert action["with"]["operation"] == operation
|
|
assert action["with"]["role-arn"] == "${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}"
|
|
assert action["with"]["instance-id"] == "${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}"
|
|
assert action["with"]["region"] == "${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}"
|
|
assert action["with"]["stop-schedule"] == "${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}"
|
|
|
|
|
|
def test_release_probe_skips_paid_calls_and_paid_job_has_pickup_watchdog():
|
|
jobs = workflow("release-evaluation.yml")["jobs"]
|
|
assert "inputs.runner_only != true" in jobs["evaluate"]["if"]
|
|
watch = jobs["watch-evaluate-pickup"]
|
|
assert watch["needs"] == jobs["evaluate"]["needs"]
|
|
assert "watch-evaluate-pickup" not in jobs["evaluate"]["needs"]
|
|
assert any('--job-name "Evaluate candidate and stable quality"' in step.get("run", "") for step in watch["steps"])
|
|
assert watch["steps"][-1]["if"] == "failure()"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name,paid,watch",
|
|
[
|
|
("gitnexus-skill-evolution.yml", "evolve", "watch-evolve-pickup"),
|
|
("release-evaluation.yml", "evaluate", "watch-evaluate-pickup"),
|
|
],
|
|
)
|
|
def test_pickup_watchdog_names_the_paid_job_it_bounds(name, paid, watch):
|
|
jobs = workflow(name)["jobs"]
|
|
command = f'python3 .github/scripts/evolution-runner-ready.py --job-name "{jobs[paid]["name"]}"'
|
|
assert [step.get("run") for step in jobs[watch]["steps"]].count(command) == 1
|
|
|
|
|
|
def test_release_comparison_is_scheduled_or_manual_and_gates_outcomes():
|
|
document = workflow("release-evaluation.yml")
|
|
# PyYAML parses the bare `on` key as True.
|
|
assert set(document[True]) == {"schedule", "workflow_dispatch"}
|
|
steps = document["jobs"]["evaluate"]["steps"]
|
|
scripts = "\n".join(step.get("run", "") for step in steps)
|
|
assert "releases/latest" in scripts
|
|
assert "workflow_bench.release_gate" in scripts
|
|
assert "--stable-sha" in scripts and "--candidate-sha" in scripts
|
|
|
|
|
|
def test_release_session_headroom_keeps_the_global_stop_reserve():
|
|
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
|
paid = next(step for step in steps if "workflow_bench.runner" in step.get("run", ""))
|
|
script = paid["run"]
|
|
assert "--timeout 3600" in script
|
|
assert "for runtime in stable candidate; do" in script
|
|
assert "stop_deadline_epoch }} - $(date +%s) - 5400" in script
|
|
assert 'timeout --signal=TERM --kill-after=60 "${remaining}s"' in script
|
|
assert paid["timeout-minutes"] == 1140
|
|
canary = next(step for step in steps if "tests/test_proposer_sandbox.py" in step.get("run", ""))
|
|
assert "test_real_cli_background_completion_is_one_complete_measurement" in canary["run"]
|
|
assert canary["env"]["GITNEXUS_REQUIRE_CLAUDE_CANARY"] == "1"
|
|
assert steps.index(canary) < steps.index(paid)
|
|
assert not canary.get("continue-on-error", False)
|
|
|
|
|
|
def test_both_runtimes_grade_tasks_against_one_pinned_task_dependency_checkout():
|
|
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
|
names = [step.get("name", step.get("uses", "")) for step in steps]
|
|
resolve = next(step for step in steps if step.get("id") == "tasks")
|
|
assert "workflow_bench.release_report task-sha" in resolve["run"]
|
|
assert 'echo "sha=$sha" >> "$GITHUB_OUTPUT"' in resolve["run"]
|
|
checkout = next(step for step in steps if step.get("with", {}).get("path") == "tasks-base")
|
|
assert checkout["with"]["ref"] == "${{ steps.tasks.outputs.sha }}"
|
|
assert checkout["with"]["persist-credentials"] is False
|
|
build = next(step for step in steps if "workflow_bench.release_build" in step.get("run", ""))
|
|
prepare = next(step for step in steps if "release_report prepare" in step.get("run", ""))
|
|
sessions = next(step for step in steps if "workflow_bench.runner" in step.get("run", ""))
|
|
assert steps.index(resolve) < steps.index(checkout) < steps.index(build) < steps.index(prepare)
|
|
# The task checkout is built in the same sandbox as both runtimes.
|
|
assert "for checkout in tasks-base stable candidate; do" in build["run"]
|
|
assert '--repo "$GITHUB_WORKSPACE/$checkout"' in build["run"]
|
|
# Task source and dependencies never come from the runtime under test,
|
|
# which reaches the sessions only through --gitnexus-root.
|
|
assert '--repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base"' in prepare["run"]
|
|
assert '--gitnexus-root "$GITHUB_WORKSPACE/$runtime"' in sessions["run"]
|
|
assert "tasks-base" not in sessions["run"]
|
|
assert names.count("Resolve pinned task revision") == 1
|
|
|
|
|
|
def test_both_release_graphs_are_materialized_before_any_paid_sessions():
|
|
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
|
preflight = next(step for step in steps if "workflow_bench.release_preflight" in step.get("run", ""))
|
|
paid = next(step for step in steps if "workflow_bench.runner" in step.get("run", ""))
|
|
assert steps.index(preflight) < steps.index(paid)
|
|
assert "for runtime in stable candidate; do" in preflight["run"]
|
|
assert "set -euo pipefail" in preflight["run"]
|
|
assert '--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml"' in preflight["run"]
|
|
assert '--gitnexus-root "$GITHUB_WORKSPACE/$runtime"' in preflight["run"]
|
|
assert preflight.get("continue-on-error", False) is False
|
|
assert "secrets." not in json.dumps(preflight)
|
|
assert "if" not in paid # A failed preflight must skip the paid step.
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name,paid", [("gitnexus-skill-evolution.yml", "evolve"), ("release-evaluation.yml", "evaluate")]
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"startup_attempt,current_attempt,fresh", [("1", "1", True), ("1", "2", False), ("", "2", False), ("2", "2", True)]
|
|
)
|
|
def test_partial_reruns_fail_on_hosted_runner_before_any_work(name, paid, startup_attempt, current_attempt, fresh):
|
|
jobs = workflow(name)["jobs"]
|
|
assert jobs["start-runner"]["outputs"]["run_attempt"] == "${{ github.run_attempt }}"
|
|
for name in ("runner-ready", paid):
|
|
job = jobs[name]
|
|
# Parse the actual conditional routing expression, including both JSON
|
|
# branches. A job-if skip would make the reusable workflow look green.
|
|
routing = re.fullmatch(
|
|
r"\$\{\{\s*fromJSON\(needs\.start-runner\.outputs\.run_attempt == github\.run_attempt\s*"
|
|
r"&&\s*'([^']+)'\s*\|\|\s*'([^']+)'\)\s*\}\}",
|
|
job["runs-on"],
|
|
)
|
|
assert routing is not None, job["runs-on"]
|
|
labels = json.loads(routing[1 if startup_attempt == current_attempt else 2])
|
|
assert labels == (["self-hosted", "linux", "x64", "gitnexus-evolution"] if fresh else ["ubuntu-latest"])
|
|
assert "run_attempt" not in job["if"]
|
|
guard = job["steps"][0]
|
|
assert not {"if", "uses", "env"}.intersection(guard)
|
|
assert "${{ needs.start-runner.outputs.run_attempt }}" in guard["run"]
|
|
script = guard["run"].replace("${{ needs.start-runner.outputs.run_attempt }}", startup_attempt)
|
|
result = subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-c", script],
|
|
env={**os.environ, "GITHUB_RUN_ATTEMPT": current_attempt},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
assert result.returncode == (0 if fresh else 1), result.stderr
|
|
if not fresh:
|
|
assert "Re-run all jobs" in result.stdout
|
|
|
|
|
|
def test_both_cold_boot_workflows_wait_for_package_lock():
|
|
for name, paid in (("gitnexus-skill-evolution.yml", "evolve"), ("release-evaluation.yml", "evaluate")):
|
|
scripts = "\n".join(step.get("run", "") for step in workflow(name)["jobs"][paid]["steps"])
|
|
apt_commands = [line.strip() for line in scripts.splitlines() if line.strip().startswith("sudo apt-get ")]
|
|
assert len(apt_commands) == 2
|
|
assert "sudo apt-get -o DPkg::Lock::Timeout=600 update" in apt_commands
|
|
assert (
|
|
"sudo apt-get -o DPkg::Lock::Timeout=600 install --yes --no-install-recommends bubblewrap ripgrep socat"
|
|
in apt_commands
|
|
)
|
|
|
|
|
|
def test_shared_lifecycle_jobs_stay_identical_across_both_ec2_workflows():
|
|
# The two workflows duplicate the verified lifecycle instead of sharing a
|
|
# reusable workflow; this pins them together so an edit to one copy fails.
|
|
evaluation = workflow("release-evaluation.yml")["jobs"]
|
|
evolution = workflow("gitnexus-skill-evolution.yml")["jobs"]
|
|
renamed = {"watch-evaluate-pickup": "watch-evolve-pickup", "evaluate": "evolve"}
|
|
for job in ("start-runner", "check-runner", "runner-ready", "stop-runner"):
|
|
expected = {key: value for key, value in evaluation[job].items() if key != "name"}
|
|
expected["needs"] = [renamed.get(need, need) for need in expected.get("needs", [])]
|
|
actual = {key: value for key, value in evolution[job].items() if key != "name"}
|
|
actual["needs"] = actual.get("needs", [])
|
|
assert actual == expected, job
|