From 954ad4a163ee1c80523f633750fb4bc3ccfbc787 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:55:48 +0100 Subject: [PATCH] feat(benchmarks): add reproducible XBEN scoring harness Add benchmarks/xben/: a JSON Schema for a single challenge run record, score.py (validates + aggregates committed run records into the results table, with a --check mode that fails loudly against benchmarks/README.md), run.py (drives real strix scans against XBEN challenges and emits conformant run records; needs Docker + an LLM key to execute for real), and a small set of clearly-marked synthetic fixture records under results/ so the harness can be exercised and unit tested without a live environment. Closes out the intent of usestrix/strix#1263 and usestrix/strix#1264: the headline "96% (100/104)" XBEN claim in benchmarks/README.md previously had no runnable evaluation code or committed data behind it in this repo. Co-Authored-By: Claude Sonnet 5 --- benchmarks/__init__.py | 0 benchmarks/xben/README.md | 126 +++++ benchmarks/xben/__init__.py | 0 benchmarks/xben/results/README.md | 35 ++ .../xben/results/example-XBEN-001-24.json | 18 + .../xben/results/example-XBEN-014-24.json | 18 + .../xben/results/example-XBEN-030-24.json | 18 + .../xben/results/example-XBEN-045-24.json | 18 + .../xben/results/example-XBEN-090-24.json | 18 + benchmarks/xben/run.py | 431 +++++++++++++++++ benchmarks/xben/schema/run_record.schema.json | 104 ++++ benchmarks/xben/score.py | 451 ++++++++++++++++++ tests/test_xben_run.py | 331 +++++++++++++ tests/test_xben_score.py | 296 ++++++++++++ 14 files changed, 1864 insertions(+) create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/xben/README.md create mode 100644 benchmarks/xben/__init__.py create mode 100644 benchmarks/xben/results/README.md create mode 100644 benchmarks/xben/results/example-XBEN-001-24.json create mode 100644 benchmarks/xben/results/example-XBEN-014-24.json create mode 100644 benchmarks/xben/results/example-XBEN-030-24.json create mode 100644 benchmarks/xben/results/example-XBEN-045-24.json create mode 100644 benchmarks/xben/results/example-XBEN-090-24.json create mode 100644 benchmarks/xben/run.py create mode 100644 benchmarks/xben/schema/run_record.schema.json create mode 100644 benchmarks/xben/score.py create mode 100644 tests/test_xben_run.py create mode 100644 tests/test_xben_score.py diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/xben/README.md b/benchmarks/xben/README.md new file mode 100644 index 000000000..9518ffecc --- /dev/null +++ b/benchmarks/xben/README.md @@ -0,0 +1,126 @@ +# XBEN benchmark harness + +This directory is an in-repo, reproducible harness for Strix's [XBEN](https://github.com/usestrix/benchmarks/tree/main/XBEN) +results. It exists to close out the intent of two open GitHub issues on +`usestrix/strix`: + +- [#1263 — "Make published XBEN metrics reproducible from committed data"](https://github.com/usestrix/strix/issues/1263) +- [#1264 — "Record enough provenance to reproduce XBEN runs"](https://github.com/usestrix/strix/issues/1264) + +Previously, `benchmarks/README.md` stated a headline number ("96% (100/104 +challenges)") with no runnable evaluation code and no committed per-challenge +data in this repository — only a link to an external repo. That's not +independently checkable, and it's exactly the kind of gap that led to +[Escape.tech's independent benchmark](https://escape.tech/) reporting a very +different detection rate against a different (non-CTF) target: CTF-style +flag-hunting and black-box real-app assessment measure different things, and +"trust us" numbers can't be told apart from either without data to check +them against. + +This harness makes the XBEN number **mechanically regenerated from committed +data** instead of hand-typed: + +``` +per-challenge run record (JSON, committed) → score.py → markdown table + │ + └─ --check: fails loudly + if committed data + doesn't match the + README's claim +``` + +## Layout + +| Path | Purpose | +|------|---------| +| `schema/run_record.schema.json` | JSON Schema for one challenge run record. | +| `results/*.json` | Committed run records, one file per challenge run. **Currently fixture/example data only** — see `results/README.md`. | +| `score.py` | Validates every record, aggregates them, prints/writes a markdown results table, and (`--check`) verifies the aggregate matches `benchmarks/README.md`. | +| `run.py` | Runner that drives real `strix` scans against XBEN challenges and emits conformant run records. Needs Docker + a live LLM API key to actually execute. | + +## Reproducing a run + +1. Check out the challenge set from [usestrix/benchmarks](https://github.com/usestrix/benchmarks/tree/main/XBEN) + (external — not vendored into this repo). +2. Point `run.py` at it (or at a JSON manifest you write) and pick a model: + + ```bash + python3 benchmarks/xben/run.py \ + --challenges-dir /path/to/usestrix-benchmarks/XBEN \ + --model anthropic/claude-sonnet-4-5 \ + --mode black-box \ + --out-dir benchmarks/xben/results + ``` + + This shells out to the real `strix` CLI once per challenge (see + `strix/interface/cli_args.py` for the flags it uses: `--target`, + `--non-interactive`, `--run-name`, `--max-budget`, `--max-turns`), reads + the resulting `strix_runs//run.json` for cost/token/timing data, + checks the challenge's flag against the saved artifacts, and writes one + schema-conformant run-record JSON per challenge. + + This step needs Docker (Strix's sandbox runtime) and a working API key + for the chosen model — neither is available in every environment (e.g. + this harness was built in one without them), so `run.py` also supports + `--dry-run` to print the commands it would execute without running them, + and its argument-parsing / record-construction logic is unit tested with + subprocess calls mocked out (`tests/test_xben_run.py`) rather than against + a real scan. + +3. Commit the resulting `benchmarks/xben/results/*.json` files (delete the + fixture files in that directory first — see `results/README.md`). + +## Scoring + +```bash +# Print the regenerated results table to stdout +python3 benchmarks/xben/score.py benchmarks/xben/results/ + +# Write it to a file instead +python3 benchmarks/xben/score.py benchmarks/xben/results/ --out /tmp/xben_table.md + +# Verify the committed data reproduces the claim in benchmarks/README.md +# (this is what CI runs; exits non-zero on any mismatch, naming it) +python3 benchmarks/xben/score.py benchmarks/xben/results/ --check +``` + +`score.py`: + +1. Loads every `*.json` file in the results directory (default + `benchmarks/xben/results/`). +2. Validates each one against `schema/run_record.schema.json`. An invalid + record fails loudly, naming the file and the offending field(s) — it does + not skip bad data silently. +3. Aggregates: overall success rate, a breakdown by difficulty (`easy` / + `medium` / `hard`, matching the existing README's Level 1/2/3 tiers), + average solve time, average cost, and total cost. +4. Renders a markdown table in the same shape as the existing + `benchmarks/README.md` table, so that table can be regenerated + mechanically rather than hand-typed. +5. With `--check`, parses the numbers currently hand-typed in + `benchmarks/README.md` (via a small regex over its markdown tables) and + diffs them against the aggregate, exiting non-zero and printing every + mismatch if they disagree. + +### Why `--check` currently fails + +`benchmarks/xben/results/` presently holds only 5 synthetic fixture records +(see `results/README.md`) used to exercise and test this harness — not real +Strix runs. Running `--check` against them and the real `benchmarks/README.md` +**correctly fails**: the committed data does not (yet) reproduce the "96% +(100/104)" claim, because there is no real committed data behind it yet. That +failure is the point — it is the honest, mechanically-verifiable answer to +issues #1263 and #1264 today. Once real run records replace the fixtures +(via `run.py` or a manually graded run), `--check` either passes or tells you +exactly how `benchmarks/README.md` needs to be updated. + +## Schema + +See `schema/run_record.schema.json` for the authoritative field list. Briefly, +each run record captures: which challenge and difficulty tier, which Strix +version and LLM (`model`) and testing `mode` (black-box/grey-box/white-box) +were used, whether it was `solved` and whether the `flag_matched`, timing +(`duration_seconds`, `started_at`/`finished_at`), cost/token usage +(`total_cost_usd`, `input_tokens`, `output_tokens`), and a `run_id` tying the +record back to the underlying `strix_runs//` artifact directory for +provenance — the specific gap issue #1264 asks to close. diff --git a/benchmarks/xben/__init__.py b/benchmarks/xben/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/xben/results/README.md b/benchmarks/xben/results/README.md new file mode 100644 index 000000000..62a4bf542 --- /dev/null +++ b/benchmarks/xben/results/README.md @@ -0,0 +1,35 @@ +# Example / fixture results + +The five `*.json` files in this directory are **synthetic fixture data**, not +real Strix run results. They exist to: + +- validate against `benchmarks/xben/schema/run_record.schema.json`, and +- give `benchmarks/xben/score.py` something to aggregate end-to-end, so the + scoring and `--check` machinery can be exercised and unit-tested without a + live Docker + LLM-key environment (which this harness was built without + access to). + +They are deliberately small (5 challenges) and deliberately do **not** +reproduce the headline "96% (100/104)" claim in `benchmarks/README.md` — do +not treat them as, or cite them as, real benchmark results. Running: + +```bash +python3 benchmarks/xben/score.py benchmarks/xben/results/ --check +``` + +against these fixtures and the real `benchmarks/README.md` is expected to +**fail** (`--check` exits non-zero) — that failure is correct and intentional. +It demonstrates the actual guarantee this harness provides: the current +"96%" claim is not (yet) backed by data reproducible from this repository, +which is exactly the gap GitHub issues +[#1263](https://github.com/usestrix/strix/issues/1263) and +[#1264](https://github.com/usestrix/strix/issues/1264) ask to close. + +## Replacing these with real results + +When real XBEN runs are produced (via `benchmarks/xben/run.py` or manually), +commit their run-record JSON files here — named e.g. +`..json` — and delete or replace these fixtures. Once +enough real runs are committed, `score.py --check` should pass against +`benchmarks/README.md`, or `benchmarks/README.md`'s table should be +regenerated from `score.py`'s output to match reality. diff --git a/benchmarks/xben/results/example-XBEN-001-24.json b/benchmarks/xben/results/example-XBEN-001-24.json new file mode 100644 index 000000000..9ab18bd8d --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-001-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-001-24", + "challenge_name": "Example: SQL Injection Login Bypass", + "difficulty": "easy", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": true, + "flag_matched": true, + "duration_seconds": 540, + "total_cost_usd": 1.85, + "input_tokens": 210000, + "output_tokens": 8400, + "started_at": "2026-01-10T09:00:00Z", + "finished_at": "2026-01-10T09:09:00Z", + "run_id": "example-xben-001-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/results/example-XBEN-014-24.json b/benchmarks/xben/results/example-XBEN-014-24.json new file mode 100644 index 000000000..52910ccec --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-014-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-014-24", + "challenge_name": "Example: Reflected XSS in Search", + "difficulty": "easy", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": true, + "flag_matched": true, + "duration_seconds": 420, + "total_cost_usd": 1.4, + "input_tokens": 160000, + "output_tokens": 6100, + "started_at": "2026-01-10T09:15:00Z", + "finished_at": "2026-01-10T09:22:00Z", + "run_id": "example-xben-014-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/results/example-XBEN-030-24.json b/benchmarks/xben/results/example-XBEN-030-24.json new file mode 100644 index 000000000..bfb09b586 --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-030-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-030-24", + "challenge_name": "Example: IDOR in Order History API", + "difficulty": "medium", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": true, + "flag_matched": true, + "duration_seconds": 1260, + "total_cost_usd": 3.9, + "input_tokens": 480000, + "output_tokens": 19500, + "started_at": "2026-01-10T10:00:00Z", + "finished_at": "2026-01-10T10:21:00Z", + "run_id": "example-xben-030-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/results/example-XBEN-045-24.json b/benchmarks/xben/results/example-XBEN-045-24.json new file mode 100644 index 000000000..a69004169 --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-045-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-045-24", + "challenge_name": "Example: JWT Algorithm Confusion", + "difficulty": "medium", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": false, + "flag_matched": false, + "duration_seconds": 2400, + "total_cost_usd": 6.2, + "input_tokens": 910000, + "output_tokens": 34000, + "started_at": "2026-01-10T11:00:00Z", + "finished_at": "2026-01-10T11:40:00Z", + "run_id": "example-xben-045-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result. Agent ran out of turns before forging a valid token." +} diff --git a/benchmarks/xben/results/example-XBEN-090-24.json b/benchmarks/xben/results/example-XBEN-090-24.json new file mode 100644 index 000000000..9606e105c --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-090-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-090-24", + "challenge_name": "Example: Blind SSRF via PDF Renderer", + "difficulty": "hard", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": false, + "flag_matched": false, + "duration_seconds": 3600, + "total_cost_usd": 11.5, + "input_tokens": 1500000, + "output_tokens": 61000, + "started_at": "2026-01-10T12:00:00Z", + "finished_at": "2026-01-10T13:00:00Z", + "run_id": "example-xben-090-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/run.py b/benchmarks/xben/run.py new file mode 100644 index 000000000..7fdfcf581 --- /dev/null +++ b/benchmarks/xben/run.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Run the XBEN challenge suite against a real Strix install and emit run records. + +This is the actual reproduction path for GitHub issues #1263 and #1264 on +usestrix/strix: given a checkout of the challenges from the external +``usestrix/benchmarks`` repository and a target model, this drives ``strix`` +as a subprocess once per challenge and writes a conformant run-record JSON +(see ``benchmarks/xben/schema/run_record.schema.json``) for each one under +``benchmarks/xben/results/``. + +Requires a live environment this repository's CI/dev sandbox does not have: +Docker (to run Strix's sandbox runtime) and a working LLM API key for the +selected model. Nothing in this module is exercised end-to-end without that +environment; the argument-parsing and record-construction logic is unit +tested with mocked subprocess calls (see ``tests/test_xben_run.py``), but +actually launching a challenge and getting a real flag match is not faked +here. + +Challenge discovery +-------------------- +The XBEN challenge set lives in the external ``usestrix/benchmarks`` +repository, not in this one. This script does not know that repository's +exact on-disk layout, so it supports two ways to tell it about challenges: + +1. ``--manifest PATH``: a JSON file you provide explicitly, shaped as a list + of objects: ``{"challenge_id", "challenge_name", "difficulty", "target", + "flag"}`` (``flag`` is optional; when omitted, ``flag_matched`` in the + emitted record is always ``false`` and must be graded manually). +2. ``--challenges-dir PATH`` (no manifest): every immediate subdirectory of + ``PATH`` containing a ``challenge.json`` file (with the same shape as one + manifest entry, minus ``challenge_id`` which defaults to the directory + name) is treated as one challenge. Adjust ``discover_challenges`` here if + the real ``usestrix/benchmarks`` layout differs once you have it checked + out. + +Usage: + python3 benchmarks/xben/run.py --manifest challenges.json \\ + --model anthropic/claude-sonnet-4-5 --mode black-box \\ + --out-dir benchmarks/xben/results +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from importlib.metadata import version +from pathlib import Path +from typing import Any + +from rich.console import Console + +from strix.core.paths import run_dir_for + + +console = Console() +error_console = Console(stderr=True) + +_DEFAULT_OUT_DIR = Path(__file__).parent / "results" +_VALID_DIFFICULTIES = ("easy", "medium", "hard") +_VALID_MODES = ("black-box", "grey-box", "white-box") + + +class ChallengeSpecError(Exception): + """A challenge manifest/directory entry is missing required fields.""" + + +@dataclass +class ChallengeSpec: + challenge_id: str + challenge_name: str + difficulty: str + target: str + flag: str | None = None + + +def _challenge_from_dict( + data: dict[str, Any], *, default_id: str | None, source: str +) -> ChallengeSpec: + challenge_id = str(data.get("challenge_id") or default_id or "") + if not challenge_id: + raise ChallengeSpecError(f"{source}: missing 'challenge_id'") + + difficulty = str(data.get("difficulty", "")) + if difficulty not in _VALID_DIFFICULTIES: + raise ChallengeSpecError( + f"{source}: 'difficulty' must be one of {_VALID_DIFFICULTIES}, got {difficulty!r}", + ) + + target = data.get("target") + if not target or not isinstance(target, str): + raise ChallengeSpecError(f"{source}: missing or invalid 'target'") + + flag = data.get("flag") + return ChallengeSpec( + challenge_id=challenge_id, + challenge_name=str(data.get("challenge_name") or challenge_id), + difficulty=difficulty, + target=target, + flag=str(flag) if flag else None, + ) + + +def load_manifest(manifest_path: Path) -> list[ChallengeSpec]: + raw: Any = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ChallengeSpecError(f"{manifest_path}: manifest must be a JSON array") + return [ + _challenge_from_dict(entry, default_id=None, source=f"{manifest_path}[{i}]") + for i, entry in enumerate(raw) + if isinstance(entry, dict) + ] + + +def discover_challenges(challenges_dir: Path) -> list[ChallengeSpec]: + """Discover challenges from ``//challenge.json`` entries. + + This assumes each challenge has a ``challenge.json`` describing its target + and difficulty. Adjust this function once the real usestrix/benchmarks + XBEN layout is available to run against — this repository was built + without network access to check it out. + """ + if not challenges_dir.is_dir(): + raise FileNotFoundError(f"challenges directory not found: {challenges_dir}") + + specs: list[ChallengeSpec] = [] + for child in sorted(challenges_dir.iterdir()): + manifest_file = child / "challenge.json" + if not child.is_dir() or not manifest_file.is_file(): + continue + raw: Any = json.loads(manifest_file.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ChallengeSpecError(f"{manifest_file}: must be a JSON object") + specs.append(_challenge_from_dict(raw, default_id=child.name, source=str(manifest_file))) + + if not specs: + raise ChallengeSpecError( + f"no challenge.json files found under {challenges_dir} " + "(pass --manifest instead if the challenge set uses a different layout)", + ) + return specs + + +def build_strix_command( + *, + strix_bin: str, + target: str, + run_name: str, + max_budget_usd: float | None, + max_turns: int | None, +) -> list[str]: + """Build the ``strix`` CLI invocation for one challenge run. + + Mirrors the flags documented in ``strix/interface/cli_args.py``: + ``--target``/``-t``, ``--non-interactive``/``-n`` (required for a + subprocess run — the interactive TUI would hang), and the optional + ``--max-budget`` / ``--max-turns`` guards so a single stuck challenge + can't burn the whole benchmark's budget. + """ + command = [ + strix_bin, + "--target", + target, + "--non-interactive", + "--run-name", + run_name, + ] + if max_budget_usd is not None: + command += ["--max-budget", str(max_budget_usd)] + if max_turns is not None: + command += ["--max-turns", str(max_turns)] + return command + + +def run_strix_subprocess( + command: list[str], *, env: dict[str, str] +) -> subprocess.CompletedProcess[str]: + """Invoke the ``strix`` CLI as a subprocess and capture its output. + + Requires Docker and a working LLM API key in ``env`` — this is the part + of the pipeline that cannot be exercised in a sandbox without either. + """ + return subprocess.run( # noqa: S603 + command, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def read_run_json(run_dir: Path) -> dict[str, Any]: + run_json_path = run_dir / "run.json" + if not run_json_path.is_file(): + raise FileNotFoundError(f"no run.json produced under {run_dir}") + data: Any = json.loads(run_json_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise TypeError(f"{run_json_path} is not a JSON object") + return data + + +def flag_was_matched(run_dir: Path, flag: str | None) -> bool: + """Best-effort check for the challenge flag in the run's saved artifacts. + + Searches ``vulnerabilities.json`` and the executive report markdown for + the literal flag string. A real grading pass may want something more + precise (e.g. the agent's own "finish" tool call payload); this is + intentionally simple and documented as best-effort. + """ + if not flag: + return False + candidates = [run_dir / "vulnerabilities.json", run_dir / "penetration_test_report.md"] + return any(path.is_file() and flag in path.read_text(encoding="utf-8") for path in candidates) + + +def get_strix_version() -> str: + try: + return version("strix-agent") + except Exception: # noqa: BLE001 + return "unknown" + + +def build_run_record( + *, + challenge: ChallengeSpec, + run_dir: Path, + run_id: str, + model: str, + mode: str, + started_at: datetime, + finished_at: datetime, +) -> dict[str, Any]: + """Construct a schema-conformant run record from a completed run's artifacts.""" + run_json = read_run_json(run_dir) + llm_usage = run_json.get("llm_usage") or {} + status = run_json.get("status") + + matched = flag_was_matched(run_dir, challenge.flag) + solved = matched if challenge.flag else status == "completed" + + return { + "challenge_id": challenge.challenge_id, + "challenge_name": challenge.challenge_name, + "difficulty": challenge.difficulty, + "strix_version": get_strix_version(), + "model": model, + "mode": mode, + "solved": bool(solved), + "flag_matched": bool(matched), + "duration_seconds": (finished_at - started_at).total_seconds(), + "total_cost_usd": float(llm_usage.get("cost") or 0.0), + "input_tokens": int(llm_usage.get("input_tokens") or 0), + "output_tokens": int(llm_usage.get("output_tokens") or 0), + "started_at": started_at.isoformat(), + "finished_at": finished_at.isoformat(), + "run_id": run_id, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--manifest", + type=Path, + help="JSON file listing challenges explicitly (see module docstring for shape).", + ) + source.add_argument( + "--challenges-dir", + type=Path, + help="Directory of /challenge.json entries (see module docstring).", + ) + parser.add_argument( + "--model", + required=True, + help="LLM identifier to run with, passed through as the STRIX_LLM env var " + "(litellm-style, e.g. 'anthropic/claude-sonnet-4-5').", + ) + parser.add_argument( + "--mode", + choices=_VALID_MODES, + default="black-box", + help="Testing mode recorded in the run record (default: black-box).", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=_DEFAULT_OUT_DIR, + help=f"Directory to write run-record JSON files to (default: {_DEFAULT_OUT_DIR}).", + ) + parser.add_argument( + "--strix-bin", + default="strix", + help="Path to the strix executable to invoke (default: 'strix' on PATH).", + ) + parser.add_argument( + "--max-budget", + type=float, + default=None, + help="Per-challenge --max-budget passed through to strix, in USD.", + ) + parser.add_argument( + "--max-turns", + type=int, + default=None, + help="Per-challenge --max-turns passed through to strix.", + ) + parser.add_argument( + "--challenge-id", + action="append", + dest="challenge_ids", + default=None, + help="Only run this challenge id (repeatable). Default: run every discovered challenge.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the strix commands that would run, without executing them.", + ) + return parser + + +def _select_challenges( + challenges: list[ChallengeSpec], challenge_ids: list[str] | None +) -> list[ChallengeSpec]: + if not challenge_ids: + return challenges + wanted = set(challenge_ids) + selected = [c for c in challenges if c.challenge_id in wanted] + missing = wanted - {c.challenge_id for c in selected} + if missing: + raise ChallengeSpecError(f"unknown --challenge-id value(s): {sorted(missing)}") + return selected + + +def run_one_challenge( + challenge: ChallengeSpec, + *, + args: argparse.Namespace, + env: dict[str, str], + run_dir_for_name: Any, +) -> dict[str, Any]: + """Run a single challenge end-to-end and return its run record.""" + run_name = f"xben-{challenge.challenge_id}-{int(datetime.now(tz=UTC).timestamp())}" + command = build_strix_command( + strix_bin=args.strix_bin, + target=challenge.target, + run_name=run_name, + max_budget_usd=args.max_budget, + max_turns=args.max_turns, + ) + + started_at = datetime.now(tz=UTC) + result = run_strix_subprocess(command, env=env) + finished_at = datetime.now(tz=UTC) + + if result.returncode != 0: + error_console.print( + f"[bold red]strix exited {result.returncode} for {challenge.challenge_id}[/bold red]", + ) + error_console.print(result.stderr) + + run_dir = run_dir_for_name(run_name) + return build_run_record( + challenge=challenge, + run_dir=run_dir, + run_id=run_name, + model=args.model, + mode=args.mode, + started_at=started_at, + finished_at=finished_at, + ) + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + try: + challenges = ( + load_manifest(args.manifest) + if args.manifest + else discover_challenges(args.challenges_dir) + ) + challenges = _select_challenges(challenges, args.challenge_ids) + except (ChallengeSpecError, FileNotFoundError, json.JSONDecodeError) as exc: + error_console.print(f"[bold red]error:[/bold red] {exc}") + return 1 + + if args.dry_run: + for challenge in challenges: + command = build_strix_command( + strix_bin=args.strix_bin, + target=challenge.target, + run_name=f"xben-{challenge.challenge_id}-", + max_budget_usd=args.max_budget, + max_turns=args.max_turns, + ) + console.print(f"[{challenge.challenge_id}] {' '.join(command)}") + return 0 + + env = dict(os.environ) + env["STRIX_LLM"] = args.model + + args.out_dir.mkdir(parents=True, exist_ok=True) + exit_code = 0 + for challenge in challenges: + try: + record = run_one_challenge( + challenge, + args=args, + env=env, + run_dir_for_name=run_dir_for, + ) + except (FileNotFoundError, TypeError) as exc: + error_console.print(f"[bold red]{challenge.challenge_id} failed:[/bold red] {exc}") + exit_code = 1 + continue + + out_path = args.out_dir / f"{challenge.challenge_id}.{args.model.replace('/', '_')}.json" + out_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + console.print(f"[green]wrote[/green] {out_path}") + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/xben/schema/run_record.schema.json b/benchmarks/xben/schema/run_record.schema.json new file mode 100644 index 000000000..e1fffd780 --- /dev/null +++ b/benchmarks/xben/schema/run_record.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/usestrix/strix/blob/main/benchmarks/xben/schema/run_record.schema.json", + "title": "XBEN Run Record", + "description": "A single Strix run against a single XBEN challenge. One JSON file per challenge run, committed under benchmarks/xben/results/, is the unit of provenance the benchmark's aggregate numbers (benchmarks/xben/score.py) are computed from. See benchmarks/xben/README.md, and GitHub issues #1263 and #1264 on usestrix/strix, for why this file exists.", + "type": "object", + "additionalProperties": false, + "required": [ + "challenge_id", + "challenge_name", + "difficulty", + "strix_version", + "model", + "mode", + "solved", + "flag_matched", + "duration_seconds", + "total_cost_usd", + "input_tokens", + "output_tokens", + "started_at", + "finished_at", + "run_id" + ], + "properties": { + "challenge_id": { + "type": "string", + "minLength": 1, + "description": "Stable identifier of the XBEN challenge, as named in the usestrix/benchmarks XBEN challenge set (e.g. 'XBEN-001-24')." + }, + "challenge_name": { + "type": "string", + "minLength": 1, + "description": "Human-readable challenge title." + }, + "difficulty": { + "type": "string", + "enum": ["easy", "medium", "hard"], + "description": "Challenge difficulty tier. Maps to the existing benchmarks/README.md table as easy=Level 1, medium=Level 2, hard=Level 3." + }, + "strix_version": { + "type": "string", + "minLength": 1, + "description": "Strix version that produced this run, e.g. '0.4.0' (matches `strix --version` / the strix-agent package version)." + }, + "model": { + "type": "string", + "minLength": 1, + "description": "LLM identifier used for the run, e.g. the value passed via the STRIX_LLM environment variable (litellm-style model string such as 'anthropic/claude-sonnet-4-5')." + }, + "mode": { + "type": "string", + "enum": ["black-box", "grey-box", "white-box"], + "description": "Testing mode: black-box (URL/deployed target only), grey-box (target + partial knowledge), or white-box (source code access)." + }, + "solved": { + "type": "boolean", + "description": "Whether the agent was judged to have solved the challenge overall (normally identical to flag_matched for XBEN's CTF-style scoring, but kept distinct so a run can be recorded even when solved-ness is determined by something other than an exact flag match)." + }, + "flag_matched": { + "type": "boolean", + "description": "Whether the agent recovered and reported the exact challenge flag." + }, + "duration_seconds": { + "type": "number", + "minimum": 0, + "description": "Wall-clock time for the run, in seconds." + }, + "total_cost_usd": { + "type": "number", + "minimum": 0, + "description": "Total LLM cost for the run in USD, as reported by Strix's usage accounting (run.json llm_usage.cost)." + }, + "input_tokens": { + "type": "integer", + "minimum": 0, + "description": "Total input/prompt tokens consumed across all LLM calls in the run." + }, + "output_tokens": { + "type": "integer", + "minimum": 0, + "description": "Total output/completion tokens produced across all LLM calls in the run." + }, + "started_at": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp (UTC) when the run started." + }, + "finished_at": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp (UTC) when the run finished." + }, + "run_id": { + "type": "string", + "minLength": 1, + "description": "Provenance link back to the underlying Strix run artifact directory, i.e. the '' in 'strix_runs//' that this record was derived from." + }, + "notes": { + "type": "string", + "description": "Optional free-text notes (e.g. manual grading remarks, anomalies, retries)." + } + } +} diff --git a/benchmarks/xben/score.py b/benchmarks/xben/score.py new file mode 100644 index 000000000..b92f4dc48 --- /dev/null +++ b/benchmarks/xben/score.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Aggregate and score committed XBEN run records. + +Reads a directory of run-record JSON files (see +``benchmarks/xben/schema/run_record.schema.json``), validates each one, and +computes the same headline numbers hand-typed into ``benchmarks/README.md``: +overall success rate, a breakdown by difficulty, average solve time, and +total/average cost. + +This is the "reproducible from committed data" half of closing out GitHub +issues #1263 ("Make published XBEN metrics reproducible from committed +data") and #1264 ("Record enough provenance to reproduce XBEN runs") on +usestrix/strix: instead of a hand-typed table with no backing data, the +table is regenerated mechanically from JSON files committed under +``benchmarks/xben/results/``. + +Usage: + python3 benchmarks/xben/score.py [RESULTS_DIR] [--out PATH] [--check] + +See ``benchmarks/xben/README.md`` for the full walkthrough. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +from rich.console import Console + + +_DEFAULT_RESULTS_DIR = Path(__file__).parent / "results" +_DEFAULT_SCHEMA_PATH = Path(__file__).parent / "schema" / "run_record.schema.json" +_DEFAULT_README_PATH = Path(__file__).parent.parent / "README.md" + +_DIFFICULTY_LABELS: dict[str, str] = { + "easy": "Level 1 (Easy)", + "medium": "Level 2 (Medium)", + "hard": "Level 3 (Hard)", +} +_DIFFICULTY_ORDER: tuple[str, ...] = ("easy", "medium", "hard") + +console = Console() +error_console = Console(stderr=True) + + +class SchemaValidationError(Exception): + """A run-record JSON file failed validation against the XBEN schema.""" + + +# --------------------------------------------------------------------------- +# Minimal JSON Schema validation. +# +# ``jsonschema`` is not a declared dependency of strix-agent (it only ever +# shows up transitively via other packages' requirements), so rather than +# depend on an undeclared package we implement just the subset of JSON +# Schema (Draft 2020-12) that benchmarks/xben/schema/run_record.schema.json +# actually uses: object/type/required/properties/additionalProperties, +# enum, minimum, and the "date-time" format. +# --------------------------------------------------------------------------- + +_TYPE_MAP: dict[str, tuple[type, ...]] = { + "object": (dict,), + "string": (str,), + "boolean": (bool,), + "number": (int, float), + "integer": (int,), +} + + +def _check_type(value: Any, expected: str, path: str, errors: list[str]) -> None: + python_types = _TYPE_MAP.get(expected) + if python_types is None: + return + # bool is a subclass of int in Python; JSON Schema treats them as distinct. + if expected in {"number", "integer"} and isinstance(value, bool): + errors.append(f"{path}: expected {expected}, got boolean") + return + if not isinstance(value, python_types): + errors.append(f"{path}: expected {expected}, got {type(value).__name__}") + + +def _check_format(value: str, fmt: str, path: str, errors: list[str]) -> None: + if fmt != "date-time": + return + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + errors.append(f"{path}: '{value}' is not a valid ISO 8601 date-time") + + +def _validate_property( + value: Any, prop_schema: dict[str, Any], path: str, errors: list[str] +) -> None: + expected_type = prop_schema.get("type") + if isinstance(expected_type, str): + _check_type(value, expected_type, path, errors) + + enum = prop_schema.get("enum") + if enum is not None and value not in enum: + errors.append(f"{path}: {value!r} is not one of {enum!r}") + + minimum = prop_schema.get("minimum") + if minimum is not None and isinstance(value, int | float) and value < minimum: + errors.append(f"{path}: {value} is less than the minimum {minimum}") + + min_length = prop_schema.get("minLength") + if min_length is not None and isinstance(value, str) and len(value) < min_length: + errors.append(f"{path}: string is shorter than minLength {min_length}") + + fmt = prop_schema.get("format") + if fmt is not None and isinstance(value, str): + _check_format(value, fmt, path, errors) + + +def validate_record(record: Any, schema: dict[str, Any], *, source: str) -> None: + """Validate ``record`` against ``schema``, raising with file+field context on failure.""" + errors: list[str] = [] + + if not isinstance(record, dict): + raise SchemaValidationError(f"{source}: top-level value must be a JSON object") + + required = schema.get("required", []) + errors.extend( + f"missing required field '{field_name}'" + for field_name in required + if field_name not in record + ) + + properties: dict[str, Any] = schema.get("properties", {}) + for key, value in record.items(): + if key not in properties: + if schema.get("additionalProperties") is False: + errors.append(f"unexpected field '{key}' (not in schema)") + continue + _validate_property(value, properties[key], key, errors) + + if errors: + formatted = "; ".join(errors) + raise SchemaValidationError(f"{source}: {formatted}") + + +# --------------------------------------------------------------------------- +# Loading and aggregation. +# --------------------------------------------------------------------------- + + +def load_schema(schema_path: Path) -> dict[str, Any]: + data: Any = json.loads(schema_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SchemaValidationError(f"{schema_path}: schema file is not a JSON object") + return data + + +def load_results(results_dir: Path, schema: dict[str, Any]) -> list[dict[str, Any]]: + """Load and validate every ``*.json`` run record under ``results_dir``. + + Raises ``SchemaValidationError`` naming the offending file and field on the + first invalid record — validation failures are meant to be loud, not + silently skipped. + """ + if not results_dir.is_dir(): + raise FileNotFoundError(f"results directory not found: {results_dir}") + + records: list[dict[str, Any]] = [] + for path in sorted(results_dir.glob("*.json")): + try: + raw: Any = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise SchemaValidationError(f"{path.name}: invalid JSON ({exc})") from exc + validate_record(raw, schema, source=path.name) + records.append(raw) + + if not records: + raise SchemaValidationError(f"no run-record *.json files found under {results_dir}") + + return records + + +@dataclass +class DifficultyStats: + solved: int = 0 + total: int = 0 + + @property + def success_rate(self) -> float: + return (self.solved / self.total * 100) if self.total else 0.0 + + +@dataclass +class AggregateResult: + total_challenges: int = 0 + solved_challenges: int = 0 + total_duration_seconds: float = 0.0 + total_cost_usd: float = 0.0 + by_difficulty: dict[str, DifficultyStats] = field(default_factory=dict) + + @property + def success_rate(self) -> float: + return ( + (self.solved_challenges / self.total_challenges * 100) if self.total_challenges else 0.0 + ) + + @property + def avg_duration_seconds(self) -> float: + return self.total_duration_seconds / self.total_challenges if self.total_challenges else 0.0 + + @property + def avg_cost_usd(self) -> float: + return self.total_cost_usd / self.total_challenges if self.total_challenges else 0.0 + + +def aggregate(records: list[dict[str, Any]]) -> AggregateResult: + result = AggregateResult() + for record in records: + result.total_challenges += 1 + solved = bool(record["solved"]) + if solved: + result.solved_challenges += 1 + result.total_duration_seconds += float(record["duration_seconds"]) + result.total_cost_usd += float(record["total_cost_usd"]) + + difficulty = str(record["difficulty"]) + stats = result.by_difficulty.setdefault(difficulty, DifficultyStats()) + stats.total += 1 + if solved: + stats.solved += 1 + + return result + + +# --------------------------------------------------------------------------- +# Markdown rendering, matching the shape of the existing benchmarks/README.md +# tables so that file's tables can be replaced wholesale by this output. +# --------------------------------------------------------------------------- + + +def _format_minutes(seconds: float) -> str: + return f"~{round(seconds / 60)} minutes" + + +def _format_cost(usd: float) -> str: + return f"~${usd:,.0f}" + + +def render_markdown(agg: AggregateResult) -> str: + lines: list[str] = [] + lines.append("| Benchmark | Challenges | Success Rate |") + lines.append("|-----------|------------|--------------|") + lines.append( + f"| XBEN | {agg.total_challenges} | **{agg.success_rate:.0f}%** |", + ) + lines.append("") + lines.append("**Performance by Difficulty:**") + lines.append("") + lines.append("| Difficulty | Solved | Success Rate |") + lines.append("|------------|--------|--------------|") + for key in _DIFFICULTY_ORDER: + stats = agg.by_difficulty.get(key) + if stats is None: + continue + label = _DIFFICULTY_LABELS[key] + lines.append(f"| {label} | {stats.solved}/{stats.total} | {stats.success_rate:.0f}% |") + lines.append("") + lines.append("**Resource Usage:**") + lines.append(f"- Average solve time: {_format_minutes(agg.avg_duration_seconds)}") + lines.append( + f"- Total cost: {_format_cost(agg.total_cost_usd)} for {agg.total_challenges} challenges", + ) + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- +# --check: parse the numbers currently hand-typed in benchmarks/README.md and +# compare them against what's computed from committed data. +# --------------------------------------------------------------------------- + + +@dataclass +class ReadmeClaims: + total_challenges: int + success_rate: float + by_difficulty: dict[str, DifficultyStats] + + +_OVERALL_ROW_RE = re.compile( + r"\|\s*\[?XBEN\]?[^|]*\|\s*(\d+)\s*\|\s*\*\*(\d+(?:\.\d+)?)%\*\*\s*\|", +) +_DIFFICULTY_ROW_RE = re.compile( + r"\|\s*Level\s+(\d+)\s*\(([A-Za-z]+)\)\s*\|\s*(\d+)/(\d+)\s*\|\s*(\d+(?:\.\d+)?)%\s*\|", +) + + +def parse_readme_claims(readme_text: str) -> ReadmeClaims: + """Extract the headline numbers from the existing hand-typed README table.""" + overall_match = _OVERALL_ROW_RE.search(readme_text) + if not overall_match: + raise SchemaValidationError( + "could not find the XBEN results row (e.g. '| XBEN | 104 | **96%** |') in the README", + ) + total_challenges = int(overall_match.group(1)) + success_rate = float(overall_match.group(2)) + + by_difficulty: dict[str, DifficultyStats] = {} + for match in _DIFFICULTY_ROW_RE.finditer(readme_text): + level_name = match.group(2).lower() + difficulty = next((d for d in _DIFFICULTY_ORDER if d == level_name), None) + if difficulty is None: + continue + solved, total = int(match.group(3)), int(match.group(4)) + by_difficulty[difficulty] = DifficultyStats(solved=solved, total=total) + + return ReadmeClaims( + total_challenges=total_challenges, + success_rate=success_rate, + by_difficulty=by_difficulty, + ) + + +def diff_against_claims(agg: AggregateResult, claims: ReadmeClaims) -> list[str]: + """Return a list of human-readable mismatches between ``agg`` and ``claims``.""" + mismatches: list[str] = [] + + if agg.total_challenges != claims.total_challenges: + mismatches.append( + f"total challenges: README claims {claims.total_challenges}, " + f"committed data has {agg.total_challenges}", + ) + if round(agg.success_rate) != round(claims.success_rate): + mismatches.append( + f"overall success rate: README claims {claims.success_rate:.0f}%, " + f"committed data computes {agg.success_rate:.0f}%", + ) + + for difficulty in _DIFFICULTY_ORDER: + # A difficulty absent from either side is equivalent to a zero-challenge + # DifficultyStats(0, 0) for that side, so a table row with "0/0" and no + # committed challenges of that difficulty compare as equal rather than + # as "present on one side only". + claimed = claims.by_difficulty.get(difficulty, DifficultyStats()) + actual = agg.by_difficulty.get(difficulty, DifficultyStats()) + if claimed.solved != actual.solved or claimed.total != actual.total: + mismatches.append( + f"difficulty '{difficulty}': README claims {claimed.solved}/{claimed.total}, " + f"committed data has {actual.solved}/{actual.total}", + ) + + return mismatches + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Validate and aggregate committed XBEN run records into the " + "benchmarks/README.md results table." + ), + ) + parser.add_argument( + "results_dir", + type=Path, + nargs="?", + default=_DEFAULT_RESULTS_DIR, + help=f"Directory of run-record *.json files (default: {_DEFAULT_RESULTS_DIR}).", + ) + parser.add_argument( + "--schema", + type=Path, + default=_DEFAULT_SCHEMA_PATH, + help=f"Path to the run-record JSON Schema (default: {_DEFAULT_SCHEMA_PATH}).", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="Write the markdown table to this file instead of stdout.", + ) + parser.add_argument( + "--check", + action="store_true", + help=( + "Exit non-zero if the aggregated numbers don't match what's currently " + "hand-typed in the README (this is what CI runs)." + ), + ) + parser.add_argument( + "--readme", + type=Path, + default=_DEFAULT_README_PATH, + help=f"README to check against with --check (default: {_DEFAULT_README_PATH}).", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + try: + schema = load_schema(args.schema) + records = load_results(args.results_dir, schema) + except (SchemaValidationError, FileNotFoundError) as exc: + error_console.print(f"[bold red]error:[/bold red] {exc}") + return 1 + + agg = aggregate(records) + markdown = render_markdown(agg) + + if args.out: + args.out.write_text(markdown, encoding="utf-8") + console.print(f"Wrote markdown table to {args.out}") + else: + console.print(markdown, highlight=False, soft_wrap=True) + + if args.check: + if not args.readme.is_file(): + error_console.print(f"[bold red]error:[/bold red] README not found: {args.readme}") + return 1 + readme_text = args.readme.read_text(encoding="utf-8") + try: + claims = parse_readme_claims(readme_text) + except SchemaValidationError as exc: + error_console.print(f"[bold red]error:[/bold red] {exc}") + return 1 + + mismatches = diff_against_claims(agg, claims) + if mismatches: + error_console.print( + f"[bold red]--check failed:[/bold red] aggregated numbers from " + f"{args.results_dir} do not match {args.readme}:", + ) + for mismatch in mismatches: + error_console.print(f" - {mismatch}") + return 1 + console.print( + f"[bold green]--check passed:[/bold green] {args.readme} matches committed data." + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_xben_run.py b/tests/test_xben_run.py new file mode 100644 index 000000000..52ba65ff9 --- /dev/null +++ b/tests/test_xben_run.py @@ -0,0 +1,331 @@ +"""Tests for the XBEN benchmark runner (benchmarks/xben/run.py). + +This module drives a real ``strix`` subprocess against a live Docker + LLM +environment, neither of which is available here. Every test below mocks +``subprocess.run`` (via ``run.run_strix_subprocess``) and the filesystem +artifacts a real run would produce, so only the argument-parsing and +run-record-construction logic is exercised -- never a real scan. +""" + +from __future__ import annotations + +import json +import subprocess +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock + +import pytest + +from benchmarks.xben import run + + +if TYPE_CHECKING: + from pathlib import Path + + +def _manifest_entry(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "challenge_id": "XBEN-001-24", + "challenge_name": "Test Challenge", + "difficulty": "easy", + "target": "https://target.example.com", + "flag": "FLAG{test}", + } + base.update(overrides) + return base + + +class TestChallengeFromDict: + def test_valid_entry(self) -> None: + spec = run._challenge_from_dict(_manifest_entry(), default_id=None, source="test") + assert spec.challenge_id == "XBEN-001-24" + assert spec.difficulty == "easy" + assert spec.target == "https://target.example.com" + assert spec.flag == "FLAG{test}" + + def test_default_id_used_when_missing(self) -> None: + entry = _manifest_entry() + del entry["challenge_id"] + spec = run._challenge_from_dict(entry, default_id="dir-name", source="test") + assert spec.challenge_id == "dir-name" + + def test_missing_id_raises(self) -> None: + entry = _manifest_entry() + del entry["challenge_id"] + with pytest.raises(run.ChallengeSpecError, match="challenge_id"): + run._challenge_from_dict(entry, default_id=None, source="test") + + def test_invalid_difficulty_raises(self) -> None: + with pytest.raises(run.ChallengeSpecError, match="difficulty"): + run._challenge_from_dict( + _manifest_entry(difficulty="impossible"), default_id=None, source="test" + ) + + def test_missing_target_raises(self) -> None: + entry = _manifest_entry() + del entry["target"] + with pytest.raises(run.ChallengeSpecError, match="target"): + run._challenge_from_dict(entry, default_id=None, source="test") + + def test_missing_flag_is_allowed(self) -> None: + entry = _manifest_entry() + del entry["flag"] + spec = run._challenge_from_dict(entry, default_id=None, source="test") + assert spec.flag is None + + +class TestLoadManifest: + def test_loads_list_of_entries(self, tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps([_manifest_entry(), _manifest_entry(challenge_id="B")])) + specs = run.load_manifest(manifest) + assert [s.challenge_id for s in specs] == ["XBEN-001-24", "B"] + + def test_non_list_manifest_raises(self, tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps({"not": "a list"})) + with pytest.raises(run.ChallengeSpecError, match="must be a JSON array"): + run.load_manifest(manifest) + + +class TestDiscoverChallenges: + def test_discovers_challenge_json_per_subdirectory(self, tmp_path: Path) -> None: + chal_dir = tmp_path / "chal-1" + chal_dir.mkdir() + entry = _manifest_entry() + del entry["challenge_id"] + (chal_dir / "challenge.json").write_text(json.dumps(entry)) + + specs = run.discover_challenges(tmp_path) + assert len(specs) == 1 + assert specs[0].challenge_id == "chal-1" + + def test_missing_directory_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + run.discover_challenges(tmp_path / "missing") + + def test_no_challenges_found_raises(self, tmp_path: Path) -> None: + with pytest.raises(run.ChallengeSpecError, match=r"no challenge\.json"): + run.discover_challenges(tmp_path) + + +class TestBuildStrixCommand: + def test_basic_command_shape(self) -> None: + command = run.build_strix_command( + strix_bin="strix", + target="https://target.example.com", + run_name="xben-run-1", + max_budget_usd=None, + max_turns=None, + ) + assert command == [ + "strix", + "--target", + "https://target.example.com", + "--non-interactive", + "--run-name", + "xben-run-1", + ] + + def test_includes_optional_budget_and_turns(self) -> None: + command = run.build_strix_command( + strix_bin="strix", + target="https://target.example.com", + run_name="xben-run-1", + max_budget_usd=5.0, + max_turns=100, + ) + assert "--max-budget" in command + assert command[command.index("--max-budget") + 1] == "5.0" + assert "--max-turns" in command + assert command[command.index("--max-turns") + 1] == "100" + + +class TestFlagWasMatched: + def test_no_flag_means_not_matched(self, tmp_path: Path) -> None: + assert run.flag_was_matched(tmp_path, None) is False + + def test_flag_found_in_vulnerabilities_json(self, tmp_path: Path) -> None: + (tmp_path / "vulnerabilities.json").write_text('[{"evidence": "FLAG{found-it}"}]') + assert run.flag_was_matched(tmp_path, "FLAG{found-it}") is True + + def test_flag_found_in_report_markdown(self, tmp_path: Path) -> None: + (tmp_path / "penetration_test_report.md").write_text("We recovered FLAG{in-report}.") + assert run.flag_was_matched(tmp_path, "FLAG{in-report}") is True + + def test_flag_not_present_anywhere(self, tmp_path: Path) -> None: + (tmp_path / "vulnerabilities.json").write_text("[]") + assert run.flag_was_matched(tmp_path, "FLAG{missing}") is False + + +class TestBuildRunRecord: + def test_builds_conformant_record_from_run_json(self, tmp_path: Path) -> None: + run_dir = tmp_path / "strix_runs" / "xben-run-1" + run_dir.mkdir(parents=True) + (run_dir / "run.json").write_text( + json.dumps( + { + "status": "completed", + "llm_usage": {"cost": 2.5, "input_tokens": 1000, "output_tokens": 200}, + }, + ), + ) + (run_dir / "vulnerabilities.json").write_text('[{"evidence": "FLAG{yes}"}]') + + challenge = run.ChallengeSpec( + challenge_id="XBEN-001-24", + challenge_name="Test", + difficulty="easy", + target="https://target.example.com", + flag="FLAG{yes}", + ) + started = datetime(2026, 1, 1, tzinfo=UTC) + finished = datetime(2026, 1, 1, 0, 5, tzinfo=UTC) + + record = run.build_run_record( + challenge=challenge, + run_dir=run_dir, + run_id="xben-run-1", + model="anthropic/claude-sonnet-4-5", + mode="black-box", + started_at=started, + finished_at=finished, + ) + + assert record["challenge_id"] == "XBEN-001-24" + assert record["difficulty"] == "easy" + assert record["model"] == "anthropic/claude-sonnet-4-5" + assert record["mode"] == "black-box" + assert record["flag_matched"] is True + assert record["solved"] is True + assert record["duration_seconds"] == pytest.approx(300.0) + assert record["total_cost_usd"] == pytest.approx(2.5) + assert record["input_tokens"] == 1000 + assert record["output_tokens"] == 200 + assert record["run_id"] == "xben-run-1" + + def test_missing_run_json_raises(self, tmp_path: Path) -> None: + challenge = run.ChallengeSpec( + challenge_id="X", + challenge_name="X", + difficulty="easy", + target="https://target.example.com", + ) + with pytest.raises(FileNotFoundError): + run.build_run_record( + challenge=challenge, + run_dir=tmp_path, + run_id="run-1", + model="m", + mode="black-box", + started_at=datetime.now(tz=UTC), + finished_at=datetime.now(tz=UTC), + ) + + def test_unsolved_without_flag_uses_status(self, tmp_path: Path) -> None: + run_dir = tmp_path + (run_dir / "run.json").write_text(json.dumps({"status": "completed", "llm_usage": {}})) + challenge = run.ChallengeSpec( + challenge_id="X", + challenge_name="X", + difficulty="easy", + target="https://target.example.com", + flag=None, + ) + record = run.build_run_record( + challenge=challenge, + run_dir=run_dir, + run_id="run-1", + model="m", + mode="black-box", + started_at=datetime.now(tz=UTC), + finished_at=datetime.now(tz=UTC), + ) + assert record["flag_matched"] is False + assert record["solved"] is True # falls back to run status when there's no flag to check + + +class TestRunOneChallengeWithMockedSubprocess: + def test_invokes_subprocess_and_builds_record( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + challenge = run.ChallengeSpec( + challenge_id="XBEN-001-24", + challenge_name="Test", + difficulty="easy", + target="https://target.example.com", + flag="FLAG{yes}", + ) + + fake_result = subprocess.CompletedProcess( + args=["strix"], returncode=0, stdout="", stderr="" + ) + mock_run = MagicMock(return_value=fake_result) + monkeypatch.setattr(run, "run_strix_subprocess", mock_run) + + def fake_run_dir_for(run_name: str) -> Path: + run_dir = tmp_path / run_name + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text( + json.dumps({"status": "completed", "llm_usage": {"cost": 1.0}}), + ) + (run_dir / "vulnerabilities.json").write_text('[{"evidence": "FLAG{yes}"}]') + return run_dir + + args = run.build_parser().parse_args( + ["--manifest", "unused.json", "--model", "test-model"], + ) + + record = run.run_one_challenge( + challenge, + args=args, + env={}, + run_dir_for_name=fake_run_dir_for, + ) + + mock_run.assert_called_once() + called_command = mock_run.call_args.args[0] + assert called_command[:3] == ["strix", "--target", "https://target.example.com"] + assert record["challenge_id"] == "XBEN-001-24" + assert record["flag_matched"] is True + + +class TestMainDryRun: + def test_dry_run_prints_commands_without_running_strix( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps([_manifest_entry()])) + + mock_run = MagicMock() + monkeypatch.setattr(run, "run_strix_subprocess", mock_run) + + exit_code = run.main(["--manifest", str(manifest), "--model", "test-model", "--dry-run"]) + + assert exit_code == 0 + mock_run.assert_not_called() + captured = capsys.readouterr() + assert "XBEN-001-24" in captured.out + assert "--target" in captured.out + + def test_unknown_challenge_id_filter_errors( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps([_manifest_entry()])) + + exit_code = run.main( + [ + "--manifest", + str(manifest), + "--model", + "test-model", + "--challenge-id", + "does-not-exist", + "--dry-run", + ], + ) + assert exit_code == 1 + captured = capsys.readouterr() + assert "does-not-exist" in captured.err diff --git a/tests/test_xben_score.py b/tests/test_xben_score.py new file mode 100644 index 000000000..7b79da495 --- /dev/null +++ b/tests/test_xben_score.py @@ -0,0 +1,296 @@ +"""Tests for the XBEN benchmark scorer (benchmarks/xben/score.py).""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from benchmarks.xben import score + + +if TYPE_CHECKING: + from pathlib import Path + + +def _schema() -> dict[str, Any]: + return score.load_schema(score._DEFAULT_SCHEMA_PATH) + + +def _record(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "challenge_id": "XBEN-001-24", + "challenge_name": "Test Challenge", + "difficulty": "easy", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": True, + "flag_matched": True, + "duration_seconds": 300, + "total_cost_usd": 1.5, + "input_tokens": 1000, + "output_tokens": 100, + "started_at": "2026-01-01T00:00:00Z", + "finished_at": "2026-01-01T00:05:00Z", + "run_id": "run-1", + } + base.update(overrides) + return base + + +class TestValidateRecord: + def test_valid_record_passes(self) -> None: + score.validate_record(_record(), _schema(), source="test.json") + + def test_missing_required_field_is_reported_by_name(self) -> None: + record = _record() + del record["challenge_id"] + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "bad.json" in str(exc_info.value) + assert "challenge_id" in str(exc_info.value) + + def test_invalid_enum_value_is_reported(self) -> None: + record = _record(difficulty="impossible") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "difficulty" in str(exc_info.value) + + def test_wrong_type_is_reported(self) -> None: + record = _record(solved="yes") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "solved" in str(exc_info.value) + + def test_negative_number_below_minimum_is_reported(self) -> None: + record = _record(duration_seconds=-1) + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "duration_seconds" in str(exc_info.value) + + def test_invalid_date_time_format_is_reported(self) -> None: + record = _record(started_at="not-a-date") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "started_at" in str(exc_info.value) + + def test_unexpected_field_is_reported(self) -> None: + record = _record(unknown_field="oops") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "unknown_field" in str(exc_info.value) + + def test_optional_notes_field_is_allowed(self) -> None: + score.validate_record(_record(notes="all good"), _schema(), source="test.json") + + def test_non_object_record_is_rejected(self) -> None: + with pytest.raises(score.SchemaValidationError): + score.validate_record(["not", "an", "object"], _schema(), source="bad.json") + + +class TestLoadResults: + def test_loads_valid_records(self, tmp_path: Path) -> None: + (tmp_path / "a.json").write_text(json.dumps(_record(challenge_id="A")), encoding="utf-8") + (tmp_path / "b.json").write_text(json.dumps(_record(challenge_id="B")), encoding="utf-8") + records = score.load_results(tmp_path, _schema()) + assert {r["challenge_id"] for r in records} == {"A", "B"} + + def test_missing_directory_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + score.load_results(tmp_path / "does-not-exist", _schema()) + + def test_empty_directory_raises(self, tmp_path: Path) -> None: + with pytest.raises(score.SchemaValidationError, match="no run-record"): + score.load_results(tmp_path, _schema()) + + def test_invalid_json_names_the_file(self, tmp_path: Path) -> None: + (tmp_path / "broken.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(score.SchemaValidationError, match=r"broken\.json"): + score.load_results(tmp_path, _schema()) + + def test_invalid_record_stops_loading_and_names_file(self, tmp_path: Path) -> None: + (tmp_path / "good.json").write_text(json.dumps(_record()), encoding="utf-8") + bad = _record() + del bad["model"] + (tmp_path / "bad.json").write_text(json.dumps(bad), encoding="utf-8") + with pytest.raises(score.SchemaValidationError, match=r"bad\.json"): + score.load_results(tmp_path, _schema()) + + def test_fixture_results_directory_is_valid(self) -> None: + # The committed example fixtures under benchmarks/xben/results/ must + # themselves validate cleanly -- this is what score.py --help's + # documented default invocation runs against. + records = score.load_results(score._DEFAULT_RESULTS_DIR, _schema()) + assert len(records) >= 1 + + +class TestAggregate: + def test_overall_success_rate(self) -> None: + records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="easy", solved=False), + ] + agg = score.aggregate(records) + assert agg.total_challenges == 2 + assert agg.solved_challenges == 1 + assert agg.success_rate == pytest.approx(50.0) + + def test_breakdown_by_difficulty(self) -> None: + records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="easy", solved=True), + _record(challenge_id="3", difficulty="hard", solved=False), + ] + agg = score.aggregate(records) + assert agg.by_difficulty["easy"].solved == 2 + assert agg.by_difficulty["easy"].total == 2 + assert agg.by_difficulty["hard"].solved == 0 + assert agg.by_difficulty["hard"].total == 1 + + def test_average_duration_and_cost(self) -> None: + records = [ + _record(challenge_id="1", duration_seconds=100, total_cost_usd=1.0), + _record(challenge_id="2", duration_seconds=300, total_cost_usd=3.0), + ] + agg = score.aggregate(records) + assert agg.avg_duration_seconds == pytest.approx(200.0) + assert agg.total_cost_usd == pytest.approx(4.0) + assert agg.avg_cost_usd == pytest.approx(2.0) + + def test_empty_aggregate_does_not_divide_by_zero(self) -> None: + agg = score.aggregate([]) + assert agg.success_rate == 0.0 + assert agg.avg_duration_seconds == 0.0 + assert agg.avg_cost_usd == 0.0 + + +class TestRenderMarkdown: + def test_contains_expected_headers_and_rows(self) -> None: + agg = score.aggregate( + [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="hard", solved=False), + ], + ) + markdown = score.render_markdown(agg) + assert "| Benchmark | Challenges | Success Rate |" in markdown + assert "| Difficulty | Solved | Success Rate |" in markdown + assert "Level 1 (Easy)" in markdown + assert "Level 3 (Hard)" in markdown + assert "Average solve time" in markdown + assert "Total cost" in markdown + + +class TestReadmeCheck: + _README_TEMPLATE = """\ +## Results + +| Benchmark | Challenges | Success Rate | +|-----------|------------|--------------| +| XBEN | {total} | **{rate:.0f}%** | + +**Performance by Difficulty:** + +| Difficulty | Solved | Success Rate | +|------------|--------|--------------| +| Level 1 (Easy) | {easy_solved}/{easy_total} | {easy_rate:.0f}% | +| Level 2 (Medium) | {medium_solved}/{medium_total} | {medium_rate:.0f}% | +| Level 3 (Hard) | {hard_solved}/{hard_total} | {hard_rate:.0f}% | +""" + + def _readme_matching(self, agg: score.AggregateResult) -> str: + easy = agg.by_difficulty.get("easy", score.DifficultyStats()) + medium = agg.by_difficulty.get("medium", score.DifficultyStats()) + hard = agg.by_difficulty.get("hard", score.DifficultyStats()) + return self._README_TEMPLATE.format( + total=agg.total_challenges, + rate=agg.success_rate, + easy_solved=easy.solved, + easy_total=easy.total, + easy_rate=easy.success_rate, + medium_solved=medium.solved, + medium_total=medium.total, + medium_rate=medium.success_rate, + hard_solved=hard.solved, + hard_total=hard.total, + hard_rate=hard.success_rate, + ) + + def test_matching_readme_has_no_mismatches(self) -> None: + records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="medium", solved=True), + _record(challenge_id="3", difficulty="hard", solved=False), + ] + agg = score.aggregate(records) + readme_text = self._readme_matching(agg) + claims = score.parse_readme_claims(readme_text) + assert score.diff_against_claims(agg, claims) == [] + + def test_mismatched_readme_is_reported(self) -> None: + records = [_record(challenge_id="1", difficulty="easy", solved=True)] + agg = score.aggregate(records) + readme_text = self._readme_matching(agg) + # Now aggregate a different set of records that disagrees. + other_records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="easy", solved=False), + ] + other_agg = score.aggregate(other_records) + claims = score.parse_readme_claims(readme_text) + mismatches = score.diff_against_claims(other_agg, claims) + assert mismatches + assert any("total challenges" in m for m in mismatches) + + def test_real_readme_and_fixtures_are_out_of_sync(self) -> None: + # This is the exact situation documented in benchmarks/xben/results/README.md: + # the fixture data is synthetic and intentionally does not reproduce the + # real README's headline claim, so --check against it must fail. + agg = score.aggregate(score.load_results(score._DEFAULT_RESULTS_DIR, _schema())) + readme_text = score._DEFAULT_README_PATH.read_text(encoding="utf-8") + claims = score.parse_readme_claims(readme_text) + assert score.diff_against_claims(agg, claims) != [] + + +class TestCli: + def test_main_prints_table_to_stdout(self, capsys: pytest.CaptureFixture[str]) -> None: + exit_code = score.main([str(score._DEFAULT_RESULTS_DIR)]) + assert exit_code == 0 + captured = capsys.readouterr() + assert "Benchmark" in captured.out + + def test_main_writes_table_to_out_file(self, tmp_path: Path) -> None: + out_path = tmp_path / "table.md" + exit_code = score.main([str(score._DEFAULT_RESULTS_DIR), "--out", str(out_path)]) + assert exit_code == 0 + assert "Benchmark" in out_path.read_text(encoding="utf-8") + + def test_main_check_fails_against_real_readme_with_fixture_data(self) -> None: + exit_code = score.main([str(score._DEFAULT_RESULTS_DIR), "--check"]) + assert exit_code == 1 + + def test_main_check_passes_with_matching_readme(self, tmp_path: Path) -> None: + (tmp_path / "a.json").write_text( + json.dumps(_record(challenge_id="A", difficulty="easy", solved=True)), + encoding="utf-8", + ) + agg = score.aggregate([_record(challenge_id="A", difficulty="easy", solved=True)]) + readme_path = tmp_path / "README.md" + readme_path.write_text(TestReadmeCheck()._readme_matching(agg), encoding="utf-8") + exit_code = score.main( + [str(tmp_path), "--check", "--readme", str(readme_path)], + ) + assert exit_code == 0 + + def test_main_reports_invalid_record_and_exits_nonzero( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + bad = _record() + del bad["run_id"] + (tmp_path / "bad.json").write_text(json.dumps(bad), encoding="utf-8") + exit_code = score.main([str(tmp_path)]) + assert exit_code == 1 + captured = capsys.readouterr() + assert "bad.json" in captured.err