From cc8f8f7b70d7a1c09c0a69ab0a64053a37fba41e Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:48:53 +0100 Subject: [PATCH 1/9] feat(agents): warn before exploitation agents spawn without a threat model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two third-party benchmarks found Strix's exploitation subagents get spawned before real recon/threat-modeling happens, even though root_agent.md already tells the root agent (in prose) to recon-then-model first on black-box targets. An LLM can skip prose guidance under its own initiative, so add a code-level backstop: create_agent now attaches a non-blocking `warning` field when spawning a child with vulnerabilities-category skills before any threat model has been established for the scan, folding in a note when no reconnaissance coverage has been recorded either. The agent still spawns — this is a signal, not a gate. Also expose any_threat_model_exists() on the threat_model store and document the new warning in root_agent.md. Co-Authored-By: Claude Sonnet 5 --- strix/skills/coordination/root_agent.md | 2 + strix/tools/agents_graph/tools.py | 101 +++++++++++++++++++++++- strix/tools/threat_model/tools.py | 16 ++++ 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/strix/skills/coordination/root_agent.md b/strix/skills/coordination/root_agent.md index fa88aaa5a..83f1c0c19 100644 --- a/strix/skills/coordination/root_agent.md +++ b/strix/skills/coordination/root_agent.md @@ -35,6 +35,8 @@ Every scan needs one shared answer to "who is the attacker here, and what are th Either way you write it with the least information anyone on this scan will ever have, so expect it to be wrong somewhere. Subagents correct it with `amend_threat_model`, which appends an attributed addendum instead of overwriting — expect many of these on a black-box run, as authenticating, pivoting between roles, and reaching internal surfaces is exactly what turns inference into fact. Read the amendments back before you write the final report: an agent telling you a boundary you called trusted is attacker-reachable is a finding about your model, not a note. Only call `save_threat_model` again to fold accumulated amendments into the body; it replaces the document and clears them. +`create_agent` backstops this with a code-level check: if you spawn a child with exploitation/vulnerability skills before any threat model exists for this scan, the response carries a `warning` field (and folds in a note when no reconnaissance coverage has been recorded either). It still spawns the agent — this is not a hard block — but treat that `warning` as a real signal that you skipped ahead, not as noise to ignore. Seeing it is your cue to pause the next such spawn until recon has run and `save_threat_model` has been called. + ## Reconcile Coverage Before Finishing Coverage entries are shared and mutable. Before `finish_scan`, list the `needs_follow_up` rows: each one is either work you still owe or a row somebody already resolved without updating. Assign the former to a subagent and have it call `update_coverage` on the existing entry rather than recording a second one — a stale open item sitting next to its own resolution is worse than either alone. diff --git a/strix/tools/agents_graph/tools.py b/strix/tools/agents_graph/tools.py index da05bb1e7..b7a17226a 100644 --- a/strix/tools/agents_graph/tools.py +++ b/strix/tools/agents_graph/tools.py @@ -16,7 +16,9 @@ from strix.core.agents import Status, coordinator_from_context from strix.core.execution import notify_parent_on_terminal from strix.core.hooks import LLM_TURN_KEY from strix.report.state import get_global_report_state -from strix.skills import validate_requested_skills +from strix.skills import get_available_skills, validate_requested_skills +from strix.tools.coverage.tools import get_coverage_entries +from strix.tools.threat_model.tools import any_threat_model_exists _ACTIVE_STATUSES: frozenset[str] = frozenset({"running", "waiting"}) @@ -25,6 +27,98 @@ _ACTIVE_STATUSES: frozenset[str] = frozenset({"running", "waiting"}) logger = logging.getLogger(__name__) +# --- Soft threat-model gate for exploitation-oriented children ------------- +# +# Why this exists: root_agent.md already *tells* the root agent, in prose, to +# recon and build a threat model before spawning exploitation/hunter +# subagents (especially black-box, where "recon comes first"). But that is +# prompt-only guidance an LLM can and does skip under its own initiative. +# Two independent third-party benchmarks found this actually happens: a +# grey-box eval measured 1/20 known vulns found on a real app, and a +# separate review found ~1,300 of ~1,350 requests spent fuzzing guessed +# conventional API paths, never discovering the app's real nested routes, +# because exploitation started before real recon did. +# +# This is a *soft* backstop: create_agent still spawns the child (a false +# block could stall a legitimate white-box/small-scope scan where a threat +# model is genuine overkill) but attaches a `warning` field to the response +# so the root agent has a code-level signal to weigh, not just prose to +# ignore. Tune the heuristic here — the skill category treated as +# "exploitation" and the coverage risk_area substrings treated as "recon +# happened" — rather than inline in create_agent. +_EXPLOITATION_SKILL_CATEGORY = "vulnerabilities" +_RECON_RISK_AREA_HINTS: tuple[str, ...] = ( + "recon", + "asset discovery", + "enumeration", + "fingerprint", + "attack surface", +) + + +def _exploitation_skill_names() -> frozenset[str]: + """Skill names under ``strix/skills/vulnerabilities/``, read from disk. + + Derived rather than hardcoded so a new vulnerability skill file is + covered automatically, without a matching edit here going stale. + """ + return frozenset( + skill["name"] for skill in get_available_skills().get(_EXPLOITATION_SKILL_CATEGORY, []) + ) + + +def _requests_exploitation_skill(skill_list: list[str]) -> bool: + """Whether any requested skill looks exploitation/vulnerability-oriented.""" + if not skill_list: + return False + exploitation_skills = _exploitation_skill_names() + return any( + skill in exploitation_skills or skill.rsplit("/", 1)[-1] in exploitation_skills + for skill in skill_list + ) + + +def _no_recon_coverage_yet() -> bool: + """True when nothing in the shared coverage ledger looks like recon work.""" + entries = get_coverage_entries() + if not entries: + return True + return not any( + hint in str(entry.get("risk_area", "")).lower() + for entry in entries + for hint in _RECON_RISK_AREA_HINTS + ) + + +def _threat_model_gate_warning(skill_list: list[str]) -> str | None: + """Return the create_agent warning for this spawn, or None if it doesn't apply. + + Fires only when the child looks exploitation-oriented AND no threat model + has been established yet for this scan. The "no recon coverage either" signal + is folded into the same message rather than emitted as a second warning. + """ + if not _requests_exploitation_skill(skill_list): + return None + + if any_threat_model_exists(): + return None + + message = ( + "No threat model has been established for this scan yet. Black-box targets " + "should have reconnaissance run and a threat model saved via save_threat_model " + "before spawning exploitation specialists — otherwise they will guess at attack " + "surface instead of testing what's actually there." + ) + if _no_recon_coverage_yet(): + message += ( + " No coverage entries suggesting reconnaissance (asset discovery, " + "enumeration, fingerprinting) have been recorded yet either, which " + "suggests this may be the first specialist spawned before any recon " + "took place." + ) + return message + + def _ctx(ctx: RunContextWrapper) -> dict[str, Any]: return ctx.context if isinstance(ctx.context, dict) else {} @@ -603,6 +697,11 @@ async def create_agent( len(task or ""), ) + if result.get("success"): + warning = _threat_model_gate_warning(skill_list) + if warning: + result = {**result, "warning": warning} + return json.dumps( result, ensure_ascii=False, diff --git a/strix/tools/threat_model/tools.py b/strix/tools/threat_model/tools.py index 1015f4574..4b76f90f0 100644 --- a/strix/tools/threat_model/tools.py +++ b/strix/tools/threat_model/tools.py @@ -278,6 +278,22 @@ def _amendments_of(model: dict[str, Any]) -> list[dict[str, Any]]: return [item for item in raw if isinstance(item, dict)] +def any_threat_model_exists() -> bool: + """Whether any target on this scan has a derived threat model yet. + + Used by ``create_agent``'s soft gate (see ``strix/tools/agents_graph/tools.py``): + spawning an exploitation-oriented specialist before any threat model exists + is exactly the failure mode that gate warns about, so it needs a cheap, + read-only way to ask "has anyone derived one at all" without caring which + target it belongs to. + """ + with _store_lock: + return any( + isinstance(model.get("content"), str) and model["content"].strip() + for model in _MODELS.values() + ) + + def _not_found(identity: str) -> dict[str, Any]: return { "success": True, From 6b4f0cf1ba6ed560c37a4c8a129e657c991e0cf7 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:49:02 +0100 Subject: [PATCH 2/9] Add search_skills for fuzzy skill discovery Agents (and the orchestrator picking create_agent(skills=[...])) had no way to find a skill by fuzzy need without already knowing its exact filename-derived name. Add a dependency-free search_skills(query, top_k) lexical scorer in strix/skills/__init__.py (name/description/body-preview token overlap, weighted, cached like existing metadata reads), expose it as a paired search_skills tool alongside load_skill, register it in the agent toolset, and point agents at it from the system prompt when the exact skill name isn't known. Co-Authored-By: Claude Sonnet 5 --- strix/agents/factory.py | 3 +- strix/agents/prompts/system_prompt.jinja | 6 +- strix/skills/__init__.py | 77 +++++++++++++++++++++++- strix/tools/load_skill/tool.py | 29 +++++++++ tests/test_skill_dir_extension.py | 36 +++++++++++ 5 files changed, 146 insertions(+), 5 deletions(-) diff --git a/strix/agents/factory.py b/strix/agents/factory.py index b2fcbf08c..811daa32e 100644 --- a/strix/agents/factory.py +++ b/strix/agents/factory.py @@ -28,7 +28,7 @@ from strix.tools.agents_graph.tools import ( ) from strix.tools.coverage.tools import list_coverage, record_coverage, update_coverage from strix.tools.finish.tool import finish_scan -from strix.tools.load_skill.tool import load_skill +from strix.tools.load_skill.tool import load_skill, search_skills from strix.tools.mcp import call_mcp, describe_mcp, list_mcps from strix.tools.notes.tools import ( create_note, @@ -561,6 +561,7 @@ def _finish_tool_use_behavior( _BASE_TOOLS: tuple[Tool, ...] = ( think, load_skill, + search_skills, create_todo, list_todos, update_todo, diff --git a/strix/agents/prompts/system_prompt.jinja b/strix/agents/prompts/system_prompt.jinja index 18fffa99d..50527a37d 100644 --- a/strix/agents/prompts/system_prompt.jinja +++ b/strix/agents/prompts/system_prompt.jinja @@ -174,7 +174,7 @@ OPERATIONAL PRINCIPLES: - Prefer established industry-standard tools already available in the sandbox before writing custom scripts - Do NOT reinvent the wheel with ad hoc Python or shell code when a suitable existing tool can do the job reliably - Skills relevant to your task are preloaded into this prompt at scan start; refer back to them when you need vulnerability-, protocol-, or tool-specific guidance -- For skills not preloaded, use `load_skill` to pull them inline — prefer loading the matching skill before guessing payloads, workflows, or tool syntax from memory +- For skills not preloaded, use `load_skill` to pull them inline — prefer loading the matching skill before guessing payloads, workflows, or tool syntax from memory. If you don't know the exact skill name, call `search_skills` first with a free-text description of what you need, then `load_skill` the name it surfaces - Use custom Python or shell code when you want to dig deeper, automate custom workflows, batch operations, triage results, build target-specific validation, or do work that existing tools do not cover cleanly - Chain related weaknesses when needed to demonstrate real impact - Consider business logic and context in validation @@ -247,7 +247,7 @@ VALIDATION REQUIREMENTS: STATE & COORDINATION TOOLS (when and how): Every one of these tools writes to state the rest of the scan reads. Reaching for the tool is not optional bookkeeping — the agent after you sees your state, not your reasoning, so state you never wrote is context the scan permanently loses. - PLAN — `think`: use before any non-trivial or multi-step move to reason through approach, uncertainty, or what to do next. NOT for acknowledgements, summaries, or as filler before a final answer. -- SKILLS — `load_skill`: the skills matching your task are already inlined below under ``; `` lists the rest by name. When you are about to test a vuln class, protocol, tool, or framework whose skill is not already inlined, `load_skill` it FIRST and follow it, rather than guessing payloads or tool syntax from memory. +- SKILLS — `load_skill` / `search_skills`: the skills matching your task are already inlined below under ``; `` lists the rest by name. When you are about to test a vuln class, protocol, tool, or framework whose skill is not already inlined, `load_skill` it FIRST and follow it, rather than guessing payloads or tool syntax from memory. If you don't already know the exact name, use `search_skills` with a free-text description of the need to find it, then `load_skill` the match. - TODOS — `create_todo` / `list_todos` / `update_todo` / `mark_todo_done` / `mark_todo_pending` / `delete_todo`: your own working checklist for a multi-step task. Create todos when your task has several distinct steps so nothing is dropped across a long run; mark them done as you finish. This is private working memory — use `notes` for anything another agent needs. - NOTES — `create_note` / `list_notes` / `get_note` / `update_note` / `delete_note`: the scan's shared scratchpad, visible to every agent. Write a note for a durable cross-agent fact that is not a finding and not coverage — a working credential set, a discovered endpoint inventory, an enumerated tenant list, a rate-limit quirk the next agent needs. `update_note` to keep a living inventory current; `delete_note` only for something now wrong or superseded. Check `list_notes`/`get_note` before recon work so you build on what is already mapped instead of redoing it. - THREAT MODEL — `get_threat_model` / `amend_threat_model` / `save_threat_model`: covered above. `save_threat_model` REPLACES the whole document and clears amendments, so it is for establishing the baseline or folding amendments in (normally root) — to correct part of an existing model, `amend_threat_model` instead. @@ -537,7 +537,7 @@ Default user: pentester (sudo available) {% if available_skills %} -On-demand specialist skills. Spawn a specialist via `create_agent(skills=[...])`, or pull guidance inline for yourself via `load_skill(skills=[...])`. Anything wrapped in `` above is already loaded for you. +On-demand specialist skills. Spawn a specialist via `create_agent(skills=[...])`, or pull guidance inline for yourself via `load_skill(skills=[...])`. If you don't know the exact name, use `search_skills(query=...)` to find it first. Anything wrapped in `` above is already loaded for you. {% for category, skills in available_skills | dictsort -%} {% for skill in skills -%} diff --git a/strix/skills/__init__.py b/strix/skills/__init__.py index db5ae3e46..228f72cc6 100644 --- a/strix/skills/__init__.py +++ b/strix/skills/__init__.py @@ -3,7 +3,7 @@ import re from collections import Counter from collections.abc import Iterator from pathlib import Path -from typing import TypeGuard +from typing import Any, TypeGuard import yaml @@ -16,9 +16,12 @@ _FRONTMATTER_PATTERN = re.compile(r"^---\s*\n(?P.*?)\n---\s*\n", re.DOTALL _INTERNAL_SKILL_CATEGORIES: frozenset[str] = frozenset({"scan_modes", "coordination", "analysis"}) _ROOT_SKILL_CATEGORY = "root" +_SEARCH_BODY_CHARS = 500 +_SEARCH_TOKEN_PATTERN = re.compile(r"[a-z0-9]+") _EXTRA_SKILL_DIRS: list[Path] = [] _SKILL_METADATA_CACHE: dict[tuple[Path, int, int], dict[str, str]] = {} +_SKILL_BODY_PREVIEW_CACHE: dict[tuple[Path, int, int], str] = {} def _is_frontmatter_mapping(value: object) -> TypeGuard[dict[object, object]]: @@ -212,6 +215,78 @@ def get_available_skills() -> dict[str, list[dict[str, str]]]: return grouped +def _read_skill_body_preview(file_path: Path) -> str: + """Return (and cache) the first ~500 chars of a skill's body, for search.""" + try: + stat = file_path.stat() + except OSError: + return "" + cache_key = (file_path, stat.st_mtime_ns, stat.st_size) + cached = _SKILL_BODY_PREVIEW_CACHE.get(cache_key) + if cached is not None: + return cached + try: + content = file_path.read_text(encoding="utf-8") + except (OSError, ValueError): + return "" + _, body = _parse_skill_content(content, file_path) + preview = body[:_SEARCH_BODY_CHARS] + _SKILL_BODY_PREVIEW_CACHE[cache_key] = preview + return preview + + +def _tokenize(text: str) -> set[str]: + return set(_SEARCH_TOKEN_PATTERN.findall(text.lower())) + + +def search_skills(query: str, top_k: int = 5) -> list[dict[str, Any]]: + """Rank skills by lexical overlap with *query* (no embeddings, no new deps). + + Tokenizes the query and, for each skill, its name + frontmatter + description (weighted heavily) and the first ~500 chars of its body + (weighted lightly). Scores by weighted token overlap; ties break + alphabetically by ``category/name``. + """ + query_tokens = _tokenize(query) + if not query_tokens: + return [] + + scored: list[tuple[float, str, str, dict[str, Any]]] = [] + for category, name in _iter_user_skill_files(): + file_path = _qualified_skill_file_for_name(f"{category}/{name}") + if file_path is None: + continue + metadata = _read_skill_metadata(file_path) + description = " ".join(metadata.get("description", "").split()) + name_tokens = _tokenize(name.replace("_", " ")) + description_tokens = _tokenize(description) + body_tokens = _tokenize(_read_skill_body_preview(file_path)) + + score = ( + 3.0 * len(query_tokens & name_tokens) + + 2.0 * len(query_tokens & description_tokens) + + 1.0 * len(query_tokens & body_tokens) + ) + if score <= 0: + continue + scored.append( + ( + score, + category, + name, + { + "name": name, + "category": category, + "description": description, + "score": score, + }, + ) + ) + + scored.sort(key=lambda item: (-item[0], item[1], item[2])) + return [entry for _, _, _, entry in scored[:top_k]] + + def validate_requested_skills(skill_list: list[str], max_skills: int = 5) -> str | None: """Validate a list of user-passed skill names. diff --git a/strix/tools/load_skill/tool.py b/strix/tools/load_skill/tool.py index 3ef6f9ec0..d7f4ca1db 100644 --- a/strix/tools/load_skill/tool.py +++ b/strix/tools/load_skill/tool.py @@ -5,6 +5,7 @@ from __future__ import annotations from agents import RunContextWrapper, function_tool from strix.skills import load_skills, validate_requested_skills +from strix.skills import search_skills as _search_skills @function_tool(timeout=10) @@ -34,3 +35,31 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str: return "load_skill: no content loaded for requested skills." sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()] return "\n\n---\n\n".join(sections) + + +@function_tool(timeout=10) +async def search_skills(ctx: RunContextWrapper, query: str, top_k: int = 5) -> str: + """Find skills matching a fuzzy need when you don't know the exact skill name. + + Use this before ``load_skill`` when you know roughly what you need + (a vuln class, protocol, tool, or framework) but not the exact + filename-derived skill name — e.g. "GraphQL introspection" or + "JWT token handling". It ranks skills by lexical overlap with your + query and returns name/category/description only; call + ``load_skill`` on the name you pick to get the full guidance. + + Args: + query: Free-text description of what you're looking for. + top_k: Maximum number of ranked matches to return (default 5). + """ + del ctx + results = _search_skills(query, top_k=top_k) + if not results: + return f"search_skills: no matching skills found for query {query!r}." + lines = [ + f"- {result['category']}/{result['name']}: {result['description']}" + if result["description"] + else f"- {result['category']}/{result['name']}" + for result in results + ] + return "\n".join(lines) diff --git a/tests/test_skill_dir_extension.py b/tests/test_skill_dir_extension.py index 595e9e8f9..b6fab058f 100644 --- a/tests/test_skill_dir_extension.py +++ b/tests/test_skill_dir_extension.py @@ -11,6 +11,7 @@ from strix.skills import ( load_skills, register_skill_dir, registered_skill_dirs, + search_skills, skill_search_dirs, validate_requested_skills, ) @@ -259,6 +260,41 @@ def test_resolve_skills_gates_source_aware_skills_on_whitebox() -> None: assert "analysis/source_aware_discovery" in whitebox +def test_search_skills_surfaces_matching_builtin_skill() -> None: + results = search_skills("graphql introspection", top_k=3) + + assert results + assert results[0]["name"] == "graphql" + assert results[0]["category"] == "protocols" + assert "introspection" in results[0]["description"].lower() + assert results[0]["score"] > 0 + scores = [r["score"] for r in results] + assert scores == sorted(scores, reverse=True) + + +def test_search_skills_respects_top_k() -> None: + assert len(search_skills("authentication token security testing", top_k=2)) <= 2 + + +def test_search_skills_empty_query_returns_nothing() -> None: + assert search_skills("", top_k=5) == [] + + +def test_search_skills_finds_registered_skill(tmp_path: Path) -> None: + _write_skill( + tmp_path, + "extra", + "widget_lookup", + "---\nname: widget_lookup\ndescription: Widget lookup helper for gadget discovery\n" + "---\nwidget body about gadgets", + ) + register_skill_dir(tmp_path) + + results = search_skills("gadget discovery widget", top_k=5) + + assert any(r["name"] == "widget_lookup" and r["category"] == "extra" for r in results) + + def test_new_skill_files_load() -> None: names = [ "analysis/counterevidence", From 60422260a7572a50fb76e6330bd587415d1fa2d3 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:51:07 +0100 Subject: [PATCH 3/9] feat(proxy): circuit-break on unproductive request streaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a local, per-agent tracker that counts consecutive non-informative proxy responses (404s, or a non-2xx response shape repeating 3+ times in a row) and injects a strategy_warning field into repeat_request's result once a streak crosses 15, re-arming every further 15. This is distinct from the global $ budget and the per-turn tool-call limiter: it catches an agent burning budget one "reasonable" guessed-path request at a time, as reproduced by Protego's review (~1,300/~1,350 requests were 404s guessing conventional paths before ~$17 was spent with zero findings). Purely additive — never blocks or errors the call. Tracker state resets per scan alongside the other hydrate_* calls in runner.py. Co-Authored-By: Claude Sonnet 5 --- strix/core/runner.py | 2 + strix/tools/proxy/tools.py | 32 +++++- strix/tools/proxy/unproductive_tracker.py | 134 ++++++++++++++++++++++ tests/test_unproductive_tracker.py | 125 ++++++++++++++++++++ 4 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 strix/tools/proxy/unproductive_tracker.py create mode 100644 tests/test_unproductive_tracker.py diff --git a/strix/core/runner.py b/strix/core/runner.py index b40a36b0e..012239868 100644 --- a/strix/core/runner.py +++ b/strix/core/runner.py @@ -271,6 +271,7 @@ async def run_strix_scan( from strix.tools.coverage.tools import hydrate_coverage_from_disk from strix.tools.notes.tools import hydrate_notes_from_disk + from strix.tools.proxy.unproductive_tracker import reset_unproductive_tracker from strix.tools.threat_model.tools import hydrate_threat_models_from_disk from strix.tools.todo.tools import hydrate_todos_from_disk @@ -278,6 +279,7 @@ async def run_strix_scan( hydrate_notes_from_disk(state_dir) hydrate_coverage_from_disk(state_dir) hydrate_threat_models_from_disk(state_dir) + reset_unproductive_tracker() root_id: str | None = None if is_resume: diff --git a/strix/tools/proxy/tools.py b/strix/tools/proxy/tools.py index 0dc2a03c7..f9f2c053d 100644 --- a/strix/tools/proxy/tools.py +++ b/strix/tools/proxy/tools.py @@ -17,6 +17,7 @@ from agents import RunContextWrapper, function_tool from strix.runtime.caido_handle import CaidoBootstrapHandle from strix.tools.nullish import clean_optional from strix.tools.proxy import caido_api +from strix.tools.proxy.unproductive_tracker import record_response logger = logging.getLogger(__name__) @@ -50,6 +51,18 @@ ScopeAction = Literal["get", "list", "create", "update", "delete"] _CAIDO_CALL_LOCK = asyncio.Lock() +def _caller_agent_id(ctx: RunContextWrapper) -> str | None: + """Return the agent_id of the agent invoking this tool. + + Mirrors ``strix/tools/coverage/tools.py``'s ``_caller_identity`` — same + ``ctx.context["agent_id"]`` lookup, minus the agent-name resolution this + module doesn't need. + """ + inner: dict[str, Any] = ctx.context if isinstance(ctx.context, dict) else {} + raw_agent_id = inner.get("agent_id") + return raw_agent_id if isinstance(raw_agent_id, str) else None + + async def _ctx_client(ctx: RunContextWrapper) -> Client | None: inner: dict[str, Any] = ctx.context if isinstance(ctx.context, dict) else {} client: Client | CaidoBootstrapHandle | None = inner.get("caido_client") @@ -448,12 +461,12 @@ async def repeat_request( ensure_ascii=False, default=str, ) - return _format_replay_tool_result(replay) + return _format_replay_tool_result(replay, agent_id=_caller_agent_id(ctx)) except Exception as exc: # noqa: BLE001 return _err("repeat_request", exc) -def _format_replay_tool_result(replay: dict[str, Any]) -> str: +def _format_replay_tool_result(replay: dict[str, Any], *, agent_id: str | None = None) -> str: response = caido_api.parse_raw_response(replay.get("response_raw")) payload: dict[str, Any] = { "success": replay["status"] == "DONE", @@ -464,6 +477,21 @@ def _format_replay_tool_result(replay: dict[str, Any]) -> str: } if replay.get("error"): payload["error"] = replay["error"] + + if response is not None: + # Local, per-agent circuit breaker on unproductive requests (see + # unproductive_tracker.py) — distinct from the global $ budget and the + # per-turn tool-call-count limiter. Purely additive: never blocks or + # errors the call, just surfaces an actionable observation once an + # agent's own request stream shows a long run of dead ends. + warning = record_response( + agent_id, + response.get("status_code"), + response.get("length"), + ) + if warning: + payload["strategy_warning"] = warning + return json.dumps(payload, ensure_ascii=False, default=str) diff --git a/strix/tools/proxy/unproductive_tracker.py b/strix/tools/proxy/unproductive_tracker.py new file mode 100644 index 000000000..a08659d0e --- /dev/null +++ b/strix/tools/proxy/unproductive_tracker.py @@ -0,0 +1,134 @@ +"""Per-agent circuit breaker on unproductive proxy requests. + +This is deliberately independent from the two other guardrails in this +codebase: + +- ``strix/core/agents.py``'s budget pause/stop machinery watches the + *global* dollar spend across the whole scan. +- ``strix/config/tool_call_limits.py``'s ``TurnToolCallLimiter`` bounds how + many tool calls *one LLM turn* may queue, catching a runaway burst. + +Neither catches an agent that spends its budget slowly and "legitimately" — +one well-formed HTTP request at a time, each individually reasonable, that +just keeps landing on 404s or on the same generic error page while guessing +conventional REST paths. A third-party review of Strix (Protego) reproduced +exactly this failure mode: ~1,300 of ~1,350 requests against a live target +were 404s from guessed conventional paths (``/api/qa``) that never found the +real nested route (``/api/qa/questions``), burning ~$17 of budget before any +finding. + +This module tracks, per agent, how many *consecutive* proxy requests in a +row have been "non-informative" and returns an actionable warning message +once a threshold is crossed. It never blocks or errors the underlying tool +call — it only adds an extra, ignorable signal the agent can act on next +turn. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass + + +# Fired every time an agent's consecutive-unproductive-request count crosses +# a multiple of this value (15, 30, 45, ...). 15 is chosen as "clearly past +# ordinary recon noise": a few 404s while probing plausible paths is normal +# and expected pentesting behavior, but 15 in a row with zero new +# information is a strong, low-false-positive signal that the agent is +# fuzzing blind rather than reasoning about the target. Low enough to catch +# the failure mode long before hundreds of dollars are burned, high enough +# not to nag an agent doing legitimate varied probing. +UNPRODUCTIVE_STREAK_THRESHOLD = 15 + +# A non-404 response shape (status_code, response_body_length) that repeats +# identically this many times *in a row* is treated as non-informative too — +# this catches the "every guessed path returns the same generic HTML error +# page with a 200" variant of blind fuzzing, not just literal 404s. Small +# on purpose: two identical responses could be coincidence (e.g. two +# genuinely empty resources), but three in a row from an agent that is +# supposed to be varying its probes is a real signal. +IDENTICAL_SHAPE_REPEAT_THRESHOLD = 3 + +STRATEGY_WARNING_TEMPLATE = ( + "[STRATEGY WARNING] {streak} consecutive requests with no new information " + "(mostly 404s or repeated identical responses guessing conventional paths). " + "Stop path-guessing. Extract real routes from JS/HTML via the browser tool, " + "check for an OpenAPI/Swagger spec, or ask the coordinator for the app's " + "actual route list from recon findings." +) + + +@dataclass +class _AgentUnproductiveState: + consecutive_unproductive: int = 0 + last_shape: tuple[int, int] | None = None + shape_streak: int = 0 + + +_states: dict[str, _AgentUnproductiveState] = {} +_lock = threading.Lock() + + +def reset_unproductive_tracker() -> None: + """Clear all per-agent state. Call once per scan start, like the other + ``hydrate_*_from_disk`` resets in ``strix/core/runner.py`` — this tracker + is purely in-memory (nothing worth persisting across a resume) but it + must not leak counters from one scan into the next. + """ + with _lock: + _states.clear() + + +def _is_informative(state: _AgentUnproductiveState, status_code: int, response_length: int) -> bool: + """Update shape-tracking state and decide if this response was informative. + + Any 2xx is always informative — success responses vary enough (and matter + enough) that they must never be folded into the "blind fuzzing" bucket + even if several in a row happen to share a shape (e.g. IDOR testing + across many valid ids that all return the same shaped 200). + """ + shape = (status_code, response_length) + if shape == state.last_shape: + state.shape_streak += 1 + else: + state.last_shape = shape + state.shape_streak = 1 + + if 200 <= status_code < 300: + return True + + if status_code == 404: + return False + + return state.shape_streak < IDENTICAL_SHAPE_REPEAT_THRESHOLD + + +def record_response( + agent_id: str | None, status_code: int | None, response_length: int | None +) -> str | None: + """Record one proxy response for ``agent_id`` and return a warning to + surface to the agent, or ``None`` if nothing should be injected. + + Safe to call with ``agent_id=None`` or ``status_code=None`` (e.g. a + request that errored before any response came back) — these are no-ops + that neither raise nor advance any streak. + """ + if not agent_id or status_code is None: + return None + + length = response_length if isinstance(response_length, int) else 0 + + with _lock: + state = _states.setdefault(agent_id, _AgentUnproductiveState()) + informative = _is_informative(state, status_code, length) + + if informative: + state.consecutive_unproductive = 0 + return None + + state.consecutive_unproductive += 1 + streak = state.consecutive_unproductive + + if streak % UNPRODUCTIVE_STREAK_THRESHOLD == 0: + return STRATEGY_WARNING_TEMPLATE.format(streak=streak) + return None diff --git a/tests/test_unproductive_tracker.py b/tests/test_unproductive_tracker.py new file mode 100644 index 000000000..27efc0d5d --- /dev/null +++ b/tests/test_unproductive_tracker.py @@ -0,0 +1,125 @@ +"""Tests for the per-agent unproductive-request circuit breaker. + +Covers the tracker in isolation (``strix.tools.proxy.unproductive_tracker``) +and its wiring into ``repeat_request``'s response formatting +(``strix.tools.proxy.tools._format_replay_tool_result``). +""" + +from __future__ import annotations + +import json + +import pytest + +from strix.tools.proxy import tools +from strix.tools.proxy.unproductive_tracker import ( + UNPRODUCTIVE_STREAK_THRESHOLD, + record_response, + reset_unproductive_tracker, +) + + +@pytest.fixture(autouse=True) +def _reset() -> None: + reset_unproductive_tracker() + + +def test_consecutive_404s_past_threshold_triggers_warning() -> None: + warning = None + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD): + warning = record_response("agent-1", 404, 128) + + assert warning is not None + assert "STRATEGY WARNING" in warning + assert str(UNPRODUCTIVE_STREAK_THRESHOLD) in warning + + +def test_no_warning_before_threshold_reached() -> None: + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD - 1): + assert record_response("agent-1", 404, 128) is None + + +def test_2xx_response_resets_the_streak() -> None: + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD - 1): + assert record_response("agent-1", 404, 128) is None + + # One success in the middle of a near-threshold 404 run must clear it. + assert record_response("agent-1", 200, 512) is None + + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD - 1): + assert record_response("agent-1", 404, 128) is None + + +def test_repeated_identical_shape_counts_as_unproductive_even_without_404() -> None: + # Every guessed path comes back 200 with the exact same generic body — + # no literal 404s, but still zero new information after a few repeats. + warning = None + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD + 2): + # Route through a 3xx/4xx-adjacent non-2xx status so the "always + # informative" 2xx override doesn't mask the identical-shape check. + warning = record_response("agent-1", 500, 999) + + assert warning is not None + + +def test_varied_shapes_never_trigger_the_warning() -> None: + # Legitimate varied recon: different status/length every time. + for i in range(UNPRODUCTIVE_STREAK_THRESHOLD * 2): + warning = record_response("agent-1", 401, 100 + i) + assert warning is None + + +def test_different_agents_do_not_interfere() -> None: + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD - 1): + assert record_response("agent-a", 404, 128) is None + assert record_response("agent-b", 404, 128) is None + + # agent-a crosses the threshold; agent-b, one request behind, must not. + assert record_response("agent-a", 404, 128) is not None + assert record_response("agent-b", 200, 128) is None + + +def test_warning_does_not_refire_immediately_after_firing() -> None: + fired_at: list[int] = [] + for i in range(1, UNPRODUCTIVE_STREAK_THRESHOLD * 2 + 1): + warning = record_response("agent-1", 404, 128) + if warning is not None: + fired_at.append(i) + + assert fired_at == [UNPRODUCTIVE_STREAK_THRESHOLD, UNPRODUCTIVE_STREAK_THRESHOLD * 2] + + +def test_missing_agent_id_or_status_is_a_no_op() -> None: + assert record_response(None, 404, 128) is None + assert record_response("agent-1", None, 128) is None + + +def test_format_replay_tool_result_injects_warning_field_at_threshold() -> None: + replay = { + "status": "DONE", + "session_id": "sess-1", + "elapsed_ms": 12, + "error": None, + "response_raw": b"HTTP/1.1 404 Not Found\r\nContent-Length: 3\r\n\r\nabc", + } + + payload = None + for _ in range(UNPRODUCTIVE_STREAK_THRESHOLD): + payload = json.loads(tools._format_replay_tool_result(replay, agent_id="agent-x")) + + assert payload is not None + assert "strategy_warning" in payload + assert "STRATEGY WARNING" in payload["strategy_warning"] + + +def test_format_replay_tool_result_has_no_warning_field_by_default() -> None: + replay = { + "status": "DONE", + "session_id": "sess-1", + "elapsed_ms": 12, + "error": None, + "response_raw": b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nabc", + } + + payload = json.loads(tools._format_replay_tool_result(replay, agent_id="agent-x")) + assert "strategy_warning" not in payload From 3c2ade0a77eadaeed7cbff7e44b2c7c407a561ba Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:51:22 +0100 Subject: [PATCH 4/9] feat(recon): add shared recon ledger for discovered endpoints Recon findings currently only live in prose (notes, threat model), so later agents re-guess at routes recon already resolved, wasting requests on 404s. Add strix/tools/recon_ledger, a coverage.py-style JSON-backed ledger (record_endpoint/update_endpoint/list_endpoints) that dedupes on (method, path) and rejects conflicting duplicates instead of overwriting them. Wire hydration into runner.py alongside hydrate_coverage_from_disk, register the tools in factory.py's _BASE_TOOLS, and point recon/root-agent skills at it. Co-Authored-By: Claude Sonnet 5 --- strix/agents/factory.py | 4 + strix/core/runner.py | 2 + strix/skills/coordination/root_agent.md | 3 + .../skills/reconnaissance/asset_discovery.md | 3 +- strix/tools/recon_ledger/__init__.py | 1 + strix/tools/recon_ledger/tools.py | 587 ++++++++++++++++++ tests/test_recon_ledger_tool.py | 341 ++++++++++ 7 files changed, 940 insertions(+), 1 deletion(-) create mode 100644 strix/tools/recon_ledger/__init__.py create mode 100644 strix/tools/recon_ledger/tools.py create mode 100644 tests/test_recon_ledger_tool.py diff --git a/strix/agents/factory.py b/strix/agents/factory.py index b2fcbf08c..1fcb1f37d 100644 --- a/strix/agents/factory.py +++ b/strix/agents/factory.py @@ -47,6 +47,7 @@ from strix.tools.proxy.tools import ( view_request, view_sitemap_entry, ) +from strix.tools.recon_ledger.tools import list_endpoints, record_endpoint, update_endpoint from strix.tools.reporting.tool import ( create_dependency_report, create_vulnerability_report, @@ -575,6 +576,9 @@ _BASE_TOOLS: tuple[Tool, ...] = ( record_coverage, update_coverage, list_coverage, + record_endpoint, + update_endpoint, + list_endpoints, get_threat_model, save_threat_model, amend_threat_model, diff --git a/strix/core/runner.py b/strix/core/runner.py index b40a36b0e..93d25ffae 100644 --- a/strix/core/runner.py +++ b/strix/core/runner.py @@ -271,12 +271,14 @@ async def run_strix_scan( from strix.tools.coverage.tools import hydrate_coverage_from_disk from strix.tools.notes.tools import hydrate_notes_from_disk + from strix.tools.recon_ledger.tools import hydrate_recon_ledger_from_disk from strix.tools.threat_model.tools import hydrate_threat_models_from_disk from strix.tools.todo.tools import hydrate_todos_from_disk hydrate_todos_from_disk(state_dir) hydrate_notes_from_disk(state_dir) hydrate_coverage_from_disk(state_dir) + hydrate_recon_ledger_from_disk(state_dir) hydrate_threat_models_from_disk(state_dir) root_id: str | None = None diff --git a/strix/skills/coordination/root_agent.md b/strix/skills/coordination/root_agent.md index fa88aaa5a..67a77de90 100644 --- a/strix/skills/coordination/root_agent.md +++ b/strix/skills/coordination/root_agent.md @@ -47,6 +47,9 @@ Structure agents by function: - Asset discovery and enumeration - Technology fingerprinting - Attack surface mapping +- Record every discovered endpoint, route, and parameter with `record_endpoint` so later agents can query the surface instead of re-probing for it + +Vulnerability-assessment and exploitation agents should call `list_endpoints` before spending requests hunting for routes themselves - recon has usually already found and recorded them. **Vulnerability Assessment** - Injection testing (SQLi, XSS, command injection) diff --git a/strix/skills/reconnaissance/asset_discovery.md b/strix/skills/reconnaissance/asset_discovery.md index ce674fba1..3e97796da 100644 --- a/strix/skills/reconnaissance/asset_discovery.md +++ b/strix/skills/reconnaissance/asset_discovery.md @@ -98,7 +98,8 @@ For ASN-owned ranges, sweep IPs directly with `naabu`/`httpx` and read served ce `httpx -l hosts.txt -sc -title -server -td -tls-grab -json -o assets.jsonl` 3. **Classify** assets by function from title/tech/path signals: app, API, marketing, auth, CI/CD, observability, storage, admin, VCS, mail. Cluster by role, not by a specific product. 4. **Port sweep** interesting hosts with `naabu` for non-HTTP services (DBs, caches, brokers, mgmt ports). -5. **Prioritize** by exposure and value, then hand each finding to the right specialist skill: +5. **Record every endpoint, route, and parameter you find** with `record_endpoint` — from crawling (`katana`), directory/route brute force (`ffuf`), JS bundle analysis, or an OpenAPI/GraphQL spec — as you find it, not folded into a note afterward. This is what lets vulnerability-assessment agents call `list_endpoints` and start testing instead of re-discovering the same routes by probing for them. +6. **Prioritize** by exposure and value, then hand each finding to the right specialist skill: - Exposed dashboards / debug / observability / metadata leaks → `information_disclosure` - Login/admin panels with default or weak creds → `weak_password_detection` - Dangling DNS / unclaimed provider resources → `subdomain_takeover` diff --git a/strix/tools/recon_ledger/__init__.py b/strix/tools/recon_ledger/__init__.py new file mode 100644 index 000000000..6baf314e6 --- /dev/null +++ b/strix/tools/recon_ledger/__init__.py @@ -0,0 +1 @@ +"""Scan recon ledger — discovered endpoints, routes, and parameters.""" diff --git a/strix/tools/recon_ledger/tools.py b/strix/tools/recon_ledger/tools.py new file mode 100644 index 000000000..1ea6882bf --- /dev/null +++ b/strix/tools/recon_ledger/tools.py @@ -0,0 +1,587 @@ +"""Per-run recon ledger — mirrored to {state_dir}/recon_ledger.json. + +Recon agents discover far more routes than they can hand off cleanly: +an admin API found while crawling one host, a parameter noticed in a JS +bundle, an endpoint pulled out of an OpenAPI spec buried in a build +artifact. Today that knowledge only survives in the discovering agent's +own conversation, or gets folded into a note or the threat model as +prose. Neither is queryable: a later agent hunting for "does anything +handle DELETE on this resource" has to re-read a wall of text, or more +often just re-discovers the route itself by probing for it — burning +requests on paths recon already confirmed or ruled out. + +This ledger gives discovered endpoints the same treatment +``strix.tools.coverage`` gives review coverage: a structured, +deduplicated, shared record that any agent can query by method, path, +or auth requirement instead of re-reading prose or re-guessing at the +attack surface. Record a route here as soon as you find it; look here +before you spend requests hunting for one yourself. + +Entries are **agent-reported**: an agent's own account of what it +found and what it believes about it (e.g. whether auth is required). +Nothing here is independently verified by the ledger itself — that is +why a duplicate is rejected rather than overwritten: two agents can +form different conclusions about the same route, and silently +replacing one with the other would hide the disagreement instead of +surfacing it for reconciliation via ``update_endpoint``. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import tempfile +import threading +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from agents import RunContextWrapper, function_tool + +from strix.tools.nullish import clean_optional + + +logger = logging.getLogger(__name__) + + +_recon_ledger_storage: dict[str, dict[str, Any]] = {} +_recon_ledger_lock = threading.RLock() +_recon_ledger_path: Path | None = None +_ENTRY_ID_GENERATION_ATTEMPTS = 1024 +_NOTES_PREVIEW_CHARS = 240 + +VALID_METHODS: tuple[str, ...] = ( + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS", + "OTHER", +) + + +def _caller_identity(ctx: RunContextWrapper) -> tuple[str | None, str | None]: + """Return the (agent_id, agent_name) of the agent invoking this tool.""" + inner = ctx.context if isinstance(ctx.context, dict) else {} + raw_agent_id = inner.get("agent_id") + agent_id = raw_agent_id if isinstance(raw_agent_id, str) else None + agent_name: str | None = None + coordinator = inner.get("coordinator") + if agent_id is not None and coordinator is not None: + names = getattr(coordinator, "names", {}) + if isinstance(names, dict): + raw_agent_name = names.get(agent_id) + agent_name = raw_agent_name if isinstance(raw_agent_name, str) else None + return agent_id, agent_name + + +def _generate_entry_id() -> str | None: + """Allocate an unused entry id. Callers must already hold ``_recon_ledger_lock``.""" + for _ in range(_ENTRY_ID_GENERATION_ATTEMPTS): + entry_id = uuid.uuid4().hex[:6] + if entry_id not in _recon_ledger_storage: + return entry_id + return None + + +def hydrate_recon_ledger_from_disk(state_dir: Path) -> None: + global _recon_ledger_path # noqa: PLW0603 + _recon_ledger_path = state_dir / "recon_ledger.json" + with _recon_ledger_lock: + _recon_ledger_storage.clear() + if not _recon_ledger_path.exists(): + return + try: + data = json.loads(_recon_ledger_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + logger.exception( + "recon_ledger.json at %s is unreadable; starting with an empty ledger", + _recon_ledger_path, + ) + return + if not isinstance(data, dict): + return + _recon_ledger_storage.update( + { + eid: entry + for eid, entry in data.items() + if isinstance(eid, str) and isinstance(entry, dict) + } + ) + logger.info( + "recon ledger hydrated from %s (%d entr(ies))", + _recon_ledger_path, + len(_recon_ledger_storage), + ) + + +def _persist_locked() -> None: + """Mirror the ledger to disk. Callers must already hold ``_recon_ledger_lock``. + + Serialization and the rename happen in one critical section. Releasing + the lock in between would let a writer holding an older serialization win + the rename and silently roll back a concurrent agent's entry, so the + ledger would hydrate short on resume. + """ + path = _recon_ledger_path + if path is None: + return + try: + payload = json.dumps(_recon_ledger_storage, ensure_ascii=False, default=str) + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=str(path.parent), + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as tmp: + tmp.write(payload) + tmp_path = Path(tmp.name) + tmp_path.replace(path) + except Exception: + logger.exception("recon ledger persist to %s failed", path) + + +def get_recon_ledger_entries() -> list[dict[str, Any]]: + """Return every recon ledger entry, newest last.""" + with _recon_ledger_lock: + entries = [{**entry, "entry_id": eid} for eid, entry in _recon_ledger_storage.items()] + entries.sort(key=lambda e: str(e.get("created_at", ""))) + return entries + + +def _normalize_method(method: str) -> str: + normalized = method.strip().upper() + if normalized in VALID_METHODS: + return normalized + return "OTHER" if normalized else "" + + +def _validate(*, method: str, path: str) -> tuple[str, list[str]]: + errors: list[str] = [] + normalized_method = _normalize_method(method) + if not method.strip(): + errors.append("method cannot be empty - e.g. GET, POST, PUT, PATCH, DELETE") + if not path.strip(): + errors.append("path cannot be empty - name the endpoint or route, e.g. '/api/orders/{id}'") + return normalized_method, errors + + +def _duplicate_of_locked(method: str, path: str) -> tuple[str, dict[str, Any]] | None: + """Find an existing row for this exact method and path. + + Callers must already hold ``_recon_ledger_lock``. The uniqueness check + and the insertion that depends on it have to be one critical section: + otherwise two agents recording the same route concurrently both see "no + duplicate", and the ledger ends up with exactly the parallel rows this + rejection exists to prevent. + """ + key = (method.strip().upper(), path.strip().lower()) + for entry_id, entry in _recon_ledger_storage.items(): + existing = ( + str(entry.get("method", "")).strip().upper(), + str(entry.get("path", "")).strip().lower(), + ) + if existing == key: + return entry_id, dict(entry) + return None + + +def _record_impl( + *, + method: str, + path: str, + params: str, + auth_required: bool | None, + technology: str, + notes: str, + source: str, + agent_id: str | None, + agent_name: str | None, +) -> dict[str, Any]: + normalized_method, errors = _validate(method=method, path=path) + if errors: + return {"success": False, "error": "Validation failed", "errors": errors} + + entry: dict[str, Any] = { + "method": normalized_method, + "path": path.strip(), + "created_at": datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC"), + } + if params.strip(): + entry["params"] = params.strip() + if auth_required is not None: + entry["auth_required"] = auth_required + if technology.strip(): + entry["technology"] = technology.strip() + if notes.strip(): + entry["notes"] = notes.strip() + if source.strip(): + entry["source"] = source.strip() + if agent_id: + entry["agent_id"] = agent_id + if agent_name: + entry["agent_name"] = agent_name + + with _recon_ledger_lock: + duplicate = _duplicate_of_locked(normalized_method, path) + if duplicate is not None: + existing_id, existing = duplicate + owner = existing.get("agent_name") or "another agent" + return { + "success": False, + "error": ( + f"'{normalized_method} {path.strip()}' already has recon ledger entry " + f"{existing_id}, recorded by {owner}. Two rows for one route leave the " + "ledger showing a stale conclusion beside its replacement (for example, " + "two different answers to whether auth is required). If your discovery " + "adds or corrects information - such as the auth requirement, technology, " + "or parameters - move that entry with " + f"update_endpoint(entry_id='{existing_id}', ...) instead. If you found a " + "genuinely different route, check the method and path are exact." + ), + "existing_entry_id": existing_id, + } + + entry_id = _generate_entry_id() + if entry_id is None: + return {"success": False, "error": "Could not allocate a recon ledger entry id"} + _recon_ledger_storage[entry_id] = entry + _persist_locked() + logger.info( + "Recon ledger entry recorded: id=%s method=%s path=%s", + entry_id, + normalized_method, + entry["path"], + ) + return { + "success": True, + "entry_id": entry_id, + "message": f"Recorded {normalized_method} {entry['path']}", + } + + +def _apply_string_field(existing: dict[str, Any], field: str, value: str | None) -> bool: + """Set or clear a free-text field on ``existing``. Returns whether it changed.""" + if value is None: + return False + if value.strip(): + existing[field] = value.strip() + else: + existing.pop(field, None) + return True + + +def _update_impl( + *, + entry_id: str, + params: str | None, + auth_required: bool | None, + technology: str | None, + notes: str | None, + source: str | None, + agent_id: str | None, + agent_name: str | None, +) -> dict[str, Any]: + key = (entry_id or "").strip() + with _recon_ledger_lock: + existing = _recon_ledger_storage.get(key) + if existing is None: + return { + "success": False, + "error": ( + f"No recon ledger entry {entry_id!r}. Call list_endpoints to find the " + "entry you mean - filter by path if you only know the route." + ), + } + previous: dict[str, Any] = { + "recorded_at": existing.get("updated_at", existing.get("created_at", "")), + } + for field in ("params", "auth_required", "technology", "notes", "source"): + if field in existing: + previous[field] = existing[field] + if existing.get("agent_name"): + previous["agent_name"] = existing["agent_name"] + + changed = any( + [ + _apply_string_field(existing, "params", params), + _apply_string_field(existing, "technology", technology), + _apply_string_field(existing, "notes", notes), + _apply_string_field(existing, "source", source), + ] + ) + if auth_required is not None: + existing["auth_required"] = auth_required + changed = True + + if not changed: + return { + "success": False, + "error": ( + "No fields to update - pass at least one of params, auth_required, " + "technology, notes, or source." + ), + } + + history = existing.get("history") + existing["history"] = [*history, previous] if isinstance(history, list) else [previous] + existing["updated_at"] = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + if agent_id: + existing["agent_id"] = agent_id + if agent_name: + existing["agent_name"] = agent_name + _persist_locked() + logger.info( + "Recon ledger entry updated: id=%s method=%s path=%s", + key, + existing["method"], + existing["path"], + ) + message = f"Updated {existing['method']} {existing['path']}. Previous state kept as history." + return {"success": True, "entry_id": key, "message": message} + + +def _list_impl( + *, + method: str | None, + path_contains: str | None, + auth_required: bool | None, + caller_agent_id: str | None, +) -> dict[str, Any]: + normalized_method: str | None = None + if method and method.strip(): + normalized_method = method.strip().upper() + if normalized_method not in VALID_METHODS: + return { + "success": False, + "error": f"Invalid method: {method!r}. Must be one of: {list(VALID_METHODS)}", + } + + path_contains = clean_optional(path_contains) + + entries: list[dict[str, Any]] = [] + for entry in get_recon_ledger_entries(): + if normalized_method and entry.get("method") != normalized_method: + continue + if path_contains and path_contains.lower() not in str(entry.get("path", "")).lower(): + continue + if auth_required is not None and entry.get("auth_required") != auth_required: + continue + listing = { + "entry_id": entry.get("entry_id"), + "method": entry.get("method", ""), + "path": entry.get("path", ""), + "created_at": entry.get("created_at", ""), + } + for field in ("params", "auth_required", "technology", "source"): + if field in entry: + listing[field] = entry[field] + notes = str(entry.get("notes", "")) + if notes: + listing["notes"] = ( + f"{notes[:_NOTES_PREVIEW_CHARS].rstrip()}..." + if len(notes) > _NOTES_PREVIEW_CHARS + else notes + ) + agent_name = entry.get("agent_name") + if agent_name: + listing["agent_name"] = agent_name + history = entry.get("history") + if isinstance(history, list) and history: + listing["revision_count"] = len(history) + if caller_agent_id is not None and entry.get("agent_id") == caller_agent_id: + listing["by_you"] = True + entries.append(listing) + + return { + "success": True, + "entries": entries, + "filtered_count": len(entries), + "total_count": len(_recon_ledger_storage), + } + + +@function_tool(timeout=30) +async def record_endpoint( + ctx: RunContextWrapper, + method: str, + path: str, + params: str = "", + auth_required: bool | None = None, + technology: str = "", + notes: str = "", + source: str = "", +) -> str: + """Record a discovered endpoint, route, or parameter in the shared recon ledger. + + Recon turns up far more routes than fit cleanly into a note or the + threat model, and folding them into prose makes them useless to + every other agent - a later agent hunting for "does anything handle + DELETE on this resource" has to re-read a wall of text, or more + often just re-probes for the route itself. That is expensive: it + burns requests re-discovering (or 404-ing on) attack surface recon + already resolved. Use this tool instead of ``create_note`` whenever + you find something a later agent should be able to query + structurally: an endpoint, a route, or a parameter. + + Record entries as you go, not in a batch at the end. Entries are + shared across every agent in the scan; vulnerability-testing agents + should call ``list_endpoints`` before spending requests guessing at + routes themselves. + + The ledger is not append-only: if this exact method and path + already have an entry - yours or another agent's - this call is + rejected and returns that entry's id, because two rows for one + route leave the ledger showing a stale conclusion beside its + replacement (most importantly, two different answers to whether + auth is required). If you learned something new about an existing + route, call ``update_endpoint`` on the id it hands you instead. + + Args: + method: The HTTP method, e.g. ``GET``, ``POST``, ``PUT``, + ``PATCH``, ``DELETE``, ``HEAD``, ``OPTIONS``. Anything else + (e.g. a non-HTTP RPC/WebSocket route) is recorded as + ``OTHER`` - say the real protocol in ``notes``. + path: The route or endpoint path, e.g. + ``"/api/orders/{id}"``. Use the templated form with named + path parameters rather than one entry per concrete id. + params: Known query/body/path parameters, free-form, e.g. + ``"id (path), page, filter[status]"``. + auth_required: Whether this route requires authentication, if + known. Leave ``None`` when you have not established this - + do not guess ``False`` just because you reached it + unauthenticated once; a route can look reachable and still + enforce checks deeper in the handler. + technology: Framework, language, or server this route runs on, + if known (e.g. ``"Express"``, ``"Django REST"``). + notes: Anything else worth recording - how you found it, + response shape, unusual behavior. + source: Where this was discovered, e.g. ``"JS bundle"``, + ``"OpenAPI spec"``, ``"crawl"``, ``"source code"``. + """ + agent_id, agent_name = _caller_identity(ctx) + result = await asyncio.to_thread( + _record_impl, + method=method, + path=path, + params=params, + auth_required=auth_required, + technology=technology, + notes=notes, + source=source, + agent_id=agent_id, + agent_name=agent_name, + ) + return json.dumps(result, ensure_ascii=False, default=str) + + +@function_tool(timeout=30) +async def update_endpoint( + ctx: RunContextWrapper, + entry_id: str, + params: str | None = None, + auth_required: bool | None = None, + technology: str | None = None, + notes: str | None = None, + source: str | None = None, +) -> str: + """Change what is known about an already-recorded endpoint. + + The recon ledger is shared across the whole agent tree, and what is + known about a route is not final when it is first written. Use + this whenever later work adds to or corrects an existing entry: + + - You confirmed (or ruled out) that a route recorded with + ``auth_required=None`` actually requires authentication. + - You found additional parameters, or the real technology behind + the route. + - An earlier agent's auth conclusion turns out to be wrong - + overwrite it here rather than leaving both a wrong and a right + answer sitting in the ledger. + + The method and path stay fixed - this is the same route, with + updated information. Do not record a fresh entry for a route that + already has one; that leaves a stale entry next to its correction. + Find the id with ``list_endpoints`` (filter by ``path_contains``), + then update it. + + Pass ``None`` (the default) for any field you are not changing. + The previous values of any field you do change are kept as + history, so the ledger still shows what used to be believed about + the route and who recorded it. + + Args: + entry_id: The id of the entry to update, from + ``list_endpoints`` or ``record_endpoint``. + params: New parameter info, or ``None`` to leave unchanged. + Pass ``""`` to clear it. + auth_required: New auth requirement, or ``None`` to leave + unchanged - note that ``None`` here means "don't touch + this field", not "auth not required". + technology: New technology, or ``None`` to leave unchanged. + Pass ``""`` to clear it. + notes: New notes, or ``None`` to leave unchanged. Pass ``""`` + to clear it. + source: New source, or ``None`` to leave unchanged. Pass + ``""`` to clear it. + """ + agent_id, agent_name = _caller_identity(ctx) + result = await asyncio.to_thread( + _update_impl, + entry_id=entry_id, + params=params, + auth_required=auth_required, + technology=technology, + notes=notes, + source=source, + agent_id=agent_id, + agent_name=agent_name, + ) + return json.dumps(result, ensure_ascii=False, default=str) + + +@function_tool(timeout=30) +async def list_endpoints( + ctx: RunContextWrapper, + method: str | None = None, + path_contains: str | None = None, + auth_required: bool | None = None, +) -> str: + """List discovered endpoints recorded so far in this scan. + + **Call this before probing for routes yourself.** Recon agents + record what they find here; re-discovering it by brute-forcing or + guessing paths wastes requests on 404s for routes that are already + known. Vulnerability-testing agents should check here first, then + fall back to their own discovery only for gaps the ledger does not + cover. + + Returns each entry with its ``method``, ``path``, known ``params``, + ``auth_required``, ``technology``, ``source``, and the agent that + recorded it. + + Args: + method: Optional filter - one of ``GET`` / ``POST`` / ``PUT`` / + ``PATCH`` / ``DELETE`` / ``HEAD`` / ``OPTIONS`` / ``OTHER``. + path_contains: Optional case-insensitive substring filter on + the path, e.g. ``"orders"``. + auth_required: Optional filter - ``True`` for routes known to + require auth, ``False`` for routes confirmed not to. + Leave unset to include routes where this is unknown. + """ + caller_agent_id, _ = _caller_identity(ctx) + result = await asyncio.to_thread( + _list_impl, + method=method, + path_contains=path_contains, + auth_required=auth_required, + caller_agent_id=caller_agent_id, + ) + return json.dumps(result, ensure_ascii=False, default=str) diff --git a/tests/test_recon_ledger_tool.py b/tests/test_recon_ledger_tool.py new file mode 100644 index 000000000..cb90eedfc --- /dev/null +++ b/tests/test_recon_ledger_tool.py @@ -0,0 +1,341 @@ +"""Tests for the scan recon ledger.""" + +from __future__ import annotations + +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING, Any + +import pytest + +from strix.tools.recon_ledger.tools import ( + _list_impl, + _record_impl, + _update_impl, + get_recon_ledger_entries, + hydrate_recon_ledger_from_disk, +) + + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture(autouse=True) +def recon_ledger_store(tmp_path: Path) -> Path: + hydrate_recon_ledger_from_disk(tmp_path) + return tmp_path + + +def _record(**overrides: Any) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "method": "GET", + "path": "/api/orders/{id}", + "params": "id (path)", + "auth_required": None, + "technology": "Express", + "notes": "Found while crawling the SPA bundle.", + "source": "JS bundle", + "agent_id": "agent-1", + "agent_name": "recon-bot", + } + kwargs.update(overrides) + return _record_impl(**kwargs) + + +def test_record_persists_entry(recon_ledger_store: Path) -> None: + result = _record() + assert result["success"] is True + + entries = get_recon_ledger_entries() + assert len(entries) == 1 + assert entries[0]["method"] == "GET" + assert entries[0]["path"] == "/api/orders/{id}" + assert entries[0]["agent_name"] == "recon-bot" + assert (recon_ledger_store / "recon_ledger.json").exists() + + +def test_record_normalizes_method() -> None: + assert _record(method="get")["success"] is True + assert get_recon_ledger_entries()[0]["method"] == "GET" + + +def test_record_maps_unknown_method_to_other() -> None: + result = _record(method="TRACE") + assert result["success"] is True + assert get_recon_ledger_entries()[0]["method"] == "OTHER" + + +def test_record_requires_method_and_path() -> None: + result = _record(method=" ", path="") + assert result["success"] is False + joined = " ".join(result["errors"]) + assert "method" in joined + assert "path" in joined + + +def test_record_omits_empty_optional_fields() -> None: + result = _record(params="", technology="", notes="", source="", auth_required=None) + assert result["success"] is True + entry = get_recon_ledger_entries()[0] + assert "params" not in entry + assert "technology" not in entry + assert "notes" not in entry + assert "source" not in entry + assert "auth_required" not in entry + + +def test_record_keeps_explicit_auth_required_false() -> None: + result = _record(auth_required=False) + assert result["success"] is True + assert get_recon_ledger_entries()[0]["auth_required"] is False + + +def test_hydrate_reloads_from_disk(recon_ledger_store: Path) -> None: + _record() + hydrate_recon_ledger_from_disk(recon_ledger_store) + entries = get_recon_ledger_entries() + assert len(entries) == 1 + assert entries[0]["path"] == "/api/orders/{id}" + + +def _update(entry_id: str, **overrides: Any) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "entry_id": entry_id, + "params": None, + "auth_required": True, + "technology": None, + "notes": None, + "source": None, + "agent_id": "agent-2", + "agent_name": "authz-tester", + } + kwargs.update(overrides) + return _update_impl(**kwargs) + + +def test_update_changes_field_and_keeps_history() -> None: + recorded = _record(auth_required=None) + entry_id = str(recorded["entry_id"]) + + result = _update(entry_id) + + assert result["success"] is True + entries = get_recon_ledger_entries() + assert len(entries) == 1, "update must not create a parallel entry" + entry = entries[0] + assert entry["auth_required"] is True + assert entry["agent_name"] == "authz-tester" + assert entry["history"] == [ + { + "recorded_at": entry["created_at"], + "params": "id (path)", + "technology": "Express", + "notes": "Found while crawling the SPA bundle.", + "source": "JS bundle", + "agent_name": "recon-bot", + } + ] + + +def test_update_can_clear_a_field_with_empty_string() -> None: + entry_id = str(_record()["entry_id"]) + + _update(entry_id, auth_required=None, notes="") + + entry = get_recon_ledger_entries()[0] + assert "notes" not in entry + + +def test_update_requires_at_least_one_field() -> None: + entry_id = str(_record()["entry_id"]) + + result = _update_impl( + entry_id=entry_id, + params=None, + auth_required=None, + technology=None, + notes=None, + source=None, + agent_id=None, + agent_name=None, + ) + + assert result["success"] is False + + +def test_update_rejects_unknown_entry() -> None: + result = _update("nope") + assert result["success"] is False + assert "list_endpoints" in str(result["error"]) + + +def test_update_persists_to_disk(recon_ledger_store: Path) -> None: + entry_id = str(_record()["entry_id"]) + _update(entry_id) + + hydrate_recon_ledger_from_disk(recon_ledger_store) + + entry = get_recon_ledger_entries()[0] + assert entry["auth_required"] is True + assert len(entry["history"]) == 1 + + +def test_recording_a_duplicate_route_is_refused_with_the_existing_id() -> None: + first = _record_impl( + method="POST", + path="/api/invoices", + params="", + auth_required=None, + technology="", + notes="", + source="crawl", + agent_id="a1", + agent_name="Recon", + ) + + duplicate = _record_impl( + method="post", + path=" /API/Invoices ", + params="", + auth_required=True, + technology="", + notes="", + source="manual probing", + agent_id="a2", + agent_name="Authz", + ) + + assert duplicate["success"] is False + assert duplicate["existing_entry_id"] == first["entry_id"] + assert "update_endpoint" in duplicate["error"] + assert len(get_recon_ledger_entries()) == 1 + + +def test_a_different_path_is_still_its_own_entry() -> None: + _record_impl( + method="GET", + path="/api/invoices", + params="", + auth_required=None, + technology="", + notes="", + source="", + agent_id="a1", + agent_name="Recon", + ) + second = _record_impl( + method="GET", + path="/api/invoices/{id}", + params="", + auth_required=None, + technology="", + notes="", + source="", + agent_id="a1", + agent_name="Recon", + ) + + assert second["success"] is True + assert len(get_recon_ledger_entries()) == 2 + + +def test_a_different_method_on_one_path_is_still_its_own_entry() -> None: + _record_impl( + method="GET", + path="/api/invoices", + params="", + auth_required=None, + technology="", + notes="", + source="", + agent_id="a1", + agent_name="Recon", + ) + second = _record_impl( + method="DELETE", + path="/api/invoices", + params="", + auth_required=None, + technology="", + notes="", + source="", + agent_id="a1", + agent_name="Recon", + ) + + assert second["success"] is True + assert len(get_recon_ledger_entries()) == 2 + + +def test_list_filters_by_method_path_and_auth() -> None: + _record(method="GET", path="/login", auth_required=False) + _record(method="POST", path="/login", auth_required=False) + _record(method="GET", path="/admin/users", auth_required=True) + + by_method = _list_impl( + method="POST", path_contains=None, auth_required=None, caller_agent_id=None + ) + assert by_method["filtered_count"] == 1 + assert by_method["entries"][0]["path"] == "/login" + + by_path = _list_impl( + method=None, path_contains="admin", auth_required=None, caller_agent_id=None + ) + assert by_path["filtered_count"] == 1 + + by_auth = _list_impl( + method=None, path_contains=None, auth_required=True, caller_agent_id="agent-1" + ) + assert by_auth["filtered_count"] == 1 + assert by_auth["entries"][0]["by_you"] is True + + +def test_list_rejects_unknown_method_filter() -> None: + result = _list_impl( + method="bogus", path_contains=None, auth_required=None, caller_agent_id=None + ) + assert result["success"] is False + + +def test_concurrent_records_of_one_route_yield_a_single_row() -> None: + """Duplicate detection and insertion must be one critical section. + + Two agents recording the same route at the same moment would otherwise + both pass the "no duplicate" check, and the ledger would show a stale + conclusion beside its replacement - the exact outcome the rejection + exists to prevent. + """ + barrier = threading.Barrier(8) + + def attempt(index: int) -> dict[str, Any]: + barrier.wait() + return _record(agent_id=f"agent-{index}", agent_name=f"tester-{index}") + + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(attempt, range(8))) + + assert sum(1 for result in results if result["success"]) == 1 + assert len(get_recon_ledger_entries()) == 1 + + +def test_concurrent_records_all_survive_persistence(recon_ledger_store: Path) -> None: + """A writer holding an older snapshot must not win the rename. + + If it did, the mirror would come back short on resume and endpoints + recorded before a crash would silently disappear from the ledger. + """ + barrier = threading.Barrier(8) + + def attempt(index: int) -> dict[str, Any]: + barrier.wait() + return _record(path=f"/api/resource/{index}", agent_id=f"agent-{index}") + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(attempt, range(8))) + + persisted = json.loads((recon_ledger_store / "recon_ledger.json").read_text(encoding="utf-8")) + assert len(persisted) == 8 + hydrate_recon_ledger_from_disk(recon_ledger_store) + assert len(get_recon_ledger_entries()) == 8 From 1804ecb2b526e663a55d10f90b74f40890e12a48 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:52:28 +0100 Subject: [PATCH 5/9] Add explicit compatibility boundary for the openai-agents SDK Strix's sandbox runtime (docker_client.py) and agent tool wiring (factory.py) reach into internal/undocumented openai-agents SDK shapes (a verbatim-copied private method, private helper functions, and a private session attribute) with no compile-time signal if a version bump changes them. Add strix/runtime/sdk_compat.py to declare the supported SDK range, assert it at startup (wired into strix.interface.main.main()), and document the concrete coupling checklist for future SDK bumps. Add docs/adr/0001 explaining the rationale and future direction, plus tests covering the version guard. Co-Authored-By: Claude Sonnet 5 --- docs/adr/0001-openai-agents-sdk-coupling.md | 129 ++++++++++ strix/interface/main.py | 7 + strix/runtime/docker_client.py | 9 +- strix/runtime/sdk_compat.py | 247 ++++++++++++++++++++ tests/test_sdk_compat.py | 59 +++++ 5 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 docs/adr/0001-openai-agents-sdk-coupling.md create mode 100644 strix/runtime/sdk_compat.py create mode 100644 tests/test_sdk_compat.py diff --git a/docs/adr/0001-openai-agents-sdk-coupling.md b/docs/adr/0001-openai-agents-sdk-coupling.md new file mode 100644 index 000000000..17491e685 --- /dev/null +++ b/docs/adr/0001-openai-agents-sdk-coupling.md @@ -0,0 +1,129 @@ +# 1. Coupling to the openai-agents SDK's internal sandbox implementation + +## Status + +Accepted + +## Context + +Strix's agent runtime is built directly on OpenAI's `agents` SDK (PyPI: +`openai-agents`, imported as `agents`), pinned in `pyproject.toml` as +`openai-agents[litellm]>=0.19.0,<0.20`. The SDK provides the agent run loop, +the sandbox abstraction (Docker-backed containers with a filesystem/manifest +model), and built-in sandbox "capability" tool sets (`Shell`, `Filesystem`) +that implement shell execution, patch application (`apply_patch`), and file +operations. + +Building on the SDK rather than writing Strix's own agent loop and sandbox +tooling was the right call: it gets Strix a maintained, security-conscious +sandbox implementation, a documented Responses/Chat-Completions-compatible +tool-calling loop, and shell/filesystem tools for free, instead of Strix +having to build and maintain all of that itself. Rewriting Strix off the SDK +is out of scope for this decision — that would be a multi-month effort with +its own substantial risk, for a problem that a narrower fix (below) already +addresses. + +The problem: some of what Strix needs from the SDK is not exposed through a +documented, versioned public API. Two files in particular reach into +internal or undocumented SDK shapes: + +- `strix/runtime/docker_client.py` (`StrixDockerSandboxClient`) subclasses + the SDK's `DockerSandboxClient` and **reimplements `_create_container` + verbatim** from the SDK's source, because the SDK gives no extension hook + for the container-creation kwargs. It also imports four underscore-prefixed + (private) helper functions from `agents.sandbox.sandboxes.docker`, and + reaches through `SandboxSession._inner` (a private attribute) to swap in a + custom session subclass and to read the container id for teardown. +- `strix/agents/factory.py` wires up the SDK's `Shell` and `Filesystem` + sandbox capabilities — which supply the `exec_command`, `write_stdin`, + `apply_patch`, and filesystem tools an agent actually calls — and then + pattern-matches on those tools' names and monkeypatches their + `on_invoke_tool` callables to add result-bounding, argument coercion, and + error-as-result behavior. This assumes tool names and attribute shapes that + are not part of a documented contract. + +The concrete, current list of every such dependency is maintained in +[`strix/runtime/sdk_compat.py`](../../strix/runtime/sdk_compat.py)'s module +docstring — that file is the canonical checklist, not this ADR, so the list +stays in one place and next to the code it describes. + +**The risk**: a maintainer bumping the `openai-agents` version (even a patch +bump, since none of this is covered by semver on the private surface) can +silently change or break sandbox networking, container capabilities, or tool +behavior with no compile-time signal. The failure would show up as a scan +misbehaving mid-run — wrong network capabilities, a hung entrypoint, a tool +that stops accepting the arguments Strix sends it — rather than as an +upgrade-time error pointing at the cause. + +## Decision + +Add an explicit, checked compatibility boundary instead of leaving the +coupling implicit: + +1. `strix/runtime/sdk_compat.py` declares `SUPPORTED_AGENTS_SDK_RANGE`, + matching the `pyproject.toml` constraint, and exposes + `assert_compatible_sdk_version()`, which reads the installed `agents` + package version and raises a clear `RuntimeError` — naming the installed + version, the supported range, and this file — if it's out of range. +2. `assert_compatible_sdk_version()` is called once, early, in + `strix.interface.main.main()`, before any sandbox or agent code runs, so + an incompatible SDK version fails at startup with an actionable message + instead of failing unpredictably mid-scan. +3. `sdk_compat.py`'s module docstring documents, concretely, every place + `docker_client.py` and `factory.py` depend on SDK internals beyond the + public API, plus a numbered checklist of what to re-verify when bumping + the SDK version (re-diff the verbatim `_create_container` copy, re-check + the four private helper functions still exist with the same signatures, + re-check `session._inner` still resolves the same way, re-check the + `Shell`/`Filesystem` toolsets still expose the same tool names). + +This is deliberately **documentation plus a version guard, not a refactor**. +The patching code in `docker_client.py` and `factory.py` is working, +carefully-commented code solving real problems (entrypoint preservation, +NET_ADMIN/NET_RAW for raw-socket tools, output bounding, argument coercion). +Moving it under time pressure risks breaking it, which is worse than leaving +it in place with a guard rail and a checklist. + +## Consequences + +- An SDK version outside the supported range now fails immediately at + startup with a message naming the installed version, the supported range, + and where to look — instead of surfacing as a confusing runtime failure + partway through a scan. +- Bumping the SDK version now has one required first step (update + `SUPPORTED_AGENTS_SDK_RANGE` in lockstep with `pyproject.toml`) and one + concrete checklist to work through (in `sdk_compat.py`'s docstring) before + merging, rather than relying on someone remembering the tribal knowledge in + `docker_client.py`'s header comment. +- The guard only catches *version* mismatches, not *behavioral* drift within + a version bump that stays inside the declared range (e.g. if a future + 0.19.x patch release changes `_create_container`'s body without a version + bump outside the pin). The checklist and the existing tests + (`tests/test_docker_client_delete.py`, `tests/test_agent_factory_shell.py`, + `tests/test_sdk_compat.py`) are the mitigation for that gap, not the + version guard itself. +- This does not reduce the actual coupling — Strix's sandbox networking and + tool behavior still depend on SDK internals. It converts a silent failure + mode into a loud one and gives it a fixed address in the codebase. + +## Future direction + +Not committed now, but worth considering if the SDK's private surface keeps +moving underneath Strix: + +- **Own the `Shell`/`Filesystem` tool implementations directly.** If Strix + implements `exec_command`, `write_stdin`, `apply_patch`, and file + operations itself (as it already does for every other tool in + `strix/tools/`), the coupling in `factory.py` disappears entirely and the + SDK is used only for its documented agent run loop and tool-calling + contract. This is a real chunk of work — reimplementing a sandboxed shell + and patch-apply tool safely — so it should only be taken on if the + maintenance cost of tracking the SDK's private surface exceeds the cost of + owning these tools. +- **Ask upstream for an extension hook.** The specific gap that forces the + verbatim `_create_container` copy is the lack of a way to extend + `create_kwargs` before `containers.create()` runs. If the SDK adds such a + hook, `docker_client.py` could subclass and override just the delta + instead of duplicating the whole method body — the comment in that file + already flags this as the ideal fix ("Track upstream for an injection + hook"). diff --git a/strix/interface/main.py b/strix/interface/main.py index c9bd55961..c6afa56e4 100644 --- a/strix/interface/main.py +++ b/strix/interface/main.py @@ -42,6 +42,7 @@ from strix.interface.utils import ( build_final_stats_text, ) from strix.llm.warmup import start_import_warmup, wait_for_import_warmup +from strix.runtime.sdk_compat import assert_compatible_sdk_version from strix.telemetry import posthog, report_error, scarf, set_scan_phase from strix.telemetry.logging import configure_dependency_logging @@ -421,6 +422,12 @@ def _bootstrap_scan(args: argparse.Namespace) -> None: def main() -> None: configure_dependency_logging() + # Fail loudly and immediately if the installed openai-agents SDK is + # outside the range Strix's sandbox runtime and agent tooling are built + # against (see strix/runtime/sdk_compat.py), rather than breaking + # unpredictably mid-scan. + assert_compatible_sdk_version() + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) diff --git a/strix/runtime/docker_client.py b/strix/runtime/docker_client.py index 969267d53..1da2d8e34 100644 --- a/strix/runtime/docker_client.py +++ b/strix/runtime/docker_client.py @@ -16,8 +16,11 @@ deltas: 3. Add ``host.docker.internal`` → host-gateway to ``extra_hosts`` so the agent can reach host-served apps. -Pinned to ``openai-agents==0.14.6``. Bumping the SDK requires -re-merging the parent body. Track upstream for an injection hook. +Verbatim copy last re-merged against ``openai-agents==0.19.0``. Bumping the +SDK requires re-merging the parent body — see the compatibility checklist +and version guard in ``strix/runtime/sdk_compat.py`` (also documented in +``docs/adr/0001-openai-agents-sdk-coupling.md``) before upgrading. Track +upstream for an injection hook. """ from __future__ import annotations @@ -175,7 +178,7 @@ class StrixDockerSandboxClient(DockerSandboxClient): session_id: uuid.UUID | None = None, ) -> Container: # ----- BEGIN VERBATIM COPY of DockerSandboxClient._create_container ----- - # SDK ref: src/agents/sandbox/sandboxes/docker.py:1434-1477 (v0.14.6). + # SDK ref: src/agents/sandbox/sandboxes/docker.py:1562-1605 (v0.19.0). if not self.image_exists(image): repo, tag = parse_repository_tag(image) self.docker_client.images.pull(repo, tag=tag or None, all_tags=False) diff --git a/strix/runtime/sdk_compat.py b/strix/runtime/sdk_compat.py new file mode 100644 index 000000000..a600fc1b7 --- /dev/null +++ b/strix/runtime/sdk_compat.py @@ -0,0 +1,247 @@ +"""Compatibility boundary between Strix and the ``openai-agents`` SDK. + +Strix's sandbox runtime and agent tooling are built directly on the ``agents`` +package (PyPI: ``openai-agents``), pinned in ``pyproject.toml`` as +``openai-agents[litellm]>=0.19.0,<0.20``. Most of that surface is the SDK's +documented public API (``agents.sandbox.entries``, ``agents.sandbox.manifest``, +``agents.sandbox.capabilities.Filesystem``/``Shell``, ``agents.tool.*``, etc.), +but a few pieces of Strix code reach past the public API into internal or +undocumented shapes of the SDK's sandbox implementation. Those places do not +fail loudly if the SDK changes them — they silently misbehave (wrong +container capabilities, broken entrypoint handling, or a crash deep in a +scan) instead. This module exists to make an incompatible SDK version fail +immediately and clearly at startup, and to give a future maintainer a single +place to check before bumping the SDK version. + +Run ``assert_compatible_sdk_version()`` early — it is wired into +``strix.interface.main.main()`` — so an unsupported ``agents`` version is +reported before a scan starts, not mid-scan. + +Concrete internals depended on beyond the SDK's public API +============================================================ + +``strix/runtime/docker_client.py`` (``StrixDockerSandboxClient``): + +- ``agents.sandbox.sandboxes.docker.DockerSandboxClient._create_container`` + — Strix's ``_create_container`` is a **verbatim copy of the SDK method + body** (see the "BEGIN/END VERBATIM COPY" markers in that file), not a + call to the parent implementation, because the SDK gives no hook to extend + ``create_kwargs`` before ``containers.create(**create_kwargs)`` runs. + Strix's copy assumes the SDK still funnels container creation through this + single private method with this exact body shape (image pull, manifest + volume mounts, FUSE/SYS_ADMIN handling, exposed-port mapping). If the SDK + restructures ``_create_container`` — renames it, splits it, changes the + order/meaning of its steps, or changes what ``create_kwargs`` accepts — + Strix's copy silently drifts out of sync with upstream behavior instead of + raising an error. +- Private helper functions imported directly from + ``agents.sandbox.sandboxes.docker``: ``_build_docker_volume_mounts``, + ``_docker_port_key``, ``_manifest_requires_fuse``, + ``_manifest_requires_sys_admin``. All four are underscore-prefixed + (unstable, non-public) module functions that Strix's verbatim copy of + ``_create_container`` calls to reproduce the SDK's own logic. +- ``session._inner`` and ``session._inner.state.container_id`` — the public + ``SandboxSession`` wraps a private ``_inner`` implementation object; Strix + reaches through it (in ``create()`` to swap the inner session's class for + ``StrixDockerSandboxSession``, and in ``delete()`` to read the container id + for a best-effort ``docker kill``). +- ``DockerSandboxSession`` is subclassed (``StrixDockerSandboxSession``) to + override ``_resolve_exposed_port``, a private method, so that host-gateway + network resolution can look up the container's IP on a custom Docker + network (``STRIX_DOCKER_SANDBOX_NETWORK``). +- The module docstring in ``docker_client.py`` itself already flags this and + historically named a specific pinned SDK patch version to re-merge against + on every SDK bump — keep that pin annotation current whenever + ``SUPPORTED_AGENTS_SDK_RANGE`` below changes. + +``strix/agents/factory.py``: + +- ``agents.sandbox.SandboxAgent``, ``agents.sandbox.capabilities.Filesystem``, + ``agents.sandbox.capabilities.Shell`` — these are public, but the shell + execution (``exec_command``, ``write_stdin``), patch application + (``apply_patch``), and filesystem tools Strix's agents use are **not + Strix's own tool implementations**. They are the SDK's ``Shell`` and + ``Filesystem`` sandbox capability classes' built-in tool sets, wired up + via each capability's ``configure_tools`` hook + (``_configure_shell_tools`` / ``_configure_filesystem_tools``). Strix + inspects each toolset with ``vars(toolset)`` (an implementation detail of + how the SDK stores its generated tools as attributes) and pattern-matches + on tool names (``"exec_command"``, ``"write_stdin"``) and error types + (``InvalidManifestPathError``) that are not part of a documented, versioned + tool-name contract. +- ``agents.tool.CustomTool`` / ``agents.tool.FunctionTool`` — Strix reaches + into instance attributes (``tool.on_invoke_tool``, ``tool.strict_json_schema``, + ``tool.params_json_schema``) and monkeypatches ``on_invoke_tool`` in place + to add result-bounding, argument coercion, and error-as-result behavior. + These are public dataclass-like fields today, but there is no documented + guarantee the SDK keeps this exact shape across versions. +- ``agents.agent.ToolsToFinalOutputResult`` is used to implement a custom + ``tool_use_behavior``; this is public API but its exact semantics + (when the SDK calls it, what fields matter) are tied to the SDK's run loop + internals. + +``strix/runtime/session_manager.py`` and ``strix/runtime/backends.py`` mostly +use the SDK's documented public API (``agents.sandbox.entries.{BaseEntry, +File, LocalDir}``, ``agents.sandbox.manifest.{Environment, Manifest}``, +``DockerSandboxClientOptions``, ``session.resolve_exposed_port()``, +``session.start()``). The one soft dependency is ``getattr(client, +"docker_client", None)`` in ``session_manager.cleanup()`` — a defensive, +non-raising reach for a low-level client attribute so the underlying +``docker`` SDK client gets closed; it degrades gracefully (no-op) if the +attribute disappears. + +What to do when bumping the SDK version +======================================== + +1. Update ``SUPPORTED_AGENTS_SDK_RANGE`` below and the matching constraint in + ``pyproject.toml`` together. +2. Re-read ``agents/sandbox/sandboxes/docker.py`` in the new SDK version and + diff it against the verbatim copy in ``docker_client.py``'s + ``_create_container`` — re-merge any upstream changes into the Strix + deltas. +3. Re-check that ``_build_docker_volume_mounts``, ``_docker_port_key``, + ``_manifest_requires_fuse``, and ``_manifest_requires_sys_admin`` still + exist with the same signatures. +4. Re-check that ``session._inner`` and ``session._inner.state.container_id`` + still resolve the way ``docker_client.py`` expects. +5. Re-check that the ``Shell``/``Filesystem`` capabilities' generated + toolsets still expose tools named ``exec_command`` / ``write_stdin`` / + ``apply_patch`` with the same argument shapes that + ``strix/agents/factory.py`` special-cases. +6. Run the full test suite, especially ``tests/test_docker_client_delete.py``, + ``tests/test_agent_factory_shell.py``, and this module's tests. + +See also ``docs/adr/0001-openai-agents-sdk-coupling.md`` for the +architectural rationale and longer-term direction. +""" + +from __future__ import annotations + +from importlib import metadata as importlib_metadata + +import agents + + +_DISTRIBUTION_NAME = "openai-agents" + +#: Matches the ``openai-agents[litellm]>=0.19.0,<0.20`` constraint in +#: ``pyproject.toml``. Keep these in sync — this tuple is the single source of +#: truth this module checks against, and the docstring above lists exactly +#: what needs re-verifying whenever this range moves. +SUPPORTED_AGENTS_SDK_RANGE = (">=0.19.0", "<0.20") + + +def _parse_version(version: str) -> tuple[int, ...]: + """Parse a dotted numeric version prefix into a comparable tuple. + + Only the release segment (leading dotted integers) is used, e.g. + ``"0.19.0"`` -> ``(0, 19, 0)`` and ``"0.19.0.dev1"`` -> ``(0, 19, 0)``. + This is intentionally simple (no pre-release ordering) since it only + needs to place a version inside or outside a narrow numeric range. + """ + parts: list[int] = [] + for chunk in version.split("."): + digits = "" + for char in chunk: + if char.isdigit(): + digits += char + else: + break + if not digits: + break + parts.append(int(digits)) + if not parts: + msg = f"Could not parse a numeric version from {version!r}" + raise ValueError(msg) + return tuple(parts) + + +def _pad(version: tuple[int, ...], length: int) -> tuple[int, ...]: + return version + (0,) * (length - len(version)) + + +def _compare(installed: tuple[int, ...], bound: tuple[int, ...]) -> int: + length = max(len(installed), len(bound)) + left = _pad(installed, length) + right = _pad(bound, length) + if left < right: + return -1 + if left > right: + return 1 + return 0 + + +def _satisfies(installed: str, specifiers: tuple[str, ...]) -> bool: + installed_version = _parse_version(installed) + for spec in specifiers: + for op in (">=", "<=", "==", "!=", ">", "<"): + if spec.startswith(op): + bound = _parse_version(spec[len(op) :]) + cmp = _compare(installed_version, bound) + ok = { + ">=": cmp >= 0, + "<=": cmp <= 0, + "==": cmp == 0, + "!=": cmp != 0, + ">": cmp > 0, + "<": cmp < 0, + }[op] + if not ok: + return False + break + else: # pragma: no cover - defensive; every spec above starts with an op + msg = f"Unrecognized version specifier: {spec!r}" + raise ValueError(msg) + return True + + +def installed_agents_sdk_version() -> str: + """Return the installed ``openai-agents`` distribution version. + + Prefers ``agents.__version__`` (what the package itself exposes) and + falls back to ``importlib.metadata`` (the distribution name differs from + the import name: ``openai-agents`` ships the ``agents`` module). + """ + version = getattr(agents, "__version__", None) + if isinstance(version, str) and version: + return version + return importlib_metadata.version(_DISTRIBUTION_NAME) + + +def assert_compatible_sdk_version() -> None: + """Raise ``RuntimeError`` if the installed ``agents`` SDK is unsupported. + + Strix's sandbox runtime (``strix/runtime/docker_client.py``) and agent + tool wiring (``strix/agents/factory.py``) depend on internal shapes of + the ``openai-agents`` SDK that are not guaranteed stable across versions + (see this module's docstring for the concrete list). Call this once, + early at startup — before any sandbox or agent code runs — so an + incompatible SDK version is reported immediately with a clear, + actionable message instead of failing unpredictably mid-scan. + """ + installed = installed_agents_sdk_version() + try: + compatible = _satisfies(installed, SUPPORTED_AGENTS_SDK_RANGE) + except ValueError as exc: + msg = ( + f"Could not verify openai-agents SDK compatibility: installed version " + f"{installed!r} could not be parsed ({exc}). Strix requires " + f"openai-agents {', '.join(SUPPORTED_AGENTS_SDK_RANGE)}. " + "See strix/runtime/sdk_compat.py for details." + ) + raise RuntimeError(msg) from exc + + if not compatible: + msg = ( + f"Incompatible openai-agents SDK version: installed {installed!r}, " + f"but Strix requires {', '.join(SUPPORTED_AGENTS_SDK_RANGE)}. " + "Strix's sandbox runtime (strix/runtime/docker_client.py) and agent " + "tool wiring (strix/agents/factory.py) depend on internal shapes of " + "this SDK that change across versions. Install a supported version " + '(e.g. `uv pip install "openai-agents[litellm]>=0.19.0,<0.20"`) or, ' + "if you are intentionally bumping the SDK, update " + "SUPPORTED_AGENTS_SDK_RANGE in strix/runtime/sdk_compat.py after " + "re-verifying the checklist documented in that file's module " + "docstring." + ) + raise RuntimeError(msg) diff --git a/tests/test_sdk_compat.py b/tests/test_sdk_compat.py new file mode 100644 index 000000000..d65fb41b2 --- /dev/null +++ b/tests/test_sdk_compat.py @@ -0,0 +1,59 @@ +"""assert_compatible_sdk_version() guards the openai-agents SDK version. + +Strix's sandbox runtime (strix/runtime/docker_client.py) and agent tool +wiring (strix/agents/factory.py) depend on internal shapes of the +``openai-agents`` SDK that are not guaranteed stable across versions (see +strix/runtime/sdk_compat.py for the full list). This module must raise a +clear RuntimeError for an out-of-range installed version and pass silently +for a version inside SUPPORTED_AGENTS_SDK_RANGE. +""" + +from __future__ import annotations + +import pytest + +from strix.runtime import sdk_compat + + +def test_currently_pinned_version_is_compatible() -> None: + """The version actually installed in this environment must pass. + + This keeps SUPPORTED_AGENTS_SDK_RANGE honest against what pyproject.toml + and uv.lock actually resolve to. + """ + installed = sdk_compat.installed_agents_sdk_version() + assert sdk_compat._satisfies(installed, sdk_compat.SUPPORTED_AGENTS_SDK_RANGE) + # Must not raise. + sdk_compat.assert_compatible_sdk_version() + + +@pytest.mark.parametrize("bad_version", ["0.14.6", "0.18.9", "0.20.0", "1.0.0"]) +def test_out_of_range_version_raises_clear_error( + monkeypatch: pytest.MonkeyPatch, bad_version: str +) -> None: + monkeypatch.setattr(sdk_compat, "installed_agents_sdk_version", lambda: bad_version) + + with pytest.raises(RuntimeError) as exc_info: + sdk_compat.assert_compatible_sdk_version() + + message = str(exc_info.value) + assert bad_version in message + assert "openai-agents" in message + for spec in sdk_compat.SUPPORTED_AGENTS_SDK_RANGE: + assert spec in message + assert "sdk_compat.py" in message + + +@pytest.mark.parametrize("good_version", ["0.19.0", "0.19.1", "0.19.99"]) +def test_in_range_versions_pass(monkeypatch: pytest.MonkeyPatch, good_version: str) -> None: + monkeypatch.setattr(sdk_compat, "installed_agents_sdk_version", lambda: good_version) + sdk_compat.assert_compatible_sdk_version() # must not raise + + +def test_unparseable_version_raises_clear_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sdk_compat, "installed_agents_sdk_version", lambda: "not-a-version") + + with pytest.raises(RuntimeError) as exc_info: + sdk_compat.assert_compatible_sdk_version() + + assert "not-a-version" in str(exc_info.value) From 954ad4a163ee1c80523f633750fb4bc3ccfbc787 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:55:48 +0100 Subject: [PATCH 6/9] feat(benchmarks): add reproducible XBEN scoring harness Add benchmarks/xben/: a JSON Schema for a single challenge run record, score.py (validates + aggregates committed run records into the results table, with a --check mode that fails loudly against benchmarks/README.md), run.py (drives real strix scans against XBEN challenges and emits conformant run records; needs Docker + an LLM key to execute for real), and a small set of clearly-marked synthetic fixture records under results/ so the harness can be exercised and unit tested without a live environment. Closes out the intent of usestrix/strix#1263 and usestrix/strix#1264: the headline "96% (100/104)" XBEN claim in benchmarks/README.md previously had no runnable evaluation code or committed data behind it in this repo. Co-Authored-By: Claude Sonnet 5 --- benchmarks/__init__.py | 0 benchmarks/xben/README.md | 126 +++++ benchmarks/xben/__init__.py | 0 benchmarks/xben/results/README.md | 35 ++ .../xben/results/example-XBEN-001-24.json | 18 + .../xben/results/example-XBEN-014-24.json | 18 + .../xben/results/example-XBEN-030-24.json | 18 + .../xben/results/example-XBEN-045-24.json | 18 + .../xben/results/example-XBEN-090-24.json | 18 + benchmarks/xben/run.py | 431 +++++++++++++++++ benchmarks/xben/schema/run_record.schema.json | 104 ++++ benchmarks/xben/score.py | 451 ++++++++++++++++++ tests/test_xben_run.py | 331 +++++++++++++ tests/test_xben_score.py | 296 ++++++++++++ 14 files changed, 1864 insertions(+) create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/xben/README.md create mode 100644 benchmarks/xben/__init__.py create mode 100644 benchmarks/xben/results/README.md create mode 100644 benchmarks/xben/results/example-XBEN-001-24.json create mode 100644 benchmarks/xben/results/example-XBEN-014-24.json create mode 100644 benchmarks/xben/results/example-XBEN-030-24.json create mode 100644 benchmarks/xben/results/example-XBEN-045-24.json create mode 100644 benchmarks/xben/results/example-XBEN-090-24.json create mode 100644 benchmarks/xben/run.py create mode 100644 benchmarks/xben/schema/run_record.schema.json create mode 100644 benchmarks/xben/score.py create mode 100644 tests/test_xben_run.py create mode 100644 tests/test_xben_score.py diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/xben/README.md b/benchmarks/xben/README.md new file mode 100644 index 000000000..9518ffecc --- /dev/null +++ b/benchmarks/xben/README.md @@ -0,0 +1,126 @@ +# XBEN benchmark harness + +This directory is an in-repo, reproducible harness for Strix's [XBEN](https://github.com/usestrix/benchmarks/tree/main/XBEN) +results. It exists to close out the intent of two open GitHub issues on +`usestrix/strix`: + +- [#1263 — "Make published XBEN metrics reproducible from committed data"](https://github.com/usestrix/strix/issues/1263) +- [#1264 — "Record enough provenance to reproduce XBEN runs"](https://github.com/usestrix/strix/issues/1264) + +Previously, `benchmarks/README.md` stated a headline number ("96% (100/104 +challenges)") with no runnable evaluation code and no committed per-challenge +data in this repository — only a link to an external repo. That's not +independently checkable, and it's exactly the kind of gap that led to +[Escape.tech's independent benchmark](https://escape.tech/) reporting a very +different detection rate against a different (non-CTF) target: CTF-style +flag-hunting and black-box real-app assessment measure different things, and +"trust us" numbers can't be told apart from either without data to check +them against. + +This harness makes the XBEN number **mechanically regenerated from committed +data** instead of hand-typed: + +``` +per-challenge run record (JSON, committed) → score.py → markdown table + │ + └─ --check: fails loudly + if committed data + doesn't match the + README's claim +``` + +## Layout + +| Path | Purpose | +|------|---------| +| `schema/run_record.schema.json` | JSON Schema for one challenge run record. | +| `results/*.json` | Committed run records, one file per challenge run. **Currently fixture/example data only** — see `results/README.md`. | +| `score.py` | Validates every record, aggregates them, prints/writes a markdown results table, and (`--check`) verifies the aggregate matches `benchmarks/README.md`. | +| `run.py` | Runner that drives real `strix` scans against XBEN challenges and emits conformant run records. Needs Docker + a live LLM API key to actually execute. | + +## Reproducing a run + +1. Check out the challenge set from [usestrix/benchmarks](https://github.com/usestrix/benchmarks/tree/main/XBEN) + (external — not vendored into this repo). +2. Point `run.py` at it (or at a JSON manifest you write) and pick a model: + + ```bash + python3 benchmarks/xben/run.py \ + --challenges-dir /path/to/usestrix-benchmarks/XBEN \ + --model anthropic/claude-sonnet-4-5 \ + --mode black-box \ + --out-dir benchmarks/xben/results + ``` + + This shells out to the real `strix` CLI once per challenge (see + `strix/interface/cli_args.py` for the flags it uses: `--target`, + `--non-interactive`, `--run-name`, `--max-budget`, `--max-turns`), reads + the resulting `strix_runs//run.json` for cost/token/timing data, + checks the challenge's flag against the saved artifacts, and writes one + schema-conformant run-record JSON per challenge. + + This step needs Docker (Strix's sandbox runtime) and a working API key + for the chosen model — neither is available in every environment (e.g. + this harness was built in one without them), so `run.py` also supports + `--dry-run` to print the commands it would execute without running them, + and its argument-parsing / record-construction logic is unit tested with + subprocess calls mocked out (`tests/test_xben_run.py`) rather than against + a real scan. + +3. Commit the resulting `benchmarks/xben/results/*.json` files (delete the + fixture files in that directory first — see `results/README.md`). + +## Scoring + +```bash +# Print the regenerated results table to stdout +python3 benchmarks/xben/score.py benchmarks/xben/results/ + +# Write it to a file instead +python3 benchmarks/xben/score.py benchmarks/xben/results/ --out /tmp/xben_table.md + +# Verify the committed data reproduces the claim in benchmarks/README.md +# (this is what CI runs; exits non-zero on any mismatch, naming it) +python3 benchmarks/xben/score.py benchmarks/xben/results/ --check +``` + +`score.py`: + +1. Loads every `*.json` file in the results directory (default + `benchmarks/xben/results/`). +2. Validates each one against `schema/run_record.schema.json`. An invalid + record fails loudly, naming the file and the offending field(s) — it does + not skip bad data silently. +3. Aggregates: overall success rate, a breakdown by difficulty (`easy` / + `medium` / `hard`, matching the existing README's Level 1/2/3 tiers), + average solve time, average cost, and total cost. +4. Renders a markdown table in the same shape as the existing + `benchmarks/README.md` table, so that table can be regenerated + mechanically rather than hand-typed. +5. With `--check`, parses the numbers currently hand-typed in + `benchmarks/README.md` (via a small regex over its markdown tables) and + diffs them against the aggregate, exiting non-zero and printing every + mismatch if they disagree. + +### Why `--check` currently fails + +`benchmarks/xben/results/` presently holds only 5 synthetic fixture records +(see `results/README.md`) used to exercise and test this harness — not real +Strix runs. Running `--check` against them and the real `benchmarks/README.md` +**correctly fails**: the committed data does not (yet) reproduce the "96% +(100/104)" claim, because there is no real committed data behind it yet. That +failure is the point — it is the honest, mechanically-verifiable answer to +issues #1263 and #1264 today. Once real run records replace the fixtures +(via `run.py` or a manually graded run), `--check` either passes or tells you +exactly how `benchmarks/README.md` needs to be updated. + +## Schema + +See `schema/run_record.schema.json` for the authoritative field list. Briefly, +each run record captures: which challenge and difficulty tier, which Strix +version and LLM (`model`) and testing `mode` (black-box/grey-box/white-box) +were used, whether it was `solved` and whether the `flag_matched`, timing +(`duration_seconds`, `started_at`/`finished_at`), cost/token usage +(`total_cost_usd`, `input_tokens`, `output_tokens`), and a `run_id` tying the +record back to the underlying `strix_runs//` artifact directory for +provenance — the specific gap issue #1264 asks to close. diff --git a/benchmarks/xben/__init__.py b/benchmarks/xben/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/xben/results/README.md b/benchmarks/xben/results/README.md new file mode 100644 index 000000000..62a4bf542 --- /dev/null +++ b/benchmarks/xben/results/README.md @@ -0,0 +1,35 @@ +# Example / fixture results + +The five `*.json` files in this directory are **synthetic fixture data**, not +real Strix run results. They exist to: + +- validate against `benchmarks/xben/schema/run_record.schema.json`, and +- give `benchmarks/xben/score.py` something to aggregate end-to-end, so the + scoring and `--check` machinery can be exercised and unit-tested without a + live Docker + LLM-key environment (which this harness was built without + access to). + +They are deliberately small (5 challenges) and deliberately do **not** +reproduce the headline "96% (100/104)" claim in `benchmarks/README.md` — do +not treat them as, or cite them as, real benchmark results. Running: + +```bash +python3 benchmarks/xben/score.py benchmarks/xben/results/ --check +``` + +against these fixtures and the real `benchmarks/README.md` is expected to +**fail** (`--check` exits non-zero) — that failure is correct and intentional. +It demonstrates the actual guarantee this harness provides: the current +"96%" claim is not (yet) backed by data reproducible from this repository, +which is exactly the gap GitHub issues +[#1263](https://github.com/usestrix/strix/issues/1263) and +[#1264](https://github.com/usestrix/strix/issues/1264) ask to close. + +## Replacing these with real results + +When real XBEN runs are produced (via `benchmarks/xben/run.py` or manually), +commit their run-record JSON files here — named e.g. +`..json` — and delete or replace these fixtures. Once +enough real runs are committed, `score.py --check` should pass against +`benchmarks/README.md`, or `benchmarks/README.md`'s table should be +regenerated from `score.py`'s output to match reality. diff --git a/benchmarks/xben/results/example-XBEN-001-24.json b/benchmarks/xben/results/example-XBEN-001-24.json new file mode 100644 index 000000000..9ab18bd8d --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-001-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-001-24", + "challenge_name": "Example: SQL Injection Login Bypass", + "difficulty": "easy", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": true, + "flag_matched": true, + "duration_seconds": 540, + "total_cost_usd": 1.85, + "input_tokens": 210000, + "output_tokens": 8400, + "started_at": "2026-01-10T09:00:00Z", + "finished_at": "2026-01-10T09:09:00Z", + "run_id": "example-xben-001-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/results/example-XBEN-014-24.json b/benchmarks/xben/results/example-XBEN-014-24.json new file mode 100644 index 000000000..52910ccec --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-014-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-014-24", + "challenge_name": "Example: Reflected XSS in Search", + "difficulty": "easy", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": true, + "flag_matched": true, + "duration_seconds": 420, + "total_cost_usd": 1.4, + "input_tokens": 160000, + "output_tokens": 6100, + "started_at": "2026-01-10T09:15:00Z", + "finished_at": "2026-01-10T09:22:00Z", + "run_id": "example-xben-014-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/results/example-XBEN-030-24.json b/benchmarks/xben/results/example-XBEN-030-24.json new file mode 100644 index 000000000..bfb09b586 --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-030-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-030-24", + "challenge_name": "Example: IDOR in Order History API", + "difficulty": "medium", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": true, + "flag_matched": true, + "duration_seconds": 1260, + "total_cost_usd": 3.9, + "input_tokens": 480000, + "output_tokens": 19500, + "started_at": "2026-01-10T10:00:00Z", + "finished_at": "2026-01-10T10:21:00Z", + "run_id": "example-xben-030-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/results/example-XBEN-045-24.json b/benchmarks/xben/results/example-XBEN-045-24.json new file mode 100644 index 000000000..a69004169 --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-045-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-045-24", + "challenge_name": "Example: JWT Algorithm Confusion", + "difficulty": "medium", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": false, + "flag_matched": false, + "duration_seconds": 2400, + "total_cost_usd": 6.2, + "input_tokens": 910000, + "output_tokens": 34000, + "started_at": "2026-01-10T11:00:00Z", + "finished_at": "2026-01-10T11:40:00Z", + "run_id": "example-xben-045-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result. Agent ran out of turns before forging a valid token." +} diff --git a/benchmarks/xben/results/example-XBEN-090-24.json b/benchmarks/xben/results/example-XBEN-090-24.json new file mode 100644 index 000000000..9606e105c --- /dev/null +++ b/benchmarks/xben/results/example-XBEN-090-24.json @@ -0,0 +1,18 @@ +{ + "challenge_id": "XBEN-090-24", + "challenge_name": "Example: Blind SSRF via PDF Renderer", + "difficulty": "hard", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": false, + "flag_matched": false, + "duration_seconds": 3600, + "total_cost_usd": 11.5, + "input_tokens": 1500000, + "output_tokens": 61000, + "started_at": "2026-01-10T12:00:00Z", + "finished_at": "2026-01-10T13:00:00Z", + "run_id": "example-xben-090-24-run", + "notes": "Synthetic fixture data for testing benchmarks/xben/score.py; not a real result." +} diff --git a/benchmarks/xben/run.py b/benchmarks/xben/run.py new file mode 100644 index 000000000..7fdfcf581 --- /dev/null +++ b/benchmarks/xben/run.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Run the XBEN challenge suite against a real Strix install and emit run records. + +This is the actual reproduction path for GitHub issues #1263 and #1264 on +usestrix/strix: given a checkout of the challenges from the external +``usestrix/benchmarks`` repository and a target model, this drives ``strix`` +as a subprocess once per challenge and writes a conformant run-record JSON +(see ``benchmarks/xben/schema/run_record.schema.json``) for each one under +``benchmarks/xben/results/``. + +Requires a live environment this repository's CI/dev sandbox does not have: +Docker (to run Strix's sandbox runtime) and a working LLM API key for the +selected model. Nothing in this module is exercised end-to-end without that +environment; the argument-parsing and record-construction logic is unit +tested with mocked subprocess calls (see ``tests/test_xben_run.py``), but +actually launching a challenge and getting a real flag match is not faked +here. + +Challenge discovery +-------------------- +The XBEN challenge set lives in the external ``usestrix/benchmarks`` +repository, not in this one. This script does not know that repository's +exact on-disk layout, so it supports two ways to tell it about challenges: + +1. ``--manifest PATH``: a JSON file you provide explicitly, shaped as a list + of objects: ``{"challenge_id", "challenge_name", "difficulty", "target", + "flag"}`` (``flag`` is optional; when omitted, ``flag_matched`` in the + emitted record is always ``false`` and must be graded manually). +2. ``--challenges-dir PATH`` (no manifest): every immediate subdirectory of + ``PATH`` containing a ``challenge.json`` file (with the same shape as one + manifest entry, minus ``challenge_id`` which defaults to the directory + name) is treated as one challenge. Adjust ``discover_challenges`` here if + the real ``usestrix/benchmarks`` layout differs once you have it checked + out. + +Usage: + python3 benchmarks/xben/run.py --manifest challenges.json \\ + --model anthropic/claude-sonnet-4-5 --mode black-box \\ + --out-dir benchmarks/xben/results +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from importlib.metadata import version +from pathlib import Path +from typing import Any + +from rich.console import Console + +from strix.core.paths import run_dir_for + + +console = Console() +error_console = Console(stderr=True) + +_DEFAULT_OUT_DIR = Path(__file__).parent / "results" +_VALID_DIFFICULTIES = ("easy", "medium", "hard") +_VALID_MODES = ("black-box", "grey-box", "white-box") + + +class ChallengeSpecError(Exception): + """A challenge manifest/directory entry is missing required fields.""" + + +@dataclass +class ChallengeSpec: + challenge_id: str + challenge_name: str + difficulty: str + target: str + flag: str | None = None + + +def _challenge_from_dict( + data: dict[str, Any], *, default_id: str | None, source: str +) -> ChallengeSpec: + challenge_id = str(data.get("challenge_id") or default_id or "") + if not challenge_id: + raise ChallengeSpecError(f"{source}: missing 'challenge_id'") + + difficulty = str(data.get("difficulty", "")) + if difficulty not in _VALID_DIFFICULTIES: + raise ChallengeSpecError( + f"{source}: 'difficulty' must be one of {_VALID_DIFFICULTIES}, got {difficulty!r}", + ) + + target = data.get("target") + if not target or not isinstance(target, str): + raise ChallengeSpecError(f"{source}: missing or invalid 'target'") + + flag = data.get("flag") + return ChallengeSpec( + challenge_id=challenge_id, + challenge_name=str(data.get("challenge_name") or challenge_id), + difficulty=difficulty, + target=target, + flag=str(flag) if flag else None, + ) + + +def load_manifest(manifest_path: Path) -> list[ChallengeSpec]: + raw: Any = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ChallengeSpecError(f"{manifest_path}: manifest must be a JSON array") + return [ + _challenge_from_dict(entry, default_id=None, source=f"{manifest_path}[{i}]") + for i, entry in enumerate(raw) + if isinstance(entry, dict) + ] + + +def discover_challenges(challenges_dir: Path) -> list[ChallengeSpec]: + """Discover challenges from ``//challenge.json`` entries. + + This assumes each challenge has a ``challenge.json`` describing its target + and difficulty. Adjust this function once the real usestrix/benchmarks + XBEN layout is available to run against — this repository was built + without network access to check it out. + """ + if not challenges_dir.is_dir(): + raise FileNotFoundError(f"challenges directory not found: {challenges_dir}") + + specs: list[ChallengeSpec] = [] + for child in sorted(challenges_dir.iterdir()): + manifest_file = child / "challenge.json" + if not child.is_dir() or not manifest_file.is_file(): + continue + raw: Any = json.loads(manifest_file.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ChallengeSpecError(f"{manifest_file}: must be a JSON object") + specs.append(_challenge_from_dict(raw, default_id=child.name, source=str(manifest_file))) + + if not specs: + raise ChallengeSpecError( + f"no challenge.json files found under {challenges_dir} " + "(pass --manifest instead if the challenge set uses a different layout)", + ) + return specs + + +def build_strix_command( + *, + strix_bin: str, + target: str, + run_name: str, + max_budget_usd: float | None, + max_turns: int | None, +) -> list[str]: + """Build the ``strix`` CLI invocation for one challenge run. + + Mirrors the flags documented in ``strix/interface/cli_args.py``: + ``--target``/``-t``, ``--non-interactive``/``-n`` (required for a + subprocess run — the interactive TUI would hang), and the optional + ``--max-budget`` / ``--max-turns`` guards so a single stuck challenge + can't burn the whole benchmark's budget. + """ + command = [ + strix_bin, + "--target", + target, + "--non-interactive", + "--run-name", + run_name, + ] + if max_budget_usd is not None: + command += ["--max-budget", str(max_budget_usd)] + if max_turns is not None: + command += ["--max-turns", str(max_turns)] + return command + + +def run_strix_subprocess( + command: list[str], *, env: dict[str, str] +) -> subprocess.CompletedProcess[str]: + """Invoke the ``strix`` CLI as a subprocess and capture its output. + + Requires Docker and a working LLM API key in ``env`` — this is the part + of the pipeline that cannot be exercised in a sandbox without either. + """ + return subprocess.run( # noqa: S603 + command, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def read_run_json(run_dir: Path) -> dict[str, Any]: + run_json_path = run_dir / "run.json" + if not run_json_path.is_file(): + raise FileNotFoundError(f"no run.json produced under {run_dir}") + data: Any = json.loads(run_json_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise TypeError(f"{run_json_path} is not a JSON object") + return data + + +def flag_was_matched(run_dir: Path, flag: str | None) -> bool: + """Best-effort check for the challenge flag in the run's saved artifacts. + + Searches ``vulnerabilities.json`` and the executive report markdown for + the literal flag string. A real grading pass may want something more + precise (e.g. the agent's own "finish" tool call payload); this is + intentionally simple and documented as best-effort. + """ + if not flag: + return False + candidates = [run_dir / "vulnerabilities.json", run_dir / "penetration_test_report.md"] + return any(path.is_file() and flag in path.read_text(encoding="utf-8") for path in candidates) + + +def get_strix_version() -> str: + try: + return version("strix-agent") + except Exception: # noqa: BLE001 + return "unknown" + + +def build_run_record( + *, + challenge: ChallengeSpec, + run_dir: Path, + run_id: str, + model: str, + mode: str, + started_at: datetime, + finished_at: datetime, +) -> dict[str, Any]: + """Construct a schema-conformant run record from a completed run's artifacts.""" + run_json = read_run_json(run_dir) + llm_usage = run_json.get("llm_usage") or {} + status = run_json.get("status") + + matched = flag_was_matched(run_dir, challenge.flag) + solved = matched if challenge.flag else status == "completed" + + return { + "challenge_id": challenge.challenge_id, + "challenge_name": challenge.challenge_name, + "difficulty": challenge.difficulty, + "strix_version": get_strix_version(), + "model": model, + "mode": mode, + "solved": bool(solved), + "flag_matched": bool(matched), + "duration_seconds": (finished_at - started_at).total_seconds(), + "total_cost_usd": float(llm_usage.get("cost") or 0.0), + "input_tokens": int(llm_usage.get("input_tokens") or 0), + "output_tokens": int(llm_usage.get("output_tokens") or 0), + "started_at": started_at.isoformat(), + "finished_at": finished_at.isoformat(), + "run_id": run_id, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--manifest", + type=Path, + help="JSON file listing challenges explicitly (see module docstring for shape).", + ) + source.add_argument( + "--challenges-dir", + type=Path, + help="Directory of /challenge.json entries (see module docstring).", + ) + parser.add_argument( + "--model", + required=True, + help="LLM identifier to run with, passed through as the STRIX_LLM env var " + "(litellm-style, e.g. 'anthropic/claude-sonnet-4-5').", + ) + parser.add_argument( + "--mode", + choices=_VALID_MODES, + default="black-box", + help="Testing mode recorded in the run record (default: black-box).", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=_DEFAULT_OUT_DIR, + help=f"Directory to write run-record JSON files to (default: {_DEFAULT_OUT_DIR}).", + ) + parser.add_argument( + "--strix-bin", + default="strix", + help="Path to the strix executable to invoke (default: 'strix' on PATH).", + ) + parser.add_argument( + "--max-budget", + type=float, + default=None, + help="Per-challenge --max-budget passed through to strix, in USD.", + ) + parser.add_argument( + "--max-turns", + type=int, + default=None, + help="Per-challenge --max-turns passed through to strix.", + ) + parser.add_argument( + "--challenge-id", + action="append", + dest="challenge_ids", + default=None, + help="Only run this challenge id (repeatable). Default: run every discovered challenge.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the strix commands that would run, without executing them.", + ) + return parser + + +def _select_challenges( + challenges: list[ChallengeSpec], challenge_ids: list[str] | None +) -> list[ChallengeSpec]: + if not challenge_ids: + return challenges + wanted = set(challenge_ids) + selected = [c for c in challenges if c.challenge_id in wanted] + missing = wanted - {c.challenge_id for c in selected} + if missing: + raise ChallengeSpecError(f"unknown --challenge-id value(s): {sorted(missing)}") + return selected + + +def run_one_challenge( + challenge: ChallengeSpec, + *, + args: argparse.Namespace, + env: dict[str, str], + run_dir_for_name: Any, +) -> dict[str, Any]: + """Run a single challenge end-to-end and return its run record.""" + run_name = f"xben-{challenge.challenge_id}-{int(datetime.now(tz=UTC).timestamp())}" + command = build_strix_command( + strix_bin=args.strix_bin, + target=challenge.target, + run_name=run_name, + max_budget_usd=args.max_budget, + max_turns=args.max_turns, + ) + + started_at = datetime.now(tz=UTC) + result = run_strix_subprocess(command, env=env) + finished_at = datetime.now(tz=UTC) + + if result.returncode != 0: + error_console.print( + f"[bold red]strix exited {result.returncode} for {challenge.challenge_id}[/bold red]", + ) + error_console.print(result.stderr) + + run_dir = run_dir_for_name(run_name) + return build_run_record( + challenge=challenge, + run_dir=run_dir, + run_id=run_name, + model=args.model, + mode=args.mode, + started_at=started_at, + finished_at=finished_at, + ) + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + try: + challenges = ( + load_manifest(args.manifest) + if args.manifest + else discover_challenges(args.challenges_dir) + ) + challenges = _select_challenges(challenges, args.challenge_ids) + except (ChallengeSpecError, FileNotFoundError, json.JSONDecodeError) as exc: + error_console.print(f"[bold red]error:[/bold red] {exc}") + return 1 + + if args.dry_run: + for challenge in challenges: + command = build_strix_command( + strix_bin=args.strix_bin, + target=challenge.target, + run_name=f"xben-{challenge.challenge_id}-", + max_budget_usd=args.max_budget, + max_turns=args.max_turns, + ) + console.print(f"[{challenge.challenge_id}] {' '.join(command)}") + return 0 + + env = dict(os.environ) + env["STRIX_LLM"] = args.model + + args.out_dir.mkdir(parents=True, exist_ok=True) + exit_code = 0 + for challenge in challenges: + try: + record = run_one_challenge( + challenge, + args=args, + env=env, + run_dir_for_name=run_dir_for, + ) + except (FileNotFoundError, TypeError) as exc: + error_console.print(f"[bold red]{challenge.challenge_id} failed:[/bold red] {exc}") + exit_code = 1 + continue + + out_path = args.out_dir / f"{challenge.challenge_id}.{args.model.replace('/', '_')}.json" + out_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + console.print(f"[green]wrote[/green] {out_path}") + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/xben/schema/run_record.schema.json b/benchmarks/xben/schema/run_record.schema.json new file mode 100644 index 000000000..e1fffd780 --- /dev/null +++ b/benchmarks/xben/schema/run_record.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/usestrix/strix/blob/main/benchmarks/xben/schema/run_record.schema.json", + "title": "XBEN Run Record", + "description": "A single Strix run against a single XBEN challenge. One JSON file per challenge run, committed under benchmarks/xben/results/, is the unit of provenance the benchmark's aggregate numbers (benchmarks/xben/score.py) are computed from. See benchmarks/xben/README.md, and GitHub issues #1263 and #1264 on usestrix/strix, for why this file exists.", + "type": "object", + "additionalProperties": false, + "required": [ + "challenge_id", + "challenge_name", + "difficulty", + "strix_version", + "model", + "mode", + "solved", + "flag_matched", + "duration_seconds", + "total_cost_usd", + "input_tokens", + "output_tokens", + "started_at", + "finished_at", + "run_id" + ], + "properties": { + "challenge_id": { + "type": "string", + "minLength": 1, + "description": "Stable identifier of the XBEN challenge, as named in the usestrix/benchmarks XBEN challenge set (e.g. 'XBEN-001-24')." + }, + "challenge_name": { + "type": "string", + "minLength": 1, + "description": "Human-readable challenge title." + }, + "difficulty": { + "type": "string", + "enum": ["easy", "medium", "hard"], + "description": "Challenge difficulty tier. Maps to the existing benchmarks/README.md table as easy=Level 1, medium=Level 2, hard=Level 3." + }, + "strix_version": { + "type": "string", + "minLength": 1, + "description": "Strix version that produced this run, e.g. '0.4.0' (matches `strix --version` / the strix-agent package version)." + }, + "model": { + "type": "string", + "minLength": 1, + "description": "LLM identifier used for the run, e.g. the value passed via the STRIX_LLM environment variable (litellm-style model string such as 'anthropic/claude-sonnet-4-5')." + }, + "mode": { + "type": "string", + "enum": ["black-box", "grey-box", "white-box"], + "description": "Testing mode: black-box (URL/deployed target only), grey-box (target + partial knowledge), or white-box (source code access)." + }, + "solved": { + "type": "boolean", + "description": "Whether the agent was judged to have solved the challenge overall (normally identical to flag_matched for XBEN's CTF-style scoring, but kept distinct so a run can be recorded even when solved-ness is determined by something other than an exact flag match)." + }, + "flag_matched": { + "type": "boolean", + "description": "Whether the agent recovered and reported the exact challenge flag." + }, + "duration_seconds": { + "type": "number", + "minimum": 0, + "description": "Wall-clock time for the run, in seconds." + }, + "total_cost_usd": { + "type": "number", + "minimum": 0, + "description": "Total LLM cost for the run in USD, as reported by Strix's usage accounting (run.json llm_usage.cost)." + }, + "input_tokens": { + "type": "integer", + "minimum": 0, + "description": "Total input/prompt tokens consumed across all LLM calls in the run." + }, + "output_tokens": { + "type": "integer", + "minimum": 0, + "description": "Total output/completion tokens produced across all LLM calls in the run." + }, + "started_at": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp (UTC) when the run started." + }, + "finished_at": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp (UTC) when the run finished." + }, + "run_id": { + "type": "string", + "minLength": 1, + "description": "Provenance link back to the underlying Strix run artifact directory, i.e. the '' in 'strix_runs//' that this record was derived from." + }, + "notes": { + "type": "string", + "description": "Optional free-text notes (e.g. manual grading remarks, anomalies, retries)." + } + } +} diff --git a/benchmarks/xben/score.py b/benchmarks/xben/score.py new file mode 100644 index 000000000..b92f4dc48 --- /dev/null +++ b/benchmarks/xben/score.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Aggregate and score committed XBEN run records. + +Reads a directory of run-record JSON files (see +``benchmarks/xben/schema/run_record.schema.json``), validates each one, and +computes the same headline numbers hand-typed into ``benchmarks/README.md``: +overall success rate, a breakdown by difficulty, average solve time, and +total/average cost. + +This is the "reproducible from committed data" half of closing out GitHub +issues #1263 ("Make published XBEN metrics reproducible from committed +data") and #1264 ("Record enough provenance to reproduce XBEN runs") on +usestrix/strix: instead of a hand-typed table with no backing data, the +table is regenerated mechanically from JSON files committed under +``benchmarks/xben/results/``. + +Usage: + python3 benchmarks/xben/score.py [RESULTS_DIR] [--out PATH] [--check] + +See ``benchmarks/xben/README.md`` for the full walkthrough. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +from rich.console import Console + + +_DEFAULT_RESULTS_DIR = Path(__file__).parent / "results" +_DEFAULT_SCHEMA_PATH = Path(__file__).parent / "schema" / "run_record.schema.json" +_DEFAULT_README_PATH = Path(__file__).parent.parent / "README.md" + +_DIFFICULTY_LABELS: dict[str, str] = { + "easy": "Level 1 (Easy)", + "medium": "Level 2 (Medium)", + "hard": "Level 3 (Hard)", +} +_DIFFICULTY_ORDER: tuple[str, ...] = ("easy", "medium", "hard") + +console = Console() +error_console = Console(stderr=True) + + +class SchemaValidationError(Exception): + """A run-record JSON file failed validation against the XBEN schema.""" + + +# --------------------------------------------------------------------------- +# Minimal JSON Schema validation. +# +# ``jsonschema`` is not a declared dependency of strix-agent (it only ever +# shows up transitively via other packages' requirements), so rather than +# depend on an undeclared package we implement just the subset of JSON +# Schema (Draft 2020-12) that benchmarks/xben/schema/run_record.schema.json +# actually uses: object/type/required/properties/additionalProperties, +# enum, minimum, and the "date-time" format. +# --------------------------------------------------------------------------- + +_TYPE_MAP: dict[str, tuple[type, ...]] = { + "object": (dict,), + "string": (str,), + "boolean": (bool,), + "number": (int, float), + "integer": (int,), +} + + +def _check_type(value: Any, expected: str, path: str, errors: list[str]) -> None: + python_types = _TYPE_MAP.get(expected) + if python_types is None: + return + # bool is a subclass of int in Python; JSON Schema treats them as distinct. + if expected in {"number", "integer"} and isinstance(value, bool): + errors.append(f"{path}: expected {expected}, got boolean") + return + if not isinstance(value, python_types): + errors.append(f"{path}: expected {expected}, got {type(value).__name__}") + + +def _check_format(value: str, fmt: str, path: str, errors: list[str]) -> None: + if fmt != "date-time": + return + try: + datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + errors.append(f"{path}: '{value}' is not a valid ISO 8601 date-time") + + +def _validate_property( + value: Any, prop_schema: dict[str, Any], path: str, errors: list[str] +) -> None: + expected_type = prop_schema.get("type") + if isinstance(expected_type, str): + _check_type(value, expected_type, path, errors) + + enum = prop_schema.get("enum") + if enum is not None and value not in enum: + errors.append(f"{path}: {value!r} is not one of {enum!r}") + + minimum = prop_schema.get("minimum") + if minimum is not None and isinstance(value, int | float) and value < minimum: + errors.append(f"{path}: {value} is less than the minimum {minimum}") + + min_length = prop_schema.get("minLength") + if min_length is not None and isinstance(value, str) and len(value) < min_length: + errors.append(f"{path}: string is shorter than minLength {min_length}") + + fmt = prop_schema.get("format") + if fmt is not None and isinstance(value, str): + _check_format(value, fmt, path, errors) + + +def validate_record(record: Any, schema: dict[str, Any], *, source: str) -> None: + """Validate ``record`` against ``schema``, raising with file+field context on failure.""" + errors: list[str] = [] + + if not isinstance(record, dict): + raise SchemaValidationError(f"{source}: top-level value must be a JSON object") + + required = schema.get("required", []) + errors.extend( + f"missing required field '{field_name}'" + for field_name in required + if field_name not in record + ) + + properties: dict[str, Any] = schema.get("properties", {}) + for key, value in record.items(): + if key not in properties: + if schema.get("additionalProperties") is False: + errors.append(f"unexpected field '{key}' (not in schema)") + continue + _validate_property(value, properties[key], key, errors) + + if errors: + formatted = "; ".join(errors) + raise SchemaValidationError(f"{source}: {formatted}") + + +# --------------------------------------------------------------------------- +# Loading and aggregation. +# --------------------------------------------------------------------------- + + +def load_schema(schema_path: Path) -> dict[str, Any]: + data: Any = json.loads(schema_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SchemaValidationError(f"{schema_path}: schema file is not a JSON object") + return data + + +def load_results(results_dir: Path, schema: dict[str, Any]) -> list[dict[str, Any]]: + """Load and validate every ``*.json`` run record under ``results_dir``. + + Raises ``SchemaValidationError`` naming the offending file and field on the + first invalid record — validation failures are meant to be loud, not + silently skipped. + """ + if not results_dir.is_dir(): + raise FileNotFoundError(f"results directory not found: {results_dir}") + + records: list[dict[str, Any]] = [] + for path in sorted(results_dir.glob("*.json")): + try: + raw: Any = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise SchemaValidationError(f"{path.name}: invalid JSON ({exc})") from exc + validate_record(raw, schema, source=path.name) + records.append(raw) + + if not records: + raise SchemaValidationError(f"no run-record *.json files found under {results_dir}") + + return records + + +@dataclass +class DifficultyStats: + solved: int = 0 + total: int = 0 + + @property + def success_rate(self) -> float: + return (self.solved / self.total * 100) if self.total else 0.0 + + +@dataclass +class AggregateResult: + total_challenges: int = 0 + solved_challenges: int = 0 + total_duration_seconds: float = 0.0 + total_cost_usd: float = 0.0 + by_difficulty: dict[str, DifficultyStats] = field(default_factory=dict) + + @property + def success_rate(self) -> float: + return ( + (self.solved_challenges / self.total_challenges * 100) if self.total_challenges else 0.0 + ) + + @property + def avg_duration_seconds(self) -> float: + return self.total_duration_seconds / self.total_challenges if self.total_challenges else 0.0 + + @property + def avg_cost_usd(self) -> float: + return self.total_cost_usd / self.total_challenges if self.total_challenges else 0.0 + + +def aggregate(records: list[dict[str, Any]]) -> AggregateResult: + result = AggregateResult() + for record in records: + result.total_challenges += 1 + solved = bool(record["solved"]) + if solved: + result.solved_challenges += 1 + result.total_duration_seconds += float(record["duration_seconds"]) + result.total_cost_usd += float(record["total_cost_usd"]) + + difficulty = str(record["difficulty"]) + stats = result.by_difficulty.setdefault(difficulty, DifficultyStats()) + stats.total += 1 + if solved: + stats.solved += 1 + + return result + + +# --------------------------------------------------------------------------- +# Markdown rendering, matching the shape of the existing benchmarks/README.md +# tables so that file's tables can be replaced wholesale by this output. +# --------------------------------------------------------------------------- + + +def _format_minutes(seconds: float) -> str: + return f"~{round(seconds / 60)} minutes" + + +def _format_cost(usd: float) -> str: + return f"~${usd:,.0f}" + + +def render_markdown(agg: AggregateResult) -> str: + lines: list[str] = [] + lines.append("| Benchmark | Challenges | Success Rate |") + lines.append("|-----------|------------|--------------|") + lines.append( + f"| XBEN | {agg.total_challenges} | **{agg.success_rate:.0f}%** |", + ) + lines.append("") + lines.append("**Performance by Difficulty:**") + lines.append("") + lines.append("| Difficulty | Solved | Success Rate |") + lines.append("|------------|--------|--------------|") + for key in _DIFFICULTY_ORDER: + stats = agg.by_difficulty.get(key) + if stats is None: + continue + label = _DIFFICULTY_LABELS[key] + lines.append(f"| {label} | {stats.solved}/{stats.total} | {stats.success_rate:.0f}% |") + lines.append("") + lines.append("**Resource Usage:**") + lines.append(f"- Average solve time: {_format_minutes(agg.avg_duration_seconds)}") + lines.append( + f"- Total cost: {_format_cost(agg.total_cost_usd)} for {agg.total_challenges} challenges", + ) + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- +# --check: parse the numbers currently hand-typed in benchmarks/README.md and +# compare them against what's computed from committed data. +# --------------------------------------------------------------------------- + + +@dataclass +class ReadmeClaims: + total_challenges: int + success_rate: float + by_difficulty: dict[str, DifficultyStats] + + +_OVERALL_ROW_RE = re.compile( + r"\|\s*\[?XBEN\]?[^|]*\|\s*(\d+)\s*\|\s*\*\*(\d+(?:\.\d+)?)%\*\*\s*\|", +) +_DIFFICULTY_ROW_RE = re.compile( + r"\|\s*Level\s+(\d+)\s*\(([A-Za-z]+)\)\s*\|\s*(\d+)/(\d+)\s*\|\s*(\d+(?:\.\d+)?)%\s*\|", +) + + +def parse_readme_claims(readme_text: str) -> ReadmeClaims: + """Extract the headline numbers from the existing hand-typed README table.""" + overall_match = _OVERALL_ROW_RE.search(readme_text) + if not overall_match: + raise SchemaValidationError( + "could not find the XBEN results row (e.g. '| XBEN | 104 | **96%** |') in the README", + ) + total_challenges = int(overall_match.group(1)) + success_rate = float(overall_match.group(2)) + + by_difficulty: dict[str, DifficultyStats] = {} + for match in _DIFFICULTY_ROW_RE.finditer(readme_text): + level_name = match.group(2).lower() + difficulty = next((d for d in _DIFFICULTY_ORDER if d == level_name), None) + if difficulty is None: + continue + solved, total = int(match.group(3)), int(match.group(4)) + by_difficulty[difficulty] = DifficultyStats(solved=solved, total=total) + + return ReadmeClaims( + total_challenges=total_challenges, + success_rate=success_rate, + by_difficulty=by_difficulty, + ) + + +def diff_against_claims(agg: AggregateResult, claims: ReadmeClaims) -> list[str]: + """Return a list of human-readable mismatches between ``agg`` and ``claims``.""" + mismatches: list[str] = [] + + if agg.total_challenges != claims.total_challenges: + mismatches.append( + f"total challenges: README claims {claims.total_challenges}, " + f"committed data has {agg.total_challenges}", + ) + if round(agg.success_rate) != round(claims.success_rate): + mismatches.append( + f"overall success rate: README claims {claims.success_rate:.0f}%, " + f"committed data computes {agg.success_rate:.0f}%", + ) + + for difficulty in _DIFFICULTY_ORDER: + # A difficulty absent from either side is equivalent to a zero-challenge + # DifficultyStats(0, 0) for that side, so a table row with "0/0" and no + # committed challenges of that difficulty compare as equal rather than + # as "present on one side only". + claimed = claims.by_difficulty.get(difficulty, DifficultyStats()) + actual = agg.by_difficulty.get(difficulty, DifficultyStats()) + if claimed.solved != actual.solved or claimed.total != actual.total: + mismatches.append( + f"difficulty '{difficulty}': README claims {claimed.solved}/{claimed.total}, " + f"committed data has {actual.solved}/{actual.total}", + ) + + return mismatches + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Validate and aggregate committed XBEN run records into the " + "benchmarks/README.md results table." + ), + ) + parser.add_argument( + "results_dir", + type=Path, + nargs="?", + default=_DEFAULT_RESULTS_DIR, + help=f"Directory of run-record *.json files (default: {_DEFAULT_RESULTS_DIR}).", + ) + parser.add_argument( + "--schema", + type=Path, + default=_DEFAULT_SCHEMA_PATH, + help=f"Path to the run-record JSON Schema (default: {_DEFAULT_SCHEMA_PATH}).", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="Write the markdown table to this file instead of stdout.", + ) + parser.add_argument( + "--check", + action="store_true", + help=( + "Exit non-zero if the aggregated numbers don't match what's currently " + "hand-typed in the README (this is what CI runs)." + ), + ) + parser.add_argument( + "--readme", + type=Path, + default=_DEFAULT_README_PATH, + help=f"README to check against with --check (default: {_DEFAULT_README_PATH}).", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + try: + schema = load_schema(args.schema) + records = load_results(args.results_dir, schema) + except (SchemaValidationError, FileNotFoundError) as exc: + error_console.print(f"[bold red]error:[/bold red] {exc}") + return 1 + + agg = aggregate(records) + markdown = render_markdown(agg) + + if args.out: + args.out.write_text(markdown, encoding="utf-8") + console.print(f"Wrote markdown table to {args.out}") + else: + console.print(markdown, highlight=False, soft_wrap=True) + + if args.check: + if not args.readme.is_file(): + error_console.print(f"[bold red]error:[/bold red] README not found: {args.readme}") + return 1 + readme_text = args.readme.read_text(encoding="utf-8") + try: + claims = parse_readme_claims(readme_text) + except SchemaValidationError as exc: + error_console.print(f"[bold red]error:[/bold red] {exc}") + return 1 + + mismatches = diff_against_claims(agg, claims) + if mismatches: + error_console.print( + f"[bold red]--check failed:[/bold red] aggregated numbers from " + f"{args.results_dir} do not match {args.readme}:", + ) + for mismatch in mismatches: + error_console.print(f" - {mismatch}") + return 1 + console.print( + f"[bold green]--check passed:[/bold green] {args.readme} matches committed data." + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_xben_run.py b/tests/test_xben_run.py new file mode 100644 index 000000000..52ba65ff9 --- /dev/null +++ b/tests/test_xben_run.py @@ -0,0 +1,331 @@ +"""Tests for the XBEN benchmark runner (benchmarks/xben/run.py). + +This module drives a real ``strix`` subprocess against a live Docker + LLM +environment, neither of which is available here. Every test below mocks +``subprocess.run`` (via ``run.run_strix_subprocess``) and the filesystem +artifacts a real run would produce, so only the argument-parsing and +run-record-construction logic is exercised -- never a real scan. +""" + +from __future__ import annotations + +import json +import subprocess +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock + +import pytest + +from benchmarks.xben import run + + +if TYPE_CHECKING: + from pathlib import Path + + +def _manifest_entry(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "challenge_id": "XBEN-001-24", + "challenge_name": "Test Challenge", + "difficulty": "easy", + "target": "https://target.example.com", + "flag": "FLAG{test}", + } + base.update(overrides) + return base + + +class TestChallengeFromDict: + def test_valid_entry(self) -> None: + spec = run._challenge_from_dict(_manifest_entry(), default_id=None, source="test") + assert spec.challenge_id == "XBEN-001-24" + assert spec.difficulty == "easy" + assert spec.target == "https://target.example.com" + assert spec.flag == "FLAG{test}" + + def test_default_id_used_when_missing(self) -> None: + entry = _manifest_entry() + del entry["challenge_id"] + spec = run._challenge_from_dict(entry, default_id="dir-name", source="test") + assert spec.challenge_id == "dir-name" + + def test_missing_id_raises(self) -> None: + entry = _manifest_entry() + del entry["challenge_id"] + with pytest.raises(run.ChallengeSpecError, match="challenge_id"): + run._challenge_from_dict(entry, default_id=None, source="test") + + def test_invalid_difficulty_raises(self) -> None: + with pytest.raises(run.ChallengeSpecError, match="difficulty"): + run._challenge_from_dict( + _manifest_entry(difficulty="impossible"), default_id=None, source="test" + ) + + def test_missing_target_raises(self) -> None: + entry = _manifest_entry() + del entry["target"] + with pytest.raises(run.ChallengeSpecError, match="target"): + run._challenge_from_dict(entry, default_id=None, source="test") + + def test_missing_flag_is_allowed(self) -> None: + entry = _manifest_entry() + del entry["flag"] + spec = run._challenge_from_dict(entry, default_id=None, source="test") + assert spec.flag is None + + +class TestLoadManifest: + def test_loads_list_of_entries(self, tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps([_manifest_entry(), _manifest_entry(challenge_id="B")])) + specs = run.load_manifest(manifest) + assert [s.challenge_id for s in specs] == ["XBEN-001-24", "B"] + + def test_non_list_manifest_raises(self, tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps({"not": "a list"})) + with pytest.raises(run.ChallengeSpecError, match="must be a JSON array"): + run.load_manifest(manifest) + + +class TestDiscoverChallenges: + def test_discovers_challenge_json_per_subdirectory(self, tmp_path: Path) -> None: + chal_dir = tmp_path / "chal-1" + chal_dir.mkdir() + entry = _manifest_entry() + del entry["challenge_id"] + (chal_dir / "challenge.json").write_text(json.dumps(entry)) + + specs = run.discover_challenges(tmp_path) + assert len(specs) == 1 + assert specs[0].challenge_id == "chal-1" + + def test_missing_directory_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + run.discover_challenges(tmp_path / "missing") + + def test_no_challenges_found_raises(self, tmp_path: Path) -> None: + with pytest.raises(run.ChallengeSpecError, match=r"no challenge\.json"): + run.discover_challenges(tmp_path) + + +class TestBuildStrixCommand: + def test_basic_command_shape(self) -> None: + command = run.build_strix_command( + strix_bin="strix", + target="https://target.example.com", + run_name="xben-run-1", + max_budget_usd=None, + max_turns=None, + ) + assert command == [ + "strix", + "--target", + "https://target.example.com", + "--non-interactive", + "--run-name", + "xben-run-1", + ] + + def test_includes_optional_budget_and_turns(self) -> None: + command = run.build_strix_command( + strix_bin="strix", + target="https://target.example.com", + run_name="xben-run-1", + max_budget_usd=5.0, + max_turns=100, + ) + assert "--max-budget" in command + assert command[command.index("--max-budget") + 1] == "5.0" + assert "--max-turns" in command + assert command[command.index("--max-turns") + 1] == "100" + + +class TestFlagWasMatched: + def test_no_flag_means_not_matched(self, tmp_path: Path) -> None: + assert run.flag_was_matched(tmp_path, None) is False + + def test_flag_found_in_vulnerabilities_json(self, tmp_path: Path) -> None: + (tmp_path / "vulnerabilities.json").write_text('[{"evidence": "FLAG{found-it}"}]') + assert run.flag_was_matched(tmp_path, "FLAG{found-it}") is True + + def test_flag_found_in_report_markdown(self, tmp_path: Path) -> None: + (tmp_path / "penetration_test_report.md").write_text("We recovered FLAG{in-report}.") + assert run.flag_was_matched(tmp_path, "FLAG{in-report}") is True + + def test_flag_not_present_anywhere(self, tmp_path: Path) -> None: + (tmp_path / "vulnerabilities.json").write_text("[]") + assert run.flag_was_matched(tmp_path, "FLAG{missing}") is False + + +class TestBuildRunRecord: + def test_builds_conformant_record_from_run_json(self, tmp_path: Path) -> None: + run_dir = tmp_path / "strix_runs" / "xben-run-1" + run_dir.mkdir(parents=True) + (run_dir / "run.json").write_text( + json.dumps( + { + "status": "completed", + "llm_usage": {"cost": 2.5, "input_tokens": 1000, "output_tokens": 200}, + }, + ), + ) + (run_dir / "vulnerabilities.json").write_text('[{"evidence": "FLAG{yes}"}]') + + challenge = run.ChallengeSpec( + challenge_id="XBEN-001-24", + challenge_name="Test", + difficulty="easy", + target="https://target.example.com", + flag="FLAG{yes}", + ) + started = datetime(2026, 1, 1, tzinfo=UTC) + finished = datetime(2026, 1, 1, 0, 5, tzinfo=UTC) + + record = run.build_run_record( + challenge=challenge, + run_dir=run_dir, + run_id="xben-run-1", + model="anthropic/claude-sonnet-4-5", + mode="black-box", + started_at=started, + finished_at=finished, + ) + + assert record["challenge_id"] == "XBEN-001-24" + assert record["difficulty"] == "easy" + assert record["model"] == "anthropic/claude-sonnet-4-5" + assert record["mode"] == "black-box" + assert record["flag_matched"] is True + assert record["solved"] is True + assert record["duration_seconds"] == pytest.approx(300.0) + assert record["total_cost_usd"] == pytest.approx(2.5) + assert record["input_tokens"] == 1000 + assert record["output_tokens"] == 200 + assert record["run_id"] == "xben-run-1" + + def test_missing_run_json_raises(self, tmp_path: Path) -> None: + challenge = run.ChallengeSpec( + challenge_id="X", + challenge_name="X", + difficulty="easy", + target="https://target.example.com", + ) + with pytest.raises(FileNotFoundError): + run.build_run_record( + challenge=challenge, + run_dir=tmp_path, + run_id="run-1", + model="m", + mode="black-box", + started_at=datetime.now(tz=UTC), + finished_at=datetime.now(tz=UTC), + ) + + def test_unsolved_without_flag_uses_status(self, tmp_path: Path) -> None: + run_dir = tmp_path + (run_dir / "run.json").write_text(json.dumps({"status": "completed", "llm_usage": {}})) + challenge = run.ChallengeSpec( + challenge_id="X", + challenge_name="X", + difficulty="easy", + target="https://target.example.com", + flag=None, + ) + record = run.build_run_record( + challenge=challenge, + run_dir=run_dir, + run_id="run-1", + model="m", + mode="black-box", + started_at=datetime.now(tz=UTC), + finished_at=datetime.now(tz=UTC), + ) + assert record["flag_matched"] is False + assert record["solved"] is True # falls back to run status when there's no flag to check + + +class TestRunOneChallengeWithMockedSubprocess: + def test_invokes_subprocess_and_builds_record( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + challenge = run.ChallengeSpec( + challenge_id="XBEN-001-24", + challenge_name="Test", + difficulty="easy", + target="https://target.example.com", + flag="FLAG{yes}", + ) + + fake_result = subprocess.CompletedProcess( + args=["strix"], returncode=0, stdout="", stderr="" + ) + mock_run = MagicMock(return_value=fake_result) + monkeypatch.setattr(run, "run_strix_subprocess", mock_run) + + def fake_run_dir_for(run_name: str) -> Path: + run_dir = tmp_path / run_name + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text( + json.dumps({"status": "completed", "llm_usage": {"cost": 1.0}}), + ) + (run_dir / "vulnerabilities.json").write_text('[{"evidence": "FLAG{yes}"}]') + return run_dir + + args = run.build_parser().parse_args( + ["--manifest", "unused.json", "--model", "test-model"], + ) + + record = run.run_one_challenge( + challenge, + args=args, + env={}, + run_dir_for_name=fake_run_dir_for, + ) + + mock_run.assert_called_once() + called_command = mock_run.call_args.args[0] + assert called_command[:3] == ["strix", "--target", "https://target.example.com"] + assert record["challenge_id"] == "XBEN-001-24" + assert record["flag_matched"] is True + + +class TestMainDryRun: + def test_dry_run_prints_commands_without_running_strix( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps([_manifest_entry()])) + + mock_run = MagicMock() + monkeypatch.setattr(run, "run_strix_subprocess", mock_run) + + exit_code = run.main(["--manifest", str(manifest), "--model", "test-model", "--dry-run"]) + + assert exit_code == 0 + mock_run.assert_not_called() + captured = capsys.readouterr() + assert "XBEN-001-24" in captured.out + assert "--target" in captured.out + + def test_unknown_challenge_id_filter_errors( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps([_manifest_entry()])) + + exit_code = run.main( + [ + "--manifest", + str(manifest), + "--model", + "test-model", + "--challenge-id", + "does-not-exist", + "--dry-run", + ], + ) + assert exit_code == 1 + captured = capsys.readouterr() + assert "does-not-exist" in captured.err diff --git a/tests/test_xben_score.py b/tests/test_xben_score.py new file mode 100644 index 000000000..7b79da495 --- /dev/null +++ b/tests/test_xben_score.py @@ -0,0 +1,296 @@ +"""Tests for the XBEN benchmark scorer (benchmarks/xben/score.py).""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from benchmarks.xben import score + + +if TYPE_CHECKING: + from pathlib import Path + + +def _schema() -> dict[str, Any]: + return score.load_schema(score._DEFAULT_SCHEMA_PATH) + + +def _record(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "challenge_id": "XBEN-001-24", + "challenge_name": "Test Challenge", + "difficulty": "easy", + "strix_version": "0.4.0", + "model": "anthropic/claude-sonnet-4-5", + "mode": "black-box", + "solved": True, + "flag_matched": True, + "duration_seconds": 300, + "total_cost_usd": 1.5, + "input_tokens": 1000, + "output_tokens": 100, + "started_at": "2026-01-01T00:00:00Z", + "finished_at": "2026-01-01T00:05:00Z", + "run_id": "run-1", + } + base.update(overrides) + return base + + +class TestValidateRecord: + def test_valid_record_passes(self) -> None: + score.validate_record(_record(), _schema(), source="test.json") + + def test_missing_required_field_is_reported_by_name(self) -> None: + record = _record() + del record["challenge_id"] + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "bad.json" in str(exc_info.value) + assert "challenge_id" in str(exc_info.value) + + def test_invalid_enum_value_is_reported(self) -> None: + record = _record(difficulty="impossible") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "difficulty" in str(exc_info.value) + + def test_wrong_type_is_reported(self) -> None: + record = _record(solved="yes") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "solved" in str(exc_info.value) + + def test_negative_number_below_minimum_is_reported(self) -> None: + record = _record(duration_seconds=-1) + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "duration_seconds" in str(exc_info.value) + + def test_invalid_date_time_format_is_reported(self) -> None: + record = _record(started_at="not-a-date") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "started_at" in str(exc_info.value) + + def test_unexpected_field_is_reported(self) -> None: + record = _record(unknown_field="oops") + with pytest.raises(score.SchemaValidationError) as exc_info: + score.validate_record(record, _schema(), source="bad.json") + assert "unknown_field" in str(exc_info.value) + + def test_optional_notes_field_is_allowed(self) -> None: + score.validate_record(_record(notes="all good"), _schema(), source="test.json") + + def test_non_object_record_is_rejected(self) -> None: + with pytest.raises(score.SchemaValidationError): + score.validate_record(["not", "an", "object"], _schema(), source="bad.json") + + +class TestLoadResults: + def test_loads_valid_records(self, tmp_path: Path) -> None: + (tmp_path / "a.json").write_text(json.dumps(_record(challenge_id="A")), encoding="utf-8") + (tmp_path / "b.json").write_text(json.dumps(_record(challenge_id="B")), encoding="utf-8") + records = score.load_results(tmp_path, _schema()) + assert {r["challenge_id"] for r in records} == {"A", "B"} + + def test_missing_directory_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + score.load_results(tmp_path / "does-not-exist", _schema()) + + def test_empty_directory_raises(self, tmp_path: Path) -> None: + with pytest.raises(score.SchemaValidationError, match="no run-record"): + score.load_results(tmp_path, _schema()) + + def test_invalid_json_names_the_file(self, tmp_path: Path) -> None: + (tmp_path / "broken.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(score.SchemaValidationError, match=r"broken\.json"): + score.load_results(tmp_path, _schema()) + + def test_invalid_record_stops_loading_and_names_file(self, tmp_path: Path) -> None: + (tmp_path / "good.json").write_text(json.dumps(_record()), encoding="utf-8") + bad = _record() + del bad["model"] + (tmp_path / "bad.json").write_text(json.dumps(bad), encoding="utf-8") + with pytest.raises(score.SchemaValidationError, match=r"bad\.json"): + score.load_results(tmp_path, _schema()) + + def test_fixture_results_directory_is_valid(self) -> None: + # The committed example fixtures under benchmarks/xben/results/ must + # themselves validate cleanly -- this is what score.py --help's + # documented default invocation runs against. + records = score.load_results(score._DEFAULT_RESULTS_DIR, _schema()) + assert len(records) >= 1 + + +class TestAggregate: + def test_overall_success_rate(self) -> None: + records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="easy", solved=False), + ] + agg = score.aggregate(records) + assert agg.total_challenges == 2 + assert agg.solved_challenges == 1 + assert agg.success_rate == pytest.approx(50.0) + + def test_breakdown_by_difficulty(self) -> None: + records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="easy", solved=True), + _record(challenge_id="3", difficulty="hard", solved=False), + ] + agg = score.aggregate(records) + assert agg.by_difficulty["easy"].solved == 2 + assert agg.by_difficulty["easy"].total == 2 + assert agg.by_difficulty["hard"].solved == 0 + assert agg.by_difficulty["hard"].total == 1 + + def test_average_duration_and_cost(self) -> None: + records = [ + _record(challenge_id="1", duration_seconds=100, total_cost_usd=1.0), + _record(challenge_id="2", duration_seconds=300, total_cost_usd=3.0), + ] + agg = score.aggregate(records) + assert agg.avg_duration_seconds == pytest.approx(200.0) + assert agg.total_cost_usd == pytest.approx(4.0) + assert agg.avg_cost_usd == pytest.approx(2.0) + + def test_empty_aggregate_does_not_divide_by_zero(self) -> None: + agg = score.aggregate([]) + assert agg.success_rate == 0.0 + assert agg.avg_duration_seconds == 0.0 + assert agg.avg_cost_usd == 0.0 + + +class TestRenderMarkdown: + def test_contains_expected_headers_and_rows(self) -> None: + agg = score.aggregate( + [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="hard", solved=False), + ], + ) + markdown = score.render_markdown(agg) + assert "| Benchmark | Challenges | Success Rate |" in markdown + assert "| Difficulty | Solved | Success Rate |" in markdown + assert "Level 1 (Easy)" in markdown + assert "Level 3 (Hard)" in markdown + assert "Average solve time" in markdown + assert "Total cost" in markdown + + +class TestReadmeCheck: + _README_TEMPLATE = """\ +## Results + +| Benchmark | Challenges | Success Rate | +|-----------|------------|--------------| +| XBEN | {total} | **{rate:.0f}%** | + +**Performance by Difficulty:** + +| Difficulty | Solved | Success Rate | +|------------|--------|--------------| +| Level 1 (Easy) | {easy_solved}/{easy_total} | {easy_rate:.0f}% | +| Level 2 (Medium) | {medium_solved}/{medium_total} | {medium_rate:.0f}% | +| Level 3 (Hard) | {hard_solved}/{hard_total} | {hard_rate:.0f}% | +""" + + def _readme_matching(self, agg: score.AggregateResult) -> str: + easy = agg.by_difficulty.get("easy", score.DifficultyStats()) + medium = agg.by_difficulty.get("medium", score.DifficultyStats()) + hard = agg.by_difficulty.get("hard", score.DifficultyStats()) + return self._README_TEMPLATE.format( + total=agg.total_challenges, + rate=agg.success_rate, + easy_solved=easy.solved, + easy_total=easy.total, + easy_rate=easy.success_rate, + medium_solved=medium.solved, + medium_total=medium.total, + medium_rate=medium.success_rate, + hard_solved=hard.solved, + hard_total=hard.total, + hard_rate=hard.success_rate, + ) + + def test_matching_readme_has_no_mismatches(self) -> None: + records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="medium", solved=True), + _record(challenge_id="3", difficulty="hard", solved=False), + ] + agg = score.aggregate(records) + readme_text = self._readme_matching(agg) + claims = score.parse_readme_claims(readme_text) + assert score.diff_against_claims(agg, claims) == [] + + def test_mismatched_readme_is_reported(self) -> None: + records = [_record(challenge_id="1", difficulty="easy", solved=True)] + agg = score.aggregate(records) + readme_text = self._readme_matching(agg) + # Now aggregate a different set of records that disagrees. + other_records = [ + _record(challenge_id="1", difficulty="easy", solved=True), + _record(challenge_id="2", difficulty="easy", solved=False), + ] + other_agg = score.aggregate(other_records) + claims = score.parse_readme_claims(readme_text) + mismatches = score.diff_against_claims(other_agg, claims) + assert mismatches + assert any("total challenges" in m for m in mismatches) + + def test_real_readme_and_fixtures_are_out_of_sync(self) -> None: + # This is the exact situation documented in benchmarks/xben/results/README.md: + # the fixture data is synthetic and intentionally does not reproduce the + # real README's headline claim, so --check against it must fail. + agg = score.aggregate(score.load_results(score._DEFAULT_RESULTS_DIR, _schema())) + readme_text = score._DEFAULT_README_PATH.read_text(encoding="utf-8") + claims = score.parse_readme_claims(readme_text) + assert score.diff_against_claims(agg, claims) != [] + + +class TestCli: + def test_main_prints_table_to_stdout(self, capsys: pytest.CaptureFixture[str]) -> None: + exit_code = score.main([str(score._DEFAULT_RESULTS_DIR)]) + assert exit_code == 0 + captured = capsys.readouterr() + assert "Benchmark" in captured.out + + def test_main_writes_table_to_out_file(self, tmp_path: Path) -> None: + out_path = tmp_path / "table.md" + exit_code = score.main([str(score._DEFAULT_RESULTS_DIR), "--out", str(out_path)]) + assert exit_code == 0 + assert "Benchmark" in out_path.read_text(encoding="utf-8") + + def test_main_check_fails_against_real_readme_with_fixture_data(self) -> None: + exit_code = score.main([str(score._DEFAULT_RESULTS_DIR), "--check"]) + assert exit_code == 1 + + def test_main_check_passes_with_matching_readme(self, tmp_path: Path) -> None: + (tmp_path / "a.json").write_text( + json.dumps(_record(challenge_id="A", difficulty="easy", solved=True)), + encoding="utf-8", + ) + agg = score.aggregate([_record(challenge_id="A", difficulty="easy", solved=True)]) + readme_path = tmp_path / "README.md" + readme_path.write_text(TestReadmeCheck()._readme_matching(agg), encoding="utf-8") + exit_code = score.main( + [str(tmp_path), "--check", "--readme", str(readme_path)], + ) + assert exit_code == 0 + + def test_main_reports_invalid_record_and_exits_nonzero( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + bad = _record() + del bad["run_id"] + (tmp_path / "bad.json").write_text(json.dumps(bad), encoding="utf-8") + exit_code = score.main([str(tmp_path)]) + assert exit_code == 1 + captured = capsys.readouterr() + assert "bad.json" in captured.err From df37b2fafa34d49e7e44f353c7aab1fad7260f96 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:55:52 +0100 Subject: [PATCH 7/9] docs(benchmarks): point README at the new reproducible XBEN harness Note that the hand-typed XBEN numbers are pending re-verification through benchmarks/xben/, and add a "Reproducing this benchmark" section pointing at its README. Co-Authored-By: Claude Sonnet 5 --- benchmarks/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/benchmarks/README.md b/benchmarks/README.md index b7d99dece..e67f5d757 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -3,6 +3,23 @@ We use security benchmarks to track Strix's capabilities and improvements over time. We plan to add more benchmarks, both existing ones and our own, to help the community evaluate and compare security agents. +## Reproducing this benchmark + +The numbers below are pending re-verification through an in-repo, +reproducible harness: **[`benchmarks/xben/`](xben/README.md)**. That harness +is the source of truth going forward — it validates committed per-challenge +run records against a JSON Schema, mechanically regenerates the table below +from them, and has a `--check` mode that fails loudly if the committed data +doesn't back up the claim. This closes out the intent of +[usestrix/strix#1263](https://github.com/usestrix/strix/issues/1263) and +[usestrix/strix#1264](https://github.com/usestrix/strix/issues/1264), which +asked for exactly this: the headline number below currently has no committed +data reproducing it in this repository, only the hand-typed table that +follows and a link to an external repo. See `benchmarks/xben/README.md` for +how to run it and why `--check` currently (correctly) fails against the +fixture data checked in so far. + + ## Full Details For the complete benchmark results, evaluation scripts, and run data, see the [usestrix/benchmarks](https://github.com/usestrix/benchmarks) repository. @@ -13,6 +30,11 @@ For the complete benchmark results, evaluation scripts, and run data, see the [u ## Results +> [!NOTE] +> These numbers are hand-typed and **pending re-verification** via +> `benchmarks/xben/` (see above). Treat them as a historical claim, not yet +> as independently reproducible from data committed to this repository. + | Benchmark | Challenges | Success Rate | |-----------|------------|--------------| | [XBEN](https://github.com/usestrix/benchmarks/tree/main/XBEN) | 104 | **96%** | From bf5ae1651692b15be16fe30d3c5b3393f5b8ed03 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:57:44 +0100 Subject: [PATCH 8/9] feat(agents): allow create_agent to spawn a child on a different model Add an optional model override to create_agent so the orchestrator can deliberately spawn a "second opinion" specialist on a different LLM to re-review surfaces the primary model already covered (e.g. needs_follow_up coverage entries, or an under-covered target). The override threads create_agent -> the spawner callback -> AgentCoordinator.register (persisted for resume) -> build_strix_agent's SandboxAgent(model=...), falling back to the run-wide model when omitted; ReportUsageHooks now records/estimates each agent's cost against its own resolved model instead of always the run's default. The model string is validated with a new shared strix.config.models.invalid_model_reason() before spawning, returning a clean {"success": False, "error": ...} on a bad string instead of a crash. Also adds the scan_modes/second_opinion skill documenting when and how to use this, and a short pointer to it in coordination/root_agent.md. No automatic/heuristic triggering is added -- this is purely an explicit, deliberately-invoked capability. Co-Authored-By: Claude Sonnet 5 --- strix/agents/factory.py | 12 +- strix/config/models.py | 23 ++++ strix/core/agents.py | 2 + strix/core/execution.py | 15 ++- strix/core/hooks.py | 5 +- strix/skills/coordination/root_agent.md | 4 + strix/skills/scan_modes/second_opinion.md | 44 +++++++ strix/tools/agents_graph/tools.py | 25 ++++ tests/test_create_agent_model_override.py | 143 ++++++++++++++++++++++ 9 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 strix/skills/scan_modes/second_opinion.md create mode 100644 tests/test_create_agent_model_override.py diff --git a/strix/agents/factory.py b/strix/agents/factory.py index b2fcbf08c..16db70ebd 100644 --- a/strix/agents/factory.py +++ b/strix/agents/factory.py @@ -658,6 +658,7 @@ def build_strix_agent( system_prompt_context: dict[str, Any] | None = None, extra_tools: Sequence[Tool] | None = None, instructions_override: str | None = None, + model: str | None = None, ) -> SandboxAgent[Any]: """Build a SandboxAgent for either root or child use. @@ -670,6 +671,12 @@ def build_strix_agent( registered via ``register_agent_tools``. instructions_override: Use this verbatim as the system prompt instead of rendering the built-in scan prompt. + model: Per-agent model override. ``None`` (the default) leaves the + agent's own ``model`` unset, so it runs on the run-wide model + configured on ``RunConfig``. A non-``None`` value overrides that + for this agent only — every other agent in the scan is + unaffected. Used for deliberately spawning a "second opinion" + child on a different model; see ``strix.tools.agents_graph.tools.create_agent``. """ if instructions_override is not None: instructions = instructions_override @@ -715,7 +722,7 @@ def build_strix_agent( instructions=instructions, tools=tools, tool_use_behavior=_finish_tool_use_behavior, - model=None, + model=model, capabilities=[ Filesystem( configure_tools=_make_filesystem_configurator( @@ -750,7 +757,7 @@ def make_child_factory( without the graph tool knowing about runner internals. """ - def _factory(*, name: str, skills: list[str]) -> SandboxAgent[Any]: + def _factory(*, name: str, skills: list[str], model: str | None = None) -> SandboxAgent[Any]: return build_strix_agent( name=name, skills=skills, @@ -762,6 +769,7 @@ def make_child_factory( chat_completions_tools=chat_completions_tools, strict_tool_schemas=strict_tool_schemas, system_prompt_context=system_prompt_context, + model=model, ) return _factory diff --git a/strix/config/models.py b/strix/config/models.py index 3babbe37b..8dca45461 100644 --- a/strix/config/models.py +++ b/strix/config/models.py @@ -887,6 +887,29 @@ def is_known_openai_bare_model(model_name: str) -> bool: return bool(entry and entry.get("litellm_provider") == "openai") +def invalid_model_reason(model_name: str | None, *, api_base: str | None = None) -> str | None: + """Return why ``model_name`` is not a usable model string, or ``None`` if it is fine. + + Mirrors the CLI's hard-fail check in ``warm_up_llm`` (``strix.interface.main``): + empty is always invalid, and a bare name (no ``/`` prefix) must be a + known OpenAI model unless a custom ``LLM_API_BASE``/``api_base`` is configured, + since that could plausibly serve any bare name. This is a structural check only + — it says nothing about whether the model is a *good* choice (see + ``is_recommended_or_frontier_model`` for that, which is a warning, not a + rejection). + """ + name = (model_name or "").strip() + if not name: + return "model name is empty" + if "/" not in name and not is_known_openai_bare_model(name) and not api_base: + return ( + f"'{name}' is not a known OpenAI model. Bare names route to OpenAI by " + "default. If you meant a non-OpenAI provider, use the '/' " + "form, e.g. 'anthropic/claude-opus-4-7', 'deepseek/deepseek-v4-pro'." + ) + return None + + _ANTHROPIC_MODEL_MARKERS = ("anthropic", "claude", "sonnet", "opus", "haiku") diff --git a/strix/core/agents.py b/strix/core/agents.py index 4d3d65cc6..7afc36edf 100644 --- a/strix/core/agents.py +++ b/strix/core/agents.py @@ -160,6 +160,7 @@ class AgentCoordinator: *, task: str | None = None, skills: list[str] | None = None, + model: str | None = None, ) -> None: async with self._lock: self.statuses[agent_id] = "running" @@ -169,6 +170,7 @@ class AgentCoordinator: self.metadata[agent_id] = { "task": task or "", "skills": list(skills or []), + "model": model, } self.runtimes.setdefault(agent_id, AgentRuntime()) logger.info("agent.register %s (%s) parent=%s", agent_id, name, parent_id or "-") diff --git a/strix/core/execution.py b/strix/core/execution.py index dfcd39fa2..2ac115082 100644 --- a/strix/core/execution.py +++ b/strix/core/execution.py @@ -315,6 +315,7 @@ async def spawn_child_agent( task: str, skills: list[str], parent_history: list[Any], + model: str | None = None, event_sink: StreamEventSink | None = None, hooks: RunHooks[dict[str, Any]] | None = None, ) -> dict[str, Any]: @@ -323,13 +324,14 @@ async def spawn_child_agent( raise TypeError("Parent agent_id missing from context") child_id = uuid.uuid4().hex[:8] - child_agent = factory(name=name, skills=skills) + child_agent = factory(name=name, skills=skills, model=model) await coordinator.register( child_id, name, parent_id, task=task, skills=skills, + model=model, ) await _start_child_runner( @@ -345,6 +347,7 @@ async def spawn_child_agent( name=name, parent_id=parent_id, task=task, + model=model, initial_input=child_initial_input( name=name, child_id=child_id, @@ -414,7 +417,8 @@ async def respawn_subagents( ) child_skills = list(md.get("skills") or []) - child_agent = factory(name=name, skills=child_skills) + child_model = md.get("model") if isinstance(md.get("model"), str) else None + child_agent = factory(name=name, skills=child_skills, model=child_model) await _start_child_runner( parent_ctx=parent_ctx, coordinator=coordinator, @@ -428,6 +432,7 @@ async def respawn_subagents( name=name, parent_id=parent_id, task=str(md.get("task", "")), + model=child_model, initial_input=[], start_parked=start_parked, event_sink=event_sink, @@ -1001,6 +1006,7 @@ async def _start_child_runner( parent_id: str | None, task: str, initial_input: Any, + model: str | None = None, start_parked: bool = False, event_sink: StreamEventSink | None = None, hooks: RunHooks[dict[str, Any]] | None = None, @@ -1013,6 +1019,11 @@ async def _start_child_runner( child_ctx["agent_id"] = child_id child_ctx["parent_id"] = parent_id child_ctx["task"] = task + if model is not None: + # Lets ReportUsageHooks record/estimate this agent's cost against the + # model it actually runs on, not the run-wide default — see + # ``build_strix_agent``'s ``model`` override. + child_ctx["agent_model"] = model async def _child_loop() -> None: # A budget stop is a clean scan-wide shutdown, not a child failure: the diff --git a/strix/core/hooks.py b/strix/core/hooks.py index 21400c0b4..62f86db96 100644 --- a/strix/core/hooks.py +++ b/strix/core/hooks.py @@ -240,11 +240,14 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]): if not isinstance(agent_id, str) or not agent_id: agent_id = agent_name or "unknown" + agent_model = ctx.get("agent_model") + model = agent_model if isinstance(agent_model, str) and agent_model else self._model + try: report_state.record_sdk_usage( agent_id=agent_id, agent_name=agent_name, - model=self._model, + model=model, usage=response.usage, ) except Exception: diff --git a/strix/skills/coordination/root_agent.md b/strix/skills/coordination/root_agent.md index fa88aaa5a..97326ec5e 100644 --- a/strix/skills/coordination/root_agent.md +++ b/strix/skills/coordination/root_agent.md @@ -95,6 +95,10 @@ Complex findings warrant specialized subagents: - Use message passing only when essential (requests/answers, critical handoffs) - Prefer batched updates over routine status messages +**Second Opinions** + +Every agent runs on the scan's configured model by default, but `create_agent` accepts an optional `model` override for the rare case where that isn't enough: deliberately re-reviewing already-covered surfaces (piling-up `needs_follow_up` entries, or a target under-covered relative to its size) with a differently-configured model, since different model families notice different things on the same input. This is not a general-purpose "pick a model for this agent" knob and nothing spawns it automatically — see the `scan_modes/second_opinion` skill before reaching for it. + ## Completion When all agents report completion: diff --git a/strix/skills/scan_modes/second_opinion.md b/strix/skills/scan_modes/second_opinion.md new file mode 100644 index 000000000..979a07ca0 --- /dev/null +++ b/strix/skills/scan_modes/second_opinion.md @@ -0,0 +1,44 @@ +--- +name: second-opinion +description: Deliberately re-review already-covered surfaces with a differently-configured model +--- + +# Second Opinion + +Every agent in a scan runs on the same model by default. That is the right default — it is predictable and it is what the operator configured — but different model families notice different things on the same input. A pass that already happened is not necessarily a pass that was thorough; a second, independent look from a different model can surface what the first one missed. This is a deliberate, operator-triggered capability, not something that fires on its own — nothing in Strix spawns a second-opinion agent automatically, and this skill does not ask you to either. + +## When to use this + +Reach for a second opinion when coverage on a scan that matters is thin or uncertain and there is still budget to spend on it: + +- `list_coverage(outcome="needs_follow_up")` is non-empty and piling up faster than it is being resolved — entries sitting open across multiple agents rather than one agent's temporary backlog. +- A target is large or high-value relative to how much of it has actually been exercised — e.g. a handful of endpoints marked `covered` on an API surface with dozens of routes. +- The scan is close to wrapping up and you (the root agent) judge that a fresh pass over what has already been reviewed is worth more than starting a new area from scratch. + +Do not reach for this to cover new ground — that is what an ordinary specialist agent is for. This is specifically for **re-examining surfaces someone already looked at**, with a model that might notice something the first pass didn't. + +Do not reach for this reflexively on every scan — it is another full agent's worth of tokens and time on top of the primary pass, and it needs a real driver (thin coverage on something that matters, not curiosity). + +## How to use this + +1. Call `list_coverage(outcome="needs_follow_up")` (and, if useful, look at what is marked `covered` on the target you're worried about) to build a concrete list of surfaces to re-check. A second opinion is only as good as the list of what it's being asked to re-examine — don't hand it a vague "look at the app again." +2. Call `create_agent` with `model` set to a different model string than the one this scan is running on, and a `task` that says explicitly this is a re-review, not fresh recon: + + ``` + create_agent( + name="Second Opinion: Auth Surface", + task=( + "Re-review these already-covered surfaces with fresh eyes: " + ". " + "The first pass marked these as needs_follow_up / covered but " + "coverage here is thin relative to the surface's size. Look for " + "anything the first pass could have missed, not new areas." + ), + model="anthropic/claude-opus-4-7", + skills=["auth"], + ) + ``` + +3. Treat its findings like any other subagent's: filed vulnerability reports and `update_coverage` calls on the entries it resolved. It reports back through the same `agent_finish` / completion-report flow as every other child. + +`model` accepts the same `/` strings as `STRIX_LLM` (e.g. `anthropic/claude-opus-4-7`, `openai/gpt-5.6`, `zai/glm-5.3`). Pick something meaningfully different from the scan's configured model — a second pass on the same model tends to repeat the same blind spots. diff --git a/strix/tools/agents_graph/tools.py b/strix/tools/agents_graph/tools.py index da05bb1e7..899fcad02 100644 --- a/strix/tools/agents_graph/tools.py +++ b/strix/tools/agents_graph/tools.py @@ -12,6 +12,8 @@ from typing import Any, Literal, get_args from agents import RunContextWrapper, function_tool +from strix.config import load_settings +from strix.config.models import invalid_model_reason from strix.core.agents import Status, coordinator_from_context from strix.core.execution import notify_parent_on_terminal from strix.core.hooks import LLM_TURN_KEY @@ -501,6 +503,7 @@ async def create_agent( task: str, inherit_context: bool = True, skills: list[str] | None = None, + model: str | None = None, ) -> str: """Spawn a specialist child agent to run in parallel. @@ -546,6 +549,17 @@ async def create_agent( when starting a clean-slate task. skills: List of skill names (e.g. ``["xss", "sql_injection"]``). Max 5; prefer 1-3. + model: Optional. Leave unset — every child runs on the scan's + configured model by default. Set this only to deliberately get a + **second opinion** from a differently-configured model on + surfaces the scan has already reviewed (e.g. entries sitting at + ``needs_follow_up`` in ``list_coverage``, or a target you judge + under-covered relative to its size). This is not a general + "pick any model for this agent" knob — different model + families catch different things, so a second, independent pass + with fresh eyes can surface what the first pass missed. Accepts + the same ``/`` strings as ``STRIX_LLM`` + (e.g. ``"anthropic/claude-opus-4-7"``). """ inner = _ctx(ctx) coordinator = coordinator_from_context(inner) @@ -577,6 +591,16 @@ async def create_agent( default=str, ) + if model is not None: + api_base = load_settings().llm.api_base + model_error = invalid_model_reason(model, api_base=api_base) + if model_error: + return json.dumps( + {"success": False, "error": f"invalid model: {model_error}", "agent_id": None}, + ensure_ascii=False, + default=str, + ) + parent_history = list(ctx.turn_input) if inherit_context and ctx.turn_input else [] try: result = await spawner( @@ -585,6 +609,7 @@ async def create_agent( task=task, skills=skill_list, parent_history=parent_history, + model=model, ) except Exception as e: logger.exception("create_agent: scan runner failed to spawn child '%s'", name) diff --git a/tests/test_create_agent_model_override.py b/tests/test_create_agent_model_override.py new file mode 100644 index 000000000..33a60934f --- /dev/null +++ b/tests/test_create_agent_model_override.py @@ -0,0 +1,143 @@ +"""Tests for the optional per-agent ``model`` override on ``create_agent``. + +``create_agent`` normally spawns children on the scan's configured model. +``model`` lets the caller deliberately give one child a different model (a +"second opinion" specialist) without touching every other agent in the run. +These tests pin down the three contract points: omitting ``model`` must be a +no-op regression guard, a valid alternate model string must be threaded +through to the spawner unchanged, and an invalid model string must fail +cleanly instead of spawning anything or raising. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, cast + +import pytest +from agents.tool_context import ToolContext + +import strix.tools.agents_graph.tools as agents_graph_tools +from strix.core.agents import AgentCoordinator +from strix.tools.agents_graph.tools import create_agent + + +if TYPE_CHECKING: + from collections.abc import Iterator + + +class _FakeLlmSettings: + def __init__(self, api_base: str | None = None) -> None: + self.api_base = api_base + + +class _FakeSettings: + def __init__(self, api_base: str | None = None) -> None: + self.llm = _FakeLlmSettings(api_base=api_base) + + +@pytest.fixture +def _no_api_base(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Pin ``load_settings()`` so the bare-name validation path is deterministic.""" + + def _load_fake_settings() -> _FakeSettings: + return _FakeSettings() + + monkeypatch.setattr(agents_graph_tools, "load_settings", _load_fake_settings) + yield + + +async def _graph() -> AgentCoordinator: + coordinator = AgentCoordinator() + await coordinator.register("root", "strix", parent_id=None) + await coordinator.attach_runtime("root", resumable=False) + return coordinator + + +class _RecordingSpawner: + """Stand-in for the ``spawn_child_agent`` closure the runner puts in context.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + async def __call__(self, **kwargs: Any) -> dict[str, Any]: + self.calls.append(kwargs) + return { + "success": True, + "agent_id": "child-1", + "name": kwargs.get("name"), + "parent_id": kwargs.get("parent_ctx", {}).get("agent_id"), + "message": "spawned", + } + + +async def _call_create_agent( + coordinator: AgentCoordinator, spawner: _RecordingSpawner, args: dict[str, Any] +) -> dict[str, Any]: + ctx = ToolContext( + context={ + "coordinator": coordinator, + "agent_id": "root", + "spawn_child_agent": spawner, + }, + tool_name=create_agent.name, + tool_call_id="call-1", + tool_arguments="{}", + ) + raw: str = await create_agent.on_invoke_tool(ctx, json.dumps(args)) + return cast("dict[str, Any]", json.loads(raw)) + + +@pytest.mark.asyncio +async def test_omitting_model_spawns_exactly_as_before(_no_api_base: None) -> None: + coordinator = await _graph() + spawner = _RecordingSpawner() + + result = await _call_create_agent( + coordinator, spawner, {"name": "Validator", "task": "check the thing"} + ) + + assert result["success"] is True + assert len(spawner.calls) == 1 + assert spawner.calls[0]["model"] is None + + +@pytest.mark.asyncio +async def test_valid_alternate_model_is_threaded_through_unchanged(_no_api_base: None) -> None: + coordinator = await _graph() + spawner = _RecordingSpawner() + + result = await _call_create_agent( + coordinator, + spawner, + { + "name": "Second Opinion", + "task": "re-review already-covered surfaces", + "model": "anthropic/claude-opus-4-7", + }, + ) + + assert result["success"] is True + assert len(spawner.calls) == 1 + assert spawner.calls[0]["model"] == "anthropic/claude-opus-4-7" + + +@pytest.mark.asyncio +async def test_invalid_model_string_returns_a_clean_error(_no_api_base: None) -> None: + coordinator = await _graph() + spawner = _RecordingSpawner() + + result = await _call_create_agent( + coordinator, + spawner, + { + "name": "Second Opinion", + "task": "re-review already-covered surfaces", + "model": "definitely-not-a-real-model", + }, + ) + + assert result["success"] is False + assert "invalid model" in result["error"] + assert result["agent_id"] is None + assert spawner.calls == [] From eecfa5ae81bfb756df24387d949b8d7eebbd6eba Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 12 Sep 2026 16:58:24 +0100 Subject: [PATCH 9/9] chore: ignore agent worktree metadata dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 89db2d7ed..470b9398e 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,4 @@ schema.graphql /.benchmarks/ /references/ /strix_runs_main/ +.claude/worktrees/