feat(agents): let callers choose the report tool (#1550)

* feat(agents): let callers remove base tools from scan agents

* feat(agents): let callers choose the report tool instead of removing tools

* refactor(agents): name the report tool via the prompt template, not string replacement
This commit is contained in:
ian-at-strix 2026-10-20 20:21:31 -04:00 • committed by GitHub
parent aa2036f229
commit 3a3855fc4f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 39 additions and 11 deletions

View file

@ -724,6 +724,7 @@ def build_strix_agent(
instructions_override: str | None = None,
supports_images: bool = True,
finish_tool: Tool = finish_scan,
report_tool: Tool = create_vulnerability_report,
) -> SandboxAgent[Any]:
"""Build a SandboxAgent for either root or child use.
@ -737,6 +738,8 @@ def build_strix_agent(
instructions_override: Use this verbatim as the system prompt instead
of rendering the built-in scan prompt.
finish_tool: The tool that ends the run, given to the root agent.
report_tool: The tool every agent files findings with, in place of
``create_vulnerability_report``.
"""
if instructions_override is not None:
instructions = instructions_override
@ -750,16 +753,18 @@ def build_strix_agent(
interactive=interactive,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
report_tool=report_tool.name,
)
agent_tools = [*_EXTRA_TOOLS, *(extra_tools or [])]
if interactive:
# Yielding to the user is only meaningful when one is attached.
agent_tools.append(wait_for_user)
base_tools = [report_tool if t is create_vulnerability_report else t for t in _BASE_TOOLS]
if is_root:
tools: list[Tool] = [*_BASE_TOOLS, *agent_tools, finish_tool]
tools: list[Tool] = [*base_tools, *agent_tools, finish_tool]
else:
tools = [*_BASE_TOOLS, *agent_tools, agent_finish]
tools = [*base_tools, *agent_tools, agent_finish]
_ensure_unique_tool_names(tools)
tools = [
_with_bounded_result(_with_strictness(_with_coerced_arguments(tool), strict_tool_schemas))
@ -812,6 +817,7 @@ def make_child_factory(
strict_tool_schemas: bool = True,
system_prompt_context: dict[str, Any] | None = None,
supports_images: bool = True,
report_tool: Tool = create_vulnerability_report,
) -> Any:
"""Return the runner-owned builder used by ``spawn_child_agent``.
@ -833,6 +839,7 @@ def make_child_factory(
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=system_prompt_context,
supports_images=supports_images,
report_tool=report_tool,
)
return _factory

View file

@ -86,6 +86,7 @@ def render_system_prompt(
system_prompt_context: dict[str, Any] | None = None,
include_scope: bool = True,
supports_images: bool = True,
report_tool: str = "create_vulnerability_report",
) -> str:
"""Render the system prompt. Returns empty string on template failure.
@ -133,6 +134,7 @@ def render_system_prompt(
is_root=is_root,
system_prompt_context=system_prompt_context or {},
include_scope=include_scope,
report_tool=report_tool,
**skill_content,
)
except Exception:

View file

@ -119,7 +119,7 @@ WHITE-BOX TESTING (code provided):
- Local execution, unit/integration testing, patch verification, and HTTP requests against locally started in-scope services are normal authorized white-box validation
- If dynamically running the code proves impossible after exhaustive attempts, pivot to comprehensive static analysis.
- Try to infer how to run the code based on its structure and content.
- Derive the code fix as PART OF reporting, not as a separate later pass: create_vulnerability_report already requires the concrete patch inline (`code_locations` with verbatim `fix_before`/`fix_after` and `fix_pr_body`), so the reporting agent that analyzes the root cause is the one that produces the fix. Do NOT spawn a downstream agent afterwards to re-derive/re-apply the same patch.
- Derive the code fix as PART OF reporting, not as a separate later pass: {{ report_tool }} already requires the concrete patch inline (`code_locations` with verbatim `fix_before`/`fix_after` and `fix_pr_body`), so the reporting agent that analyzes the root cause is the one that produces the fix. Do NOT spawn a downstream agent afterwards to re-derive/re-apply the same patch.
- If you also apply and verify the patch in the repo (edit the file, re-test that the vulnerability is gone), do it in the same agent/turn while the analysis is fresh — right before or as part of filing the report — never as a second re-analysis pass.
COMBINED MODE (code + deployed target present):
@ -205,9 +205,9 @@ VALIDATION REQUIREMENTS:
- COVERAGE: record every surface you assess with `record_coverage` (surface + risk area + outcome + evidence), including the ones that came back clean — a report that only lists findings cannot say what was reviewed and cleared. Use the `needs_follow_up` outcome for anything left in an `open_proof_gap` state, and carry the same items up in `agent_finish(open_items=[...])`. The ledger is shared and mutable: when you resolve a surface another agent left open — or find that a closed one is not — move that entry with `update_coverage` instead of recording a second one for the same surface. The root agent reconciles all of it via `list_coverage` before `finish_scan`.
- THREAT MODEL: before you start testing, call `get_threat_model` on the target you were pointed at — it is the scan's shared answer to who the attacker is, where the trust boundaries sit, and what counts as critical here. It is scoped to this scan and nothing carries over from an earlier run, so `found: false` means no agent on this run has derived one yet. Read it instead of re-deriving trust boundaries yourself; where your testing disproves it — a boundary it calls trusted turns out to be attacker-reachable, a role it did not know about, a host or endpoint it never listed — record that with `amend_threat_model` so the agents after you inherit the correction. Amending is not optional politeness: a model nobody corrects turns the first agent's guesses into everyone's assumptions.
- Before filing any report, run the counterevidence pass: argue the strongest case AGAINST the finding, record what you found in the `counterevidence` field, set `confidence` honestly (a static-only trace you couldn't execute is at best `medium`), and state what evidence would change the severity. See the counterevidence and severity-calibration knowledge above.
- A vulnerability is ONLY considered reported when a reporting agent uses create_vulnerability_report (or create_dependency_report for known-CVE dependency/supply-chain findings) with full details. Mentions in agent_finish, finish_scan, or generic messages are NOT sufficient
- Reporting and fixing are ONE step, not two: when source is available, the reporting agent derives the concrete fix and files it INLINE via create_vulnerability_report (`code_locations` with `fix_before`/`fix_after` + `fix_pr_body`) — the report is not complete without it. Do NOT report first and then spawn a separate downstream agent to re-derive and re-apply the same patch; that just re-does the analysis and wastes tokens. (Do not silently patch a finding WITHOUT filing a report — the report, with its embedded fix, is the deliverable.)
- DEDUPLICATION: The create_vulnerability_report tool uses LLM-based deduplication. If it rejects your report as a duplicate, DO NOT attempt to re-submit the same vulnerability. Accept the rejection and move on to testing other areas. The vulnerability has already been reported by another agent. If your evidence proves more than the finding it matched (a working exploit where that one had only a static trace, a chain that raises the impact), revise that finding with update_vulnerability_report using the duplicate_of id — never re-file it.
- A vulnerability is ONLY considered reported when a reporting agent uses {{ report_tool }} (or create_dependency_report for known-CVE dependency/supply-chain findings) with full details. Mentions in agent_finish, finish_scan, or generic messages are NOT sufficient
- Reporting and fixing are ONE step, not two: when source is available, the reporting agent derives the concrete fix and files it INLINE via {{ report_tool }} (`code_locations` with `fix_before`/`fix_after` + `fix_pr_body`) — the report is not complete without it. Do NOT report first and then spawn a separate downstream agent to re-derive and re-apply the same patch; that just re-does the analysis and wastes tokens. (Do not silently patch a finding WITHOUT filing a report — the report, with its embedded fix, is the deliverable.)
- DEDUPLICATION: The {{ report_tool }} tool uses LLM-based deduplication. If it rejects your report as a duplicate, DO NOT attempt to re-submit the same vulnerability. Accept the rejection and move on to testing other areas. The vulnerability has already been reported by another agent. If your evidence proves more than the finding it matched (a working exploit where that one had only a static trace, a chain that raises the impact), revise that finding with update_vulnerability_report using the duplicate_of id — never re-file it.
- HTTP EVIDENCE: a finding you validated through the proxy is not fully filed until `http_exchange_ids` carries the proxy request ids of the exchanges that prove it — the request that triggers the vulnerability plus the baseline/control request it differs from (an unauthenticated success next to the authenticated one, the payload response next to the benign one). Copy the ids exactly as `list_requests`/`view_request` show them, never invent or guess one, and never omit the field to bypass validation. Leave it out only when there is no captured HTTP exchange at all (static-only code findings, dependency CVEs). If you filed before the proving exchanges existed, attach them afterwards with update_vulnerability_report. Without the ids, the finding ships as prose nobody can replay.
- REVISING A FINDING: use update_vulnerability_report (report id + the fields you want to replace + update_reason) when you learn something a finding already on file does not carry — you built the PoC after filing it, a chain raised its impact, further testing weakened it, or its counterevidence/remediation/code locations were wrong. Editing a finding needs no duplicate verdict, and it is always better than filing a second report for the same issue. Read the finding first with get_report, and pass only the fields that change.
- WITHDRAWING A FINDING: when re-testing proves a finding you filed is not a vulnerability at all — the exploit only worked because of your own test setup (a mixed-up session, a self-inflicted state change, a misread response), the behaviour is intended, or the control is in fact enforced — delete it with delete_vulnerability_report (report id + delete_reason). Never leave a disproved finding on file as a zero-CVSS, "retracted", or "false positive" report, and never rewrite its text into a note saying it was wrong: a report that exists is read as a finding. Delete is for a finding that does not exist; a finding that is real but weaker than filed is revised, not deleted.
@ -227,7 +227,7 @@ Every one of these tools writes to state the rest of the scan reads. Reaching fo
- STEER CHILDREN — `send_message_to_agent`: send a running child new information, a course correction, or a request to wrap up, without killing it. Use it to answer a child's question or narrow its scope mid-run.
- BLOCK ON CHILDREN — `wait_for_agents`: block until named children report back when your next move genuinely depends on their results. If you can keep making progress in parallel, keep working instead of waiting.
- CANCEL CHILDREN — `stop_agent`: gracefully cancel a child whose work is redundant, misdirected, or no longer needed. Prefer `send_message_to_agent` to redirect a child that is merely off-track; reserve `stop_agent` for work that should not continue at all.
- FINISH — subagents call `agent_finish` (with `open_items=[...]` for anything left unresolved); the root agent calls `finish_scan` exactly once, only after every child is wrapped up and coverage is reconciled. `agent_finish`/`finish_scan` are handoffs, not reporting channels — a vulnerability is reported only via `create_vulnerability_report`/`create_dependency_report`.
- FINISH — subagents call `agent_finish` (with `open_items=[...]` for anything left unresolved); the root agent calls `finish_scan` exactly once, only after every child is wrapped up and coverage is reconciled. `agent_finish`/`finish_scan` are handoffs, not reporting channels — a vulnerability is reported only via `{{ report_tool }}`/`create_dependency_report`.
</execution_guidelines>
<vulnerability_focus>
@ -380,7 +380,7 @@ CRITICAL RULES:
- **REALISTIC OUTCOMES** - Some tests find nothing, some validations fail
- **ONE AGENT = ONE TASK** - Don't let agents do multiple unrelated jobs
- **SPAWN REACTIVELY** - Create new agents based on what you discover
- **ONLY REPORTING AGENTS** can use create_vulnerability_report tool
- **ONLY REPORTING AGENTS** can use {{ report_tool }} tool
- **AGENT SPECIALIZATION MANDATORY** - Each agent must be highly specialized; prefer 1–3 skills, up to 5 for complex contexts
- **NO GENERIC AGENTS** - Avoid creating broad, multi-purpose agents that dilute focus

View file

@ -54,6 +54,7 @@ from strix.tools.output_store import (
WORKSPACE_SPILL_DIR,
configure_spill_writer,
)
from strix.tools.reporting.tool import create_vulnerability_report
if TYPE_CHECKING:
@ -172,6 +173,7 @@ def _compose_root_instructions_override(
interactive: bool,
system_prompt_context: dict[str, Any],
supports_images: bool,
report_tool: str,
) -> str | None:
if root_instructions_override is None:
return None
@ -186,6 +188,7 @@ def _compose_root_instructions_override(
system_prompt_context=system_prompt_context,
include_scope=False,
supports_images=supports_images,
report_tool=report_tool,
)
return (
f"{base_instructions}\n\n"
@ -218,6 +221,7 @@ async def run_strix_scan(
mcp_connection_requests: list[McpConnectionRequest] | None = None,
mcp_status_sink: McpStatusSink | None = None,
root_finish_tool: Tool = finish_scan,
report_tool: Tool = create_vulnerability_report,
) -> RunResultBase | None:
"""Run or resume one Strix scan against a sandbox.
@ -241,6 +245,8 @@ async def run_strix_scan(
way the engine does the connecting, so the caller passes inert configs plus
metadata and never live sessions.
``root_finish_tool`` is the tool the root agent ends the run with.
``report_tool`` is the tool every agent files findings with, in place of
``create_vulnerability_report``.
"""
def report(phase: str) -> None:
@ -495,6 +501,7 @@ async def run_strix_scan(
interactive=interactive,
system_prompt_context=root_context,
supports_images=supports_images,
report_tool=report_tool.name,
)
root_agent = build_strix_agent(
@ -511,6 +518,7 @@ async def run_strix_scan(
instructions_override=root_instructions,
supports_images=supports_images,
finish_tool=root_finish_tool,
report_tool=report_tool,
)
if not is_resume:
@ -531,6 +539,7 @@ async def run_strix_scan(
strict_tool_schemas=strict_tool_schemas,
system_prompt_context=scope_context,
supports_images=supports_images,
report_tool=report_tool,
)
async def spawn_child_agent(**kwargs: Any) -> dict[str, Any]:

View file

@ -20,7 +20,7 @@ state, and "I moved on" is not one of them.
**1. `confirmed`** — you have a working PoC or, in white-box, a complete
source → control → sink → impact trace plus evidence the path is
reachable. File it with `create_vulnerability_report`.
reachable. File it as a vulnerability report.
**2. `ruled_out`** — you can name the **specific control** that makes the
code safe, at a specific location, and you have checked that the control
@ -144,8 +144,7 @@ endpoint is broken/unreachable for unrelated reasons".
## Before You File a Report
Run this pass on every finding before calling
`create_vulnerability_report`:
Run this pass on every finding before filing it:
1. **Argue the other side.** Spend real effort building the strongest
case that this is *not* exploitable, or not as severe as you think.

View file

@ -93,6 +93,17 @@ def test_per_call_extra_tools_reject_duplicate_registered_names() -> None:
factory.build_strix_agent(is_root=True, extra_tools=[_tool("same_name")])
def test_report_tool_replaces_create_vulnerability_report() -> None:
for is_root in (True, False):
agent = factory.build_strix_agent(is_root=is_root, report_tool=_tool("file_pr_finding"))
names = [t.name for t in agent.tools]
assert "file_pr_finding" in names
assert "create_vulnerability_report" not in names
assert isinstance(agent.instructions, str)
assert "file_pr_finding" in agent.instructions
assert "create_vulnerability_report" not in agent.instructions
def test_instructions_override_is_used_verbatim() -> None:
custom = "You are a scan agent. Follow the provided scope."