From 760803f9839965f7df3ae45d0cb85aeb51cc3cf2 Mon Sep 17 00:00:00 2001 From: VoidChecksum Date: Tue, 9 Jun 2026 00:38:48 +0000 Subject: [PATCH] fix(reporting): drop fabricated code_locations in black-box scans In a black-box scan no source tree is available, yet create_vulnerability_report accepted code_locations unconditionally and render_vulnerability_md emitted a "Code Analysis" section from them. The model could therefore fabricate file paths, line numbers, and snippets into the customer-facing report (#321). Thread the existing is_whitebox flag into the tool run-context (it propagates to child agents via dict(parent_ctx)) and drop code_locations when the scan is black-box. Also add black-box guidance to the reporting tool docstring and the system prompt so the model does not assert source locations it cannot see. Fixes #321 --- strix/agents/prompts/system_prompt.jinja | 1 + strix/core/runner.py | 1 + strix/tools/reporting/tool.py | 13 +++++++++++++ 3 files changed, 15 insertions(+) diff --git a/strix/agents/prompts/system_prompt.jinja b/strix/agents/prompts/system_prompt.jinja index 4a2f55c2..d3f69f76 100644 --- a/strix/agents/prompts/system_prompt.jinja +++ b/strix/agents/prompts/system_prompt.jinja @@ -191,6 +191,7 @@ VALIDATION REQUIREMENTS: - Document complete attack chain - Keep going until you find something that matters - A vulnerability is ONLY considered reported when a reporting agent uses create_vulnerability_report with full details. Mentions in agent_finish, finish_scan, or generic messages are NOT sufficient +- BLACK-BOX REPORTING: when no source code is in scope (black-box), do NOT populate code_locations or assert specific source file paths/line numbers in reports — you cannot see the source, so such claims are fabricated. Base findings only on observed request/response behavior - Do NOT patch/fix before reporting: first create the vulnerability report via create_vulnerability_report (by the reporting agent). Only after reporting is completed should fixing/patching proceed - DEDUPLICATION: The create_vulnerability_report tool uses LLM-based deduplication. If it rejects your report as a duplicate, DO NOT attempt to re-submit the same vulnerability. Accept the rejection and move on to testing other areas. The vulnerability has already been reported by another agent diff --git a/strix/core/runner.py b/strix/core/runner.py index 0630a599..570b9f55 100644 --- a/strix/core/runner.py +++ b/strix/core/runner.py @@ -220,6 +220,7 @@ async def run_strix_scan( "agent_id": root_id, "parent_id": None, "interactive": interactive, + "is_whitebox": is_whitebox, "spawn_child_agent": spawn_child_agent, } diff --git a/strix/tools/reporting/tool.py b/strix/tools/reporting/tool.py index 3fdcfeca..943c96ff 100644 --- a/strix/tools/reporting/tool.py +++ b/strix/tools/reporting/tool.py @@ -351,6 +351,11 @@ async def create_vulnerability_report( for the full rules around ``fix_before`` / ``fix_after``, multi-part fixes, and informational-vs-actionable entries. + **Black-box scans**: when no source code is in scope, do NOT populate + ``code_locations`` and do not assert specific source file paths or line + numbers — you cannot see the source, so such claims are fabricated. Any + ``code_locations`` supplied in a black-box scan are dropped automatically. + **CVSS breakdown** is an object with all 8 metrics (each a single uppercase letter): @@ -484,6 +489,14 @@ async def create_vulnerability_report( - Duplicating the same change across multiple locations. """ inner = ctx.context if isinstance(ctx.context, dict) else {} + if not inner.get("is_whitebox") and code_locations: + # Black-box scan: no source tree is available, so any file paths / + # line numbers / snippets in code_locations can only be fabricated. + # Drop them so a hallucinated "Code Analysis" section can never reach + # the customer-facing report (#321). + logger.info("Black-box scan: dropping code_locations from report %r", title) + code_locations = None + raw_agent_id = inner.get("agent_id") agent_id = raw_agent_id if isinstance(raw_agent_id, str) else None agent_name = None