feat(agents): drop the coverage sections from skills and tool descriptions when the tools are off

This commit is contained in:
Ian 2026-10-23 16:29:49 -04:00 • committed by Ahmed Allam
parent 5418c767a2
commit 898bbb4741
9 changed files with 41 additions and 25 deletions

View file

@ -123,7 +123,9 @@ def render_system_prompt(
is_root=is_root,
is_diff_scoped=is_diff_scoped,
)
skill_content = load_skills(skills_to_load, supports_images=supports_images)
skill_content = load_skills(
skills_to_load, supports_images=supports_images, coverage_tools=coverage_tools
)
env.globals["get_skill"] = lambda name: skill_content.get(name, "")
# Skills every agent of this kind loads come first, so siblings share them

View file

@ -591,6 +591,7 @@ async def run_strix_scan(
"scan_targets": build_scan_targets(scan_config),
"max_context_images": settings.runtime.max_context_images,
"supports_images": supports_images,
"coverage_tools": coverage_tools,
SUBAGENT_TURN_LIMITS_KEY: dict(subagent_turn_limits or {}),
}

View file

@ -21,6 +21,7 @@ _SKILL_TEMPLATES = Environment(
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=True,
)
_INTERNAL_SKILL_CATEGORIES: frozenset[str] = frozenset({"scan_modes", "coordination", "analysis"})
@ -283,7 +284,9 @@ def skill_turn_limit(skill_names: list[str], turn_limits: Mapping[str, int]) ->
)
def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict[str, str]:
def load_skills(
skill_names: list[str], *, supports_images: bool = True, coverage_tools: bool = True
) -> dict[str, str]:
"""Load skill markdown bodies (frontmatter stripped) by name.
Skill files live at ``strix/skills/<category>/<name>.md`` (or any
@ -316,7 +319,7 @@ def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict
metadata, skill_body = _parse_skill_content(content, file_path)
if metadata.get("template") == "jinja":
skill_body = _SKILL_TEMPLATES.from_string(skill_body).render(
supports_images=supports_images
supports_images=supports_images, coverage_tools=coverage_tools
)
skill_content[var_name] = skill_body
logger.debug("Loaded skill: %s -> %s", skill_name, var_name)

View file

@ -1,6 +1,7 @@
---
name: counterevidence
description: Closure discipline for security findings — what counts as proof of safety, what does not, and how to record an unresolved candidate instead of silently dropping it
template: jinja
---
# Counterevidence and Closure Discipline
@ -32,9 +33,9 @@ that sentence, you are not in `ruled_out`.
**3. `open_proof_gap`** — the candidate is plausible, you could not
confirm it, and you also could not name a control that rules it out. This
is a legitimate, expected outcome. Record it with
`record_coverage(outcome="needs_follow_up")` if you have that tool, carry
it up in `agent_finish(open_items=[...])`, and reflect it in `counterevidence` /
is a legitimate, expected outcome. {% if coverage_tools %}Record it with
`record_coverage(outcome="needs_follow_up")`, carry{% else %}Carry{% endif %} it up in
`agent_finish(open_items=[...])`, and reflect it in `counterevidence` /
`confidence_rationale` if you file a related report. Do **not** convert
it to `ruled_out` to tidy up your worklist.
@ -97,10 +98,11 @@ What ships and what is reachable is what matters.
reduces severity — it does not make the finding unreal. Downgrade it;
do not delete it.
{% if coverage_tools %}
## Recording Closure
Closure is only useful if it is written down. If you have the coverage
tools, every surface you assess gets a `record_coverage` entry:
Closure is only useful if it is written down. Every surface you assess
gets a `record_coverage` entry:
- `confirmed` → outcome `reported`, once the report is filed.
- `ruled_out` → outcome `ruled_out`, with the named control in
@ -125,6 +127,7 @@ named control does not cover the path you just found goes back to
previous state is kept as history, so correcting the record costs
nothing and leaving it wrong costs a finding.
{% endif %}
## What DOES Rule Out a Candidate
- You executed the attack and it demonstrably failed, and you understand

View file

@ -1,6 +1,7 @@
---
name: source_aware_discovery
description: Enumeration discipline for reading code — which locations to keep as separate candidates, which safe siblings prove nothing, and the per-family sweeps that are routinely missed
template: jinja
---
# Source-Aware Discovery
@ -206,6 +207,6 @@ Not worth it: "this could use more validation" with no path, style and
maintainability complaints, and cosmetic variants of a candidate you
already opened.
Keep reading until no distinct plausible candidate remains — then record
what you swept with `record_coverage` (if you have it), including the
families that came back clean.
Keep reading until no distinct plausible candidate remains{% if coverage_tools %} — then record
what you swept with `record_coverage`, including the families that came
back clean{% endif %}.

View file

@ -1,6 +1,7 @@
---
name: root-agent
description: Orchestration layer that coordinates specialized subagents for security assessments
template: jinja
---
# Root Agent
@ -35,10 +36,12 @@ Every scan needs one shared answer to "who is the attacker here, and what are th
Either way you write it with the least information anyone on this scan will ever have, so expect it to be wrong somewhere. Subagents correct it with `amend_threat_model`, which appends an attributed addendum instead of overwriting — expect many of these on a black-box run, as authenticating, pivoting between roles, and reaching internal surfaces is exactly what turns inference into fact. Read the amendments back before you write the final report: an agent telling you a boundary you called trusted is attacker-reachable is a finding about your model, not a note. Only call `save_threat_model` again to fold accumulated amendments into the body; it replaces the document and clears them.
{% if coverage_tools %}
## Reconcile Coverage Before Finishing
Coverage entries are shared and mutable. Before `finish_scan`, list the `needs_follow_up` rows: each one is either work you still owe or a row somebody already resolved without updating. Assign the former to a subagent and have it call `update_coverage` on the existing entry rather than recording a second one — a stale open item sitting next to its own resolution is worse than either alone.
{% endif %}
## Agent Architecture
Structure agents by function:

View file

@ -535,9 +535,8 @@ async def create_agent(
success looks like, any constraints. Name the target the
child should call ``get_threat_model`` on, and any shared
state it should build on rather than rediscover — what
recon already mapped, which surfaces are already covered,
which coverage entry it is picking up. A child that is not
told what is already known repeats it.
recon already mapped, which surfaces are already covered.
A child that is not told what is already known repeats it.
inherit_context: Default ``True``. The child receives the
parent's input history as background; only set ``False``
when starting a clean-slate task.
@ -637,13 +636,12 @@ async def agent_finish(
doing: what did you test, what did you find/confirm/rule out,
what's still open.
**Close out honestly.** Before calling this, every surface you assessed
should have a ``record_coverage`` entry if you have that tool.
Anything you could neither confirm nor rule out belongs in
``open_items`` — an unresolved candidate handed up to the parent is
useful, a silently dropped one is a missed vulnerability. Reporting
nothing and listing no open items asserts the area is clean; only
say that if you mean it.
**Close out honestly.** Before calling this, anything you could
neither confirm nor rule out belongs in ``open_items`` — an
unresolved candidate handed up to the parent is useful, a silently
dropped one is a missed vulnerability. Reporting nothing and
listing no open items asserts the area is clean; only say that if
you mean it.
Args:
result_summary: What you accomplished and discovered. Concrete

View file

@ -29,7 +29,11 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str:
if err:
return f"load_skill: {err}"
context = ctx.context if isinstance(ctx.context, dict) else {}
contents = load_skills(requested, supports_images=context.get("supports_images", True))
contents = load_skills(
requested,
supports_images=context.get("supports_images", True),
coverage_tools=context.get("coverage_tools", True),
)
if not contents:
return "load_skill: no content loaded for requested skills."
sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()]

View file

@ -107,14 +107,15 @@ def test_report_tool_replaces_create_vulnerability_report() -> None:
def test_coverage_tools_can_be_turned_off() -> None:
coverage = {"record_coverage", "update_coverage", "list_coverage"}
for is_root in (True, False):
on = factory.build_strix_agent(is_root=is_root)
off = factory.build_strix_agent(is_root=is_root, coverage_tools=False)
on = factory.build_strix_agent(is_root=is_root, is_whitebox=True)
off = factory.build_strix_agent(is_root=is_root, is_whitebox=True, coverage_tools=False)
assert coverage <= {t.name for t in on.tools}
assert not coverage & {t.name for t in off.tools}
assert isinstance(on.instructions, str)
assert isinstance(off.instructions, str)
assert "- COVERAGE" in on.instructions
assert "- COVERAGE" not in off.instructions
assert "## Recording Closure" in on.instructions
assert not any(name in off.instructions for name in coverage)
def test_instructions_override_is_used_verbatim() -> None: