mirror of
https://github.com/usestrix/strix.git
synced 2026-11-01 07:02:32 +00:00
feat(agents): drop the coverage sections from skills and tool descriptions when the tools are off
This commit is contained in:
parent
5418c767a2
commit
898bbb4741
9 changed files with 41 additions and 25 deletions
|
|
@ -123,7 +123,9 @@ def render_system_prompt(
|
|||
is_root=is_root,
|
||||
is_diff_scoped=is_diff_scoped,
|
||||
)
|
||||
skill_content = load_skills(skills_to_load, supports_images=supports_images)
|
||||
skill_content = load_skills(
|
||||
skills_to_load, supports_images=supports_images, coverage_tools=coverage_tools
|
||||
)
|
||||
env.globals["get_skill"] = lambda name: skill_content.get(name, "")
|
||||
|
||||
# Skills every agent of this kind loads come first, so siblings share them
|
||||
|
|
|
|||
|
|
@ -591,6 +591,7 @@ async def run_strix_scan(
|
|||
"scan_targets": build_scan_targets(scan_config),
|
||||
"max_context_images": settings.runtime.max_context_images,
|
||||
"supports_images": supports_images,
|
||||
"coverage_tools": coverage_tools,
|
||||
SUBAGENT_TURN_LIMITS_KEY: dict(subagent_turn_limits or {}),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ _SKILL_TEMPLATES = Environment(
|
|||
undefined=StrictUndefined,
|
||||
trim_blocks=True,
|
||||
lstrip_blocks=True,
|
||||
keep_trailing_newline=True,
|
||||
)
|
||||
|
||||
_INTERNAL_SKILL_CATEGORIES: frozenset[str] = frozenset({"scan_modes", "coordination", "analysis"})
|
||||
|
|
@ -283,7 +284,9 @@ def skill_turn_limit(skill_names: list[str], turn_limits: Mapping[str, int]) ->
|
|||
)
|
||||
|
||||
|
||||
def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict[str, str]:
|
||||
def load_skills(
|
||||
skill_names: list[str], *, supports_images: bool = True, coverage_tools: bool = True
|
||||
) -> dict[str, str]:
|
||||
"""Load skill markdown bodies (frontmatter stripped) by name.
|
||||
|
||||
Skill files live at ``strix/skills/<category>/<name>.md`` (or any
|
||||
|
|
@ -316,7 +319,7 @@ def load_skills(skill_names: list[str], *, supports_images: bool = True) -> dict
|
|||
metadata, skill_body = _parse_skill_content(content, file_path)
|
||||
if metadata.get("template") == "jinja":
|
||||
skill_body = _SKILL_TEMPLATES.from_string(skill_body).render(
|
||||
supports_images=supports_images
|
||||
supports_images=supports_images, coverage_tools=coverage_tools
|
||||
)
|
||||
skill_content[var_name] = skill_body
|
||||
logger.debug("Loaded skill: %s -> %s", skill_name, var_name)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
---
|
||||
name: counterevidence
|
||||
description: Closure discipline for security findings — what counts as proof of safety, what does not, and how to record an unresolved candidate instead of silently dropping it
|
||||
template: jinja
|
||||
---
|
||||
|
||||
# Counterevidence and Closure Discipline
|
||||
|
|
@ -32,9 +33,9 @@ that sentence, you are not in `ruled_out`.
|
|||
|
||||
**3. `open_proof_gap`** — the candidate is plausible, you could not
|
||||
confirm it, and you also could not name a control that rules it out. This
|
||||
is a legitimate, expected outcome. Record it with
|
||||
`record_coverage(outcome="needs_follow_up")` if you have that tool, carry
|
||||
it up in `agent_finish(open_items=[...])`, and reflect it in `counterevidence` /
|
||||
is a legitimate, expected outcome. {% if coverage_tools %}Record it with
|
||||
`record_coverage(outcome="needs_follow_up")`, carry{% else %}Carry{% endif %} it up in
|
||||
`agent_finish(open_items=[...])`, and reflect it in `counterevidence` /
|
||||
`confidence_rationale` if you file a related report. Do **not** convert
|
||||
it to `ruled_out` to tidy up your worklist.
|
||||
|
||||
|
|
@ -97,10 +98,11 @@ What ships and what is reachable is what matters.
|
|||
reduces severity — it does not make the finding unreal. Downgrade it;
|
||||
do not delete it.
|
||||
|
||||
{% if coverage_tools %}
|
||||
## Recording Closure
|
||||
|
||||
Closure is only useful if it is written down. If you have the coverage
|
||||
tools, every surface you assess gets a `record_coverage` entry:
|
||||
Closure is only useful if it is written down. Every surface you assess
|
||||
gets a `record_coverage` entry:
|
||||
|
||||
- `confirmed` → outcome `reported`, once the report is filed.
|
||||
- `ruled_out` → outcome `ruled_out`, with the named control in
|
||||
|
|
@ -125,6 +127,7 @@ named control does not cover the path you just found goes back to
|
|||
previous state is kept as history, so correcting the record costs
|
||||
nothing and leaving it wrong costs a finding.
|
||||
|
||||
{% endif %}
|
||||
## What DOES Rule Out a Candidate
|
||||
|
||||
- You executed the attack and it demonstrably failed, and you understand
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
---
|
||||
name: source_aware_discovery
|
||||
description: Enumeration discipline for reading code — which locations to keep as separate candidates, which safe siblings prove nothing, and the per-family sweeps that are routinely missed
|
||||
template: jinja
|
||||
---
|
||||
|
||||
# Source-Aware Discovery
|
||||
|
|
@ -206,6 +207,6 @@ Not worth it: "this could use more validation" with no path, style and
|
|||
maintainability complaints, and cosmetic variants of a candidate you
|
||||
already opened.
|
||||
|
||||
Keep reading until no distinct plausible candidate remains — then record
|
||||
what you swept with `record_coverage` (if you have it), including the
|
||||
families that came back clean.
|
||||
Keep reading until no distinct plausible candidate remains{% if coverage_tools %} — then record
|
||||
what you swept with `record_coverage`, including the families that came
|
||||
back clean{% endif %}.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
---
|
||||
name: root-agent
|
||||
description: Orchestration layer that coordinates specialized subagents for security assessments
|
||||
template: jinja
|
||||
---
|
||||
|
||||
# Root Agent
|
||||
|
|
@ -35,10 +36,12 @@ Every scan needs one shared answer to "who is the attacker here, and what are th
|
|||
|
||||
Either way you write it with the least information anyone on this scan will ever have, so expect it to be wrong somewhere. Subagents correct it with `amend_threat_model`, which appends an attributed addendum instead of overwriting — expect many of these on a black-box run, as authenticating, pivoting between roles, and reaching internal surfaces is exactly what turns inference into fact. Read the amendments back before you write the final report: an agent telling you a boundary you called trusted is attacker-reachable is a finding about your model, not a note. Only call `save_threat_model` again to fold accumulated amendments into the body; it replaces the document and clears them.
|
||||
|
||||
{% if coverage_tools %}
|
||||
## Reconcile Coverage Before Finishing
|
||||
|
||||
Coverage entries are shared and mutable. Before `finish_scan`, list the `needs_follow_up` rows: each one is either work you still owe or a row somebody already resolved without updating. Assign the former to a subagent and have it call `update_coverage` on the existing entry rather than recording a second one — a stale open item sitting next to its own resolution is worse than either alone.
|
||||
|
||||
{% endif %}
|
||||
## Agent Architecture
|
||||
|
||||
Structure agents by function:
|
||||
|
|
|
|||
|
|
@ -535,9 +535,8 @@ async def create_agent(
|
|||
success looks like, any constraints. Name the target the
|
||||
child should call ``get_threat_model`` on, and any shared
|
||||
state it should build on rather than rediscover — what
|
||||
recon already mapped, which surfaces are already covered,
|
||||
which coverage entry it is picking up. A child that is not
|
||||
told what is already known repeats it.
|
||||
recon already mapped, which surfaces are already covered.
|
||||
A child that is not told what is already known repeats it.
|
||||
inherit_context: Default ``True``. The child receives the
|
||||
parent's input history as background; only set ``False``
|
||||
when starting a clean-slate task.
|
||||
|
|
@ -637,13 +636,12 @@ async def agent_finish(
|
|||
doing: what did you test, what did you find/confirm/rule out,
|
||||
what's still open.
|
||||
|
||||
**Close out honestly.** Before calling this, every surface you assessed
|
||||
should have a ``record_coverage`` entry if you have that tool.
|
||||
Anything you could neither confirm nor rule out belongs in
|
||||
``open_items`` — an unresolved candidate handed up to the parent is
|
||||
useful, a silently dropped one is a missed vulnerability. Reporting
|
||||
nothing and listing no open items asserts the area is clean; only
|
||||
say that if you mean it.
|
||||
**Close out honestly.** Before calling this, anything you could
|
||||
neither confirm nor rule out belongs in ``open_items`` — an
|
||||
unresolved candidate handed up to the parent is useful, a silently
|
||||
dropped one is a missed vulnerability. Reporting nothing and
|
||||
listing no open items asserts the area is clean; only say that if
|
||||
you mean it.
|
||||
|
||||
Args:
|
||||
result_summary: What you accomplished and discovered. Concrete
|
||||
|
|
|
|||
|
|
@ -29,7 +29,11 @@ async def load_skill(ctx: RunContextWrapper, skills: list[str]) -> str:
|
|||
if err:
|
||||
return f"load_skill: {err}"
|
||||
context = ctx.context if isinstance(ctx.context, dict) else {}
|
||||
contents = load_skills(requested, supports_images=context.get("supports_images", True))
|
||||
contents = load_skills(
|
||||
requested,
|
||||
supports_images=context.get("supports_images", True),
|
||||
coverage_tools=context.get("coverage_tools", True),
|
||||
)
|
||||
if not contents:
|
||||
return "load_skill: no content loaded for requested skills."
|
||||
sections = [f"## Skill: {name}\n\n{body}" for name, body in contents.items()]
|
||||
|
|
|
|||
|
|
@ -107,14 +107,15 @@ def test_report_tool_replaces_create_vulnerability_report() -> None:
|
|||
def test_coverage_tools_can_be_turned_off() -> None:
|
||||
coverage = {"record_coverage", "update_coverage", "list_coverage"}
|
||||
for is_root in (True, False):
|
||||
on = factory.build_strix_agent(is_root=is_root)
|
||||
off = factory.build_strix_agent(is_root=is_root, coverage_tools=False)
|
||||
on = factory.build_strix_agent(is_root=is_root, is_whitebox=True)
|
||||
off = factory.build_strix_agent(is_root=is_root, is_whitebox=True, coverage_tools=False)
|
||||
assert coverage <= {t.name for t in on.tools}
|
||||
assert not coverage & {t.name for t in off.tools}
|
||||
assert isinstance(on.instructions, str)
|
||||
assert isinstance(off.instructions, str)
|
||||
assert "- COVERAGE" in on.instructions
|
||||
assert "- COVERAGE" not in off.instructions
|
||||
assert "## Recording Closure" in on.instructions
|
||||
assert not any(name in off.instructions for name in coverage)
|
||||
|
||||
|
||||
def test_instructions_override_is_used_verbatim() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue