mirror of
https://github.com/usestrix/strix.git
synced 2026-10-16 04:28:23 +00:00
2a76196f78
A row recorded as "DirectoryTraversal" or "XSLeaks" normalizes to one token, which whole-word matching missed, so finish_scan/coverage.json could report tested work as a gap. _entry_is_about now also matches a phrasing's words joined into one token as a fallback - still a whole-token check, never a substring. The branch is additive: it can only turn a gap into a cover, never the reverse, so it introduces no false gap.
448 lines
17 KiB
Python
448 lines
17 KiB
Python
"""Tests for the coverage artifact assembled in strix.report.coverage."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from strix.report.coverage import (
|
|
_SKILL_PHRASINGS,
|
|
VULN_CLASSES,
|
|
build_coverage_document,
|
|
cwe_for_skill,
|
|
read_agent_graph,
|
|
write_coverage,
|
|
)
|
|
from strix.skills import get_available_skills
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
def _entry(**overrides: Any) -> dict[str, Any]:
|
|
base: dict[str, Any] = {
|
|
"surface": "POST /api/orders/{id}",
|
|
"risk_area": "object-level authorization",
|
|
"outcome": "no_issue_found",
|
|
"evidence": "Two tenants tested; both received 403.",
|
|
"agent_id": "agent-1",
|
|
"agent_name": "authz-tester",
|
|
"created_at": "2026-07-02 10:00:00 UTC",
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _graph(**overrides: Any) -> dict[str, Any]:
|
|
base: dict[str, Any] = {
|
|
"statuses": {"agent-1": "completed"},
|
|
"names": {"agent-1": "authz-tester"},
|
|
"metadata": {"agent-1": {"skills": ["idor"], "task": "authz review"}},
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _document(**overrides: Any) -> dict[str, Any]:
|
|
kwargs: dict[str, Any] = {
|
|
"run_record": {"run_id": "r1", "run_name": "run-1", "status": "completed"},
|
|
"entries": [_entry()],
|
|
"agent_graph": _graph(),
|
|
"vulnerability_reports": [],
|
|
}
|
|
kwargs.update(overrides)
|
|
return build_coverage_document(**kwargs)
|
|
|
|
|
|
def test_document_reports_surfaces_and_outcomes() -> None:
|
|
doc = _document()
|
|
|
|
assert doc["summary"]["surfaces_reviewed"] == 1
|
|
assert doc["summary"]["outcomes"] == {"no_issue_found": 1}
|
|
assert doc["entries"][0]["outcome_label"] == "No issue identified"
|
|
assert doc["entries"][0]["recorded_by"] == "authz-tester"
|
|
|
|
|
|
def test_ledger_entries_are_labelled_as_agent_reported() -> None:
|
|
"""A reader has to be able to tell a self-report from an observation."""
|
|
doc = _document()
|
|
|
|
assert doc["entries"][0]["source"] == "agent_reported"
|
|
assert doc["machine_observed"]["source"] == "runtime"
|
|
assert doc["machine_observed"]["skills_exercised"] == ["idor"]
|
|
|
|
|
|
def test_assigned_risk_skill_without_coverage_becomes_a_gap() -> None:
|
|
"""An agent carrying the sql_injection skill that records nothing about it
|
|
leaves the class unexamined, not clean."""
|
|
doc = _document(
|
|
agent_graph=_graph(
|
|
metadata={"agent-1": {"skills": ["idor", "sql_injection"], "task": "review"}}
|
|
)
|
|
)
|
|
|
|
gaps = [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
assert [gap["risk_area"] for gap in gaps] == ["sql injection"]
|
|
|
|
|
|
def test_substring_match_does_not_hide_a_gap() -> None:
|
|
"""A short class token matches whole words, not substrings: 'brute force' is
|
|
not RCE coverage, 'enforcement' is not 'rce', 'corridor' is not 'idor'."""
|
|
cases = [
|
|
("rce", "POST /login", "brute force protection"),
|
|
("rce", "GET /api/resources/{id}", "object-level authorization"),
|
|
("rce", "Access control enforcement", "authorization"),
|
|
("idor", "GET /building/corridor/{id}", "information disclosure"),
|
|
]
|
|
for skill, surface, risk in cases:
|
|
doc = _document(
|
|
entries=[_entry(surface=surface, risk_area=risk)],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": [skill]}}),
|
|
)
|
|
classes = [g["risk_area"] for g in doc["gaps"] if g["kind"] == "unrecorded_risk_class"]
|
|
assert skill.replace("_", " ") in classes, (skill, surface, risk, classes)
|
|
|
|
|
|
def test_joined_camelcase_name_counts_as_coverage() -> None:
|
|
"""A row recorded as a joined/CamelCase class name ('DirectoryTraversal')
|
|
still counts, via the joined-token fallback — no substring matching."""
|
|
doc = _document(
|
|
entries=[_entry(risk_area="DirectoryTraversal reviewed", surface="/download")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["path_traversal_lfi_rfi"]}}),
|
|
)
|
|
|
|
assert not [g for g in doc["gaps"] if g["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_simple_plural_still_counts_as_coverage() -> None:
|
|
"""Whole-word matching tolerates a trivial plural, so 'passwords' /
|
|
'credentials' still cover weak_password_detection."""
|
|
doc = _document(
|
|
entries=[_entry(risk_area="weak passwords and credentials reviewed", surface="/login")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["weak_password_detection"]}}),
|
|
)
|
|
|
|
assert not [g for g in doc["gaps"] if g["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_recorded_risk_class_is_not_reported_as_a_gap() -> None:
|
|
doc = _document(
|
|
entries=[_entry(risk_area="SQL injection", surface="GET /search?q=")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["sql_injection"]}}),
|
|
)
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_synonym_phrasing_counts_as_recorded_coverage() -> None:
|
|
"""The ledger says "object-level authorization"; the skill is called idor."""
|
|
doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor"]}}))
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_cspt_acronym_phrasing_counts_as_recorded_coverage() -> None:
|
|
"""The ledger spells out "Client-Side Path Traversal (CSPT)"; the skill is
|
|
called client_side_path_traversal."""
|
|
doc = _document(
|
|
entries=[
|
|
_entry(risk_area="Client-Side Path Traversal (CSPT)", surface="GET /app#/profile")
|
|
],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["client_side_path_traversal"]}}),
|
|
)
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_bundled_skill_partially_tested_flags_untested_subtopics() -> None:
|
|
"""browser_security bundles many surfaces; proving postMessage must not
|
|
mark client-side path traversal covered."""
|
|
doc = _document(
|
|
entries=[_entry(risk_area="postMessage origin validation", surface="window listener")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["browser_security"]}}),
|
|
)
|
|
|
|
flagged = {g["risk_area"] for g in doc["gaps"] if g["kind"] == "unrecorded_sub_topic"}
|
|
assert "client-side path traversal" in flagged
|
|
assert "postMessage" not in flagged # the surface that was tested is not re-flagged
|
|
# a partially-tested bundled skill is not also reported as a wholesale gap
|
|
assert not [
|
|
g
|
|
for g in doc["gaps"]
|
|
if g["kind"] == "unrecorded_risk_class" and g["risk_area"] == "browser security"
|
|
]
|
|
|
|
|
|
def test_bundled_skill_wholly_untested_is_a_single_gap() -> None:
|
|
"""A bundled skill nobody touched collapses to one class-level gap, not one
|
|
per surface — the honest short statement is "not examined"."""
|
|
doc = _document(
|
|
entries=[_entry(risk_area="SQL injection", surface="GET /search")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["browser_security"]}}),
|
|
)
|
|
|
|
class_gaps = [
|
|
g
|
|
for g in doc["gaps"]
|
|
if g["kind"] == "unrecorded_risk_class" and g["risk_area"] == "browser security"
|
|
]
|
|
assert len(class_gaps) == 1
|
|
assert "surfaces" in class_gaps[0]["detail"]
|
|
assert not [g for g in doc["gaps"] if g["kind"] == "unrecorded_sub_topic"]
|
|
|
|
|
|
def test_bundled_skill_with_every_surface_covered_is_clean() -> None:
|
|
"""Covering every surface of a bundled skill leaves no coverage gap."""
|
|
entries = [
|
|
_entry(risk_area=sub.phrasings[0], surface=sub.label)
|
|
for sub in VULN_CLASSES["browser_security"].sub_topics
|
|
]
|
|
doc = _document(
|
|
entries=entries,
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["browser_security"]}}),
|
|
)
|
|
|
|
assert not [
|
|
g
|
|
for g in doc["gaps"]
|
|
if g["kind"] in {"unrecorded_risk_class", "unrecorded_sub_topic"}
|
|
]
|
|
|
|
|
|
def test_unrelated_agents_row_does_not_cover_a_browser_subtopic() -> None:
|
|
"""An open_redirect agent's 'meta refresh' row must not mark browser_security's
|
|
navigation surface covered — only a browser_security carrier's row counts."""
|
|
doc = _document(
|
|
entries=[
|
|
_entry(
|
|
risk_area="postMessage origin validation",
|
|
surface="listener",
|
|
agent_id="agent-1",
|
|
agent_name="browser-tester",
|
|
),
|
|
_entry(
|
|
risk_area="open redirect via meta refresh",
|
|
surface="/login?next=",
|
|
agent_id="agent-2",
|
|
agent_name="redirect-tester",
|
|
),
|
|
],
|
|
agent_graph=_graph(
|
|
statuses={"agent-1": "completed", "agent-2": "completed"},
|
|
names={"agent-1": "browser-tester", "agent-2": "redirect-tester"},
|
|
metadata={
|
|
"agent-1": {"skills": ["browser_security"]},
|
|
"agent-2": {"skills": ["open_redirect"]},
|
|
},
|
|
),
|
|
)
|
|
|
|
sub = {g["risk_area"] for g in doc["gaps"] if g["kind"] == "unrecorded_sub_topic"}
|
|
assert "navigation and redirect control" in sub # not masked by the open_redirect row
|
|
assert "postMessage" not in sub # the browser agent's own row still counts
|
|
|
|
|
|
def test_updated_browser_row_keeps_its_carrier_authorship() -> None:
|
|
"""update_coverage overwrites agent_id with the updater's; a browser surface a
|
|
carrier assessed must still count via the author preserved in history."""
|
|
doc = _document(
|
|
entries=[
|
|
_entry(
|
|
risk_area="client-side path traversal",
|
|
surface="fetch from route param",
|
|
agent_id="agent-2", # current author after an unrelated update
|
|
agent_name="redirect-tester",
|
|
history=[
|
|
{
|
|
"outcome": "no_issue_found",
|
|
"agent_id": "agent-1", # the carrier who first assessed it
|
|
"agent_name": "browser-tester",
|
|
}
|
|
],
|
|
),
|
|
],
|
|
agent_graph=_graph(
|
|
statuses={"agent-1": "completed", "agent-2": "completed"},
|
|
names={"agent-1": "browser-tester", "agent-2": "redirect-tester"},
|
|
metadata={
|
|
"agent-1": {"skills": ["browser_security"]},
|
|
"agent-2": {"skills": ["open_redirect"]},
|
|
},
|
|
),
|
|
)
|
|
|
|
sub = {g["risk_area"] for g in doc["gaps"] if g["kind"] == "unrecorded_sub_topic"}
|
|
assert "client-side path traversal" not in sub # preserved carrier authorship counts
|
|
|
|
|
|
def test_registry_backs_skill_phrasings_and_cwe() -> None:
|
|
"""_SKILL_PHRASINGS is derived from the registry, so they cannot drift."""
|
|
assert _SKILL_PHRASINGS == {name: v.aliases for name, v in VULN_CLASSES.items()}
|
|
assert cwe_for_skill("sql_injection") == ("CWE-89",)
|
|
assert cwe_for_skill("vulnerabilities/ssrf") == ("CWE-918",)
|
|
assert cwe_for_skill("subdomain_takeover") == ()
|
|
|
|
|
|
def test_non_risk_skills_carry_no_coverage_obligation() -> None:
|
|
"""Tooling skills describe how an agent works, not what it hunts."""
|
|
doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor", "caido"]}}))
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap.get("risk_area") == "caido"]
|
|
|
|
|
|
def test_agent_that_recorded_nothing_is_a_gap() -> None:
|
|
doc = _document(
|
|
agent_graph=_graph(
|
|
statuses={"agent-1": "completed", "agent-2": "completed"},
|
|
names={"agent-1": "authz-tester", "agent-2": "recon"},
|
|
metadata={},
|
|
)
|
|
)
|
|
|
|
silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
|
|
assert [gap["agent_name"] for gap in silent] == ["recon"]
|
|
|
|
|
|
def test_needs_follow_up_is_carried_as_an_open_gap() -> None:
|
|
doc = _document(
|
|
entries=[_entry(outcome="needs_follow_up", evidence="Auth wall blocked testing.")]
|
|
)
|
|
|
|
assert doc["gaps"][0]["kind"] == "needs_follow_up"
|
|
assert doc["gaps"][0]["detail"] == "Auth wall blocked testing."
|
|
|
|
|
|
def test_completed_run_with_finished_agents_is_complete() -> None:
|
|
doc = _document(exit_reason="finished_by_tool")
|
|
|
|
assert doc["completeness"]["complete"] is True
|
|
assert doc["completeness"]["caveats"] == []
|
|
|
|
|
|
def test_budget_exhausted_run_is_not_a_complete_record() -> None:
|
|
"""A truncated scan must not read like a clean one."""
|
|
doc = _document(exit_reason="budget_exhausted")
|
|
|
|
assert doc["completeness"]["complete"] is False
|
|
assert "budget_exhausted" in doc["completeness"]["caveats"][0]
|
|
|
|
|
|
def test_unfinished_agent_makes_the_record_partial() -> None:
|
|
doc = _document(
|
|
agent_graph=_graph(statuses={"agent-1": "crashed"}),
|
|
exit_reason="finished_by_tool",
|
|
)
|
|
|
|
assert doc["completeness"]["complete"] is False
|
|
assert "authz-tester" in doc["completeness"]["caveats"][0]
|
|
|
|
|
|
def test_failed_run_status_makes_the_record_partial() -> None:
|
|
doc = _document(
|
|
run_record={"run_id": "r1", "status": "failed"},
|
|
exit_reason="finished_by_tool",
|
|
)
|
|
|
|
assert doc["completeness"]["complete"] is False
|
|
|
|
|
|
def test_write_coverage_emits_a_top_level_artifact(tmp_path: Path) -> None:
|
|
path = write_coverage(tmp_path, _document())
|
|
|
|
assert path == tmp_path / "coverage.json"
|
|
assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
|
|
|
|
|
|
def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
|
|
assert read_agent_graph(tmp_path) == {}
|
|
|
|
(tmp_path / "agents.json").write_text("{not json", encoding="utf-8")
|
|
assert read_agent_graph(tmp_path) == {}
|
|
|
|
|
|
def test_read_agent_graph_loads_a_snapshot(tmp_path: Path) -> None:
|
|
(tmp_path / "agents.json").write_text(json.dumps(_graph()), encoding="utf-8")
|
|
|
|
assert read_agent_graph(tmp_path)["names"] == {"agent-1": "authz-tester"}
|
|
|
|
|
|
def test_multi_token_skill_matches_how_a_pentester_writes_it() -> None:
|
|
"""An agent carrying path_traversal_lfi_rfi records "Path Traversal".
|
|
|
|
Requiring the skill's filename verbatim published a false gap for a class
|
|
that had been tested and even had a finding filed against it.
|
|
"""
|
|
doc = _document(
|
|
entries=[_entry(risk_area="Path Traversal / Directory Traversal", surface="/download")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["path_traversal_lfi_rfi"]}}),
|
|
)
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def _vulnerability_skill_names() -> set[str]:
|
|
return {skill["name"] for skill in get_available_skills()["vulnerabilities"]}
|
|
|
|
|
|
def test_every_vulnerability_skill_declares_its_phrasings() -> None:
|
|
"""A new skill without phrasings would be matched by its filename alone,
|
|
which is how the false gap above got published."""
|
|
missing = _vulnerability_skill_names() - set(_SKILL_PHRASINGS)
|
|
|
|
assert not missing, f"add ledger phrasings for: {sorted(missing)}"
|
|
|
|
|
|
def test_declared_phrasings_name_real_skills() -> None:
|
|
stale = set(_SKILL_PHRASINGS) - _vulnerability_skill_names()
|
|
|
|
assert not stale, f"phrasings for skills that no longer exist: {sorted(stale)}"
|
|
|
|
|
|
def _delegating_graph(**overrides: Any) -> dict[str, Any]:
|
|
base: dict[str, Any] = {
|
|
"statuses": {"root": "completed", "agent-1": "completed"},
|
|
"names": {"root": "Root Agent", "agent-1": "authz-tester"},
|
|
"parent_of": {"agent-1": "root"},
|
|
"metadata": {"agent-1": {"skills": ["idor"]}},
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def test_delegating_root_agent_is_not_a_coverage_gap() -> None:
|
|
"""The root delegates and reconciles; it is not a tester that went quiet.
|
|
Flagging it would put the same false line in every clean report."""
|
|
doc = _document(agent_graph=_delegating_graph())
|
|
|
|
silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
|
|
assert silent == []
|
|
|
|
|
|
def test_a_subagent_that_records_nothing_is_still_a_gap() -> None:
|
|
doc = _document(
|
|
agent_graph=_delegating_graph(
|
|
statuses={"root": "completed", "agent-1": "completed", "agent-2": "completed"},
|
|
names={"root": "Root Agent", "agent-1": "authz-tester", "agent-2": "recon"},
|
|
parent_of={"agent-1": "root", "agent-2": "root"},
|
|
)
|
|
)
|
|
|
|
silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
|
|
assert [gap["agent_name"] for gap in silent] == ["recon"]
|
|
|
|
|
|
def test_a_root_that_worked_alone_is_held_to_the_rule() -> None:
|
|
"""With no subagents there is nobody else the testing could have come
|
|
from, so silence is a real gap."""
|
|
doc = _document(
|
|
entries=[],
|
|
agent_graph={
|
|
"statuses": {"root": "completed"},
|
|
"names": {"root": "Root Agent"},
|
|
"parent_of": {},
|
|
},
|
|
)
|
|
|
|
silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
|
|
assert [gap["agent_name"] for gap in silent] == ["Root Agent"]
|