fix(coverage): restore inflected-name matching and browser heading matches

Whole-word matching opened false "unexamined" gaps that main did not
report: "Open redirection", "NoSQLi" and "Information leakage" no
longer counted. A phrasing term of four or more letters now also
matches a ledger word it starts; short tokens such as rce and xs stay
whole-word, so rce still never matches inside force.

browser_security: rows named after the skill's own headings ("Navigation
and Redirect Control", "CSP and Browser Parsing") now cover those
surfaces, and a carrier's generic "Browser security" row yields
per-surface gaps instead of claiming nothing was recorded.
This commit is contained in:
Abdelrahman Mostafa 2026-10-22 16:17:44 +03:00
parent d49130e764
commit ac70e0ed10
2 changed files with 83 additions and 6 deletions

View file

@ -61,6 +61,9 @@ _INCOMPLETE_AGENT_STATUSES = frozenset({"crashed", "stopped", "running", "waitin
#: Run statuses that mean the scan itself did not run to completion.
_INCOMPLETE_RUN_STATUSES = frozenset({"failed", "interrupted", "stopped", "running"})
#: Shortest phrasing term that may match the start of a longer ledger word.
_MIN_PREFIX_TERM = 4
#: Only this skill category names a vulnerability class. ``tooling`` and
#: ``reconnaissance`` skills describe how an agent works, not what it hunts,
#: so holding one implies no coverage obligation.
@ -149,11 +152,17 @@ VULN_CLASSES: dict[str, VulnClass] = {
SubTopic("web workers", ("web worker", "worker script")),
SubTopic(
"navigation and redirect control",
("client side redirect", "spa navigation", "history manipulation", "meta refresh"),
(
"client side redirect",
"spa navigation",
"history manipulation",
"meta refresh",
"navigation redirect",
),
),
SubTopic(
"CSP and browser parsing",
("content security policy", "csp bypass", "trusted types"),
("content security policy", "csp", "trusted types"),
),
SubTopic("local network access", ("local network", "loopback", "dns rebinding")),
SubTopic("JavaScript gadgets", ("javascript gadget", "gadget chain", "dom gadget")),
@ -330,8 +339,16 @@ def _term_in_words(term: str, words: set[str]) -> bool:
unrelated XS-Leaks row would falsely mark XSS covered. A phrasing that
needs to match a singular row carries the singular spelling as its own
alias instead.
A term of four or more letters also matches a row word it starts, so the
inflections and suffixed names agents write ("Open redirection",
"NoSQLi", "Information leakage") still count. The match is anchored at
the start of the word, and short tokens such as ``rce`` and ``xs`` are
excluded, so ``rce`` never matches inside ``force``.
"""
return term in words or f"{term}s" in words
if term in words or f"{term}s" in words:
return True
return len(term) >= _MIN_PREFIX_TERM and any(word.startswith(term) for word in words)
def _entry_is_about(entry: dict[str, Any], phrasings: list[list[str]]) -> bool:
@ -452,7 +469,19 @@ def skill_coverage_gaps(
]
if not uncovered:
continue
if len(uncovered) == len(topics):
# A carrier row that names a bundled skill only generically ("Browser
# security") is a record of work, so the gap is its unnamed surfaces,
# not "nothing recorded".
vclass = VULN_CLASSES.get(skill)
named_generically = (
vclass is not None
and bool(vclass.sub_topics)
and any(
_entry_is_about(entry, _phrasing_wordlists(vclass.aliases))
for entry in carrier_rows
)
)
if len(uncovered) == len(topics) and not named_generically:
# Nothing about this class was recorded at all — one class-level gap.
detail = (
f"Agent(s) {names} were assigned the '{skill}' skill, but no coverage "
@ -479,8 +508,8 @@ def skill_coverage_gaps(
"skill": skill,
"risk_area": str(label),
"detail": (
f"Agent(s) {names} carried '{skill}' and recorded coverage for some "
f"of its surfaces, but nothing records '{label}'. Testing one surface "
f"Agent(s) {names} carried '{skill}' and recorded coverage under it, "
f"but nothing records '{label}'. Testing one surface "
"of a multi-surface skill does not cover the others — treat "
f"'{label}' as unexamined."
),

View file

@ -499,3 +499,51 @@ def test_a_root_that_worked_alone_is_held_to_the_rule() -> None:
silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
assert [gap["agent_name"] for gap in silent] == ["Root Agent"]
def test_inflected_class_names_still_count_as_coverage() -> None:
"""Agents write "Open redirection", "NoSQLi" and "Information leakage"; a
phrasing term of four or more letters matches the start of such a word."""
for skill, risk_area in (
("open_redirect", "Open redirection"),
("nosql_injection", "NoSQLi"),
("information_disclosure", "Information leakage"),
):
doc = _document(
entries=[_entry(risk_area=risk_area, surface="GET /")],
agent_graph=_graph(metadata={"agent-1": {"skills": [skill]}}),
)
assert not [g for g in doc["gaps"] if g["kind"] == "unrecorded_risk_class"], skill
def test_browser_surfaces_match_their_skill_headings() -> None:
"""Rows named after browser_security's own section headings cover them."""
doc = _document(
entries=[
_entry(risk_area="Navigation and Redirect Control", surface="/app"),
_entry(risk_area="CSP and Browser Parsing", surface="/app"),
],
agent_graph=_graph(metadata={"agent-1": {"skills": ["browser_security"]}}),
)
sub = {g["risk_area"] for g in doc["gaps"] if g["kind"] == "unrecorded_sub_topic"}
assert "navigation and redirect control" not in sub
assert "CSP and browser parsing" not in sub
assert "postMessage" in sub
def test_generic_browser_row_names_untested_surfaces_not_nothing_recorded() -> None:
"""A carrier's generic "Browser security" row is recorded work: the gap is
each unnamed surface, not "no coverage entry records this class"."""
doc = _document(
entries=[_entry(risk_area="Browser security", surface="SPA shell /app")],
agent_graph=_graph(metadata={"agent-1": {"skills": ["browser_security"]}}),
)
assert not [
g
for g in doc["gaps"]
if g["kind"] == "unrecorded_risk_class" and g["risk_area"] == "browser security"
]
sub = {g["risk_area"] for g in doc["gaps"] if g["kind"] == "unrecorded_sub_topic"}
assert "client-side path traversal" in sub