From e97112a88d8e853ca8e21479387ba8a2bb5eeceb Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Wed, 7 Oct 2026 14:47:50 +0800 Subject: [PATCH 1/2] Fix coverage completeness: include 'failed' agent status and use word-boundary matching - Add 'failed' to _INCOMPLETE_AGENT_STATUSES so agents that exited failed correctly mark the coverage record as incomplete (fixes #1458) - Fix _entry_is_about substring matching bug: use word-boundary set membership instead of substring 'in' so terms like 'rce' don't match inside 'force' (fixes #1447: 'brute force' was falsely counted as RCE coverage) --- strix/report/coverage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/strix/report/coverage.py b/strix/report/coverage.py index c519853a..76387e61 100644 --- a/strix/report/coverage.py +++ b/strix/report/coverage.py @@ -55,7 +55,7 @@ OUTCOME_LABELS: dict[str, str] = { } #: Statuses that mean the agent stopped early rather than finishing its task. -_INCOMPLETE_AGENT_STATUSES = frozenset({"crashed", "stopped", "running", "waiting"}) +_INCOMPLETE_AGENT_STATUSES = frozenset({"crashed", "failed", "stopped", "running", "waiting"}) #: Run statuses that mean the scan itself did not run to completion. _INCOMPLETE_RUN_STATUSES = frozenset({"failed", "interrupted", "stopped", "running"}) @@ -224,7 +224,8 @@ def _skill_phrasings(skill: str) -> list[list[str]]: def _entry_is_about(entry: dict[str, Any], phrasings: list[list[str]]) -> bool: """True when a ledger row plausibly concerns any phrasing of a risk class.""" haystack = _normalized(f"{entry.get('risk_area', '')} {entry.get('surface', '')}") - return any(all(term in haystack for term in terms) for terms in phrasings) + haystack_words = set(haystack.split()) + return any(all(term in haystack_words for term in terms) for terms in phrasings) def skill_coverage_gaps( From 1e93cce7da1645d1c77dedb39c4f2d83fee48742 Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Thu, 15 Oct 2026 11:34:40 +0800 Subject: [PATCH 2/2] fix: recognize plural file upload coverage --- strix/report/coverage.py | 2 +- tests/test_report_coverage.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/strix/report/coverage.py b/strix/report/coverage.py index 76387e61..9f784391 100644 --- a/strix/report/coverage.py +++ b/strix/report/coverage.py @@ -109,7 +109,7 @@ _SKILL_PHRASINGS: dict[str, tuple[str, ...]] = { "data exposure", ), "insecure_deserialization": ("deserialization",), - "insecure_file_uploads": ("file upload",), + "insecure_file_uploads": ("file upload", "file uploads"), "llm_prompt_injection": ("prompt injection",), "mass_assignment": ("mass assignment", "parameter binding"), "nosql_injection": ("nosql",), diff --git a/tests/test_report_coverage.py b/tests/test_report_coverage.py index 76200edf..bbd53612 100644 --- a/tests/test_report_coverage.py +++ b/tests/test_report_coverage.py @@ -100,6 +100,15 @@ def test_synonym_phrasing_counts_as_recorded_coverage() -> None: assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"] +def test_plural_risk_phrase_counts_as_recorded_coverage() -> None: + doc = _document( + entries=[_entry(risk_area="File uploads")], + agent_graph=_graph(metadata={"agent-1": {"skills": ["insecure_file_uploads"]}}), + ) + + assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"] + + def test_non_risk_skills_carry_no_coverage_obligation() -> None: """Tooling skills describe how an agent works, not what it hunts.""" doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor", "caido"]}})) @@ -154,6 +163,16 @@ def test_unfinished_agent_makes_the_record_partial() -> None: assert "authz-tester" in doc["completeness"]["caveats"][0] +def test_failed_agent_makes_the_record_partial() -> None: + doc = _document( + agent_graph=_graph(statuses={"agent-1": "failed"}), + exit_reason="finished_by_tool", + ) + + assert doc["completeness"]["complete"] is False + assert "authz-tester" in doc["completeness"]["caveats"][0] + + def test_failed_run_status_makes_the_record_partial() -> None: doc = _document( run_record={"run_id": "r1", "status": "failed"},