diff --git a/strix/report/coverage.py b/strix/report/coverage.py index c519853a..9f784391 100644 --- a/strix/report/coverage.py +++ b/strix/report/coverage.py @@ -55,7 +55,7 @@ OUTCOME_LABELS: dict[str, str] = { } #: Statuses that mean the agent stopped early rather than finishing its task. -_INCOMPLETE_AGENT_STATUSES = frozenset({"crashed", "stopped", "running", "waiting"}) +_INCOMPLETE_AGENT_STATUSES = frozenset({"crashed", "failed", "stopped", "running", "waiting"}) #: Run statuses that mean the scan itself did not run to completion. _INCOMPLETE_RUN_STATUSES = frozenset({"failed", "interrupted", "stopped", "running"}) @@ -109,7 +109,7 @@ _SKILL_PHRASINGS: dict[str, tuple[str, ...]] = { "data exposure", ), "insecure_deserialization": ("deserialization",), - "insecure_file_uploads": ("file upload",), + "insecure_file_uploads": ("file upload", "file uploads"), "llm_prompt_injection": ("prompt injection",), "mass_assignment": ("mass assignment", "parameter binding"), "nosql_injection": ("nosql",), @@ -224,7 +224,8 @@ def _skill_phrasings(skill: str) -> list[list[str]]: def _entry_is_about(entry: dict[str, Any], phrasings: list[list[str]]) -> bool: """True when a ledger row plausibly concerns any phrasing of a risk class.""" haystack = _normalized(f"{entry.get('risk_area', '')} {entry.get('surface', '')}") - return any(all(term in haystack for term in terms) for terms in phrasings) + haystack_words = set(haystack.split()) + return any(all(term in haystack_words for term in terms) for terms in phrasings) def skill_coverage_gaps( diff --git a/tests/test_report_coverage.py b/tests/test_report_coverage.py index 76200edf..bbd53612 100644 --- a/tests/test_report_coverage.py +++ b/tests/test_report_coverage.py @@ -100,6 +100,15 @@ def test_synonym_phrasing_counts_as_recorded_coverage() -> None: assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"] +def test_plural_risk_phrase_counts_as_recorded_coverage() -> None: + doc = _document( + entries=[_entry(risk_area="File uploads")], + agent_graph=_graph(metadata={"agent-1": {"skills": ["insecure_file_uploads"]}}), + ) + + assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"] + + def test_non_risk_skills_carry_no_coverage_obligation() -> None: """Tooling skills describe how an agent works, not what it hunts.""" doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor", "caido"]}})) @@ -154,6 +163,16 @@ def test_unfinished_agent_makes_the_record_partial() -> None: assert "authz-tester" in doc["completeness"]["caveats"][0] +def test_failed_agent_makes_the_record_partial() -> None: + doc = _document( + agent_graph=_graph(statuses={"agent-1": "failed"}), + exit_reason="finished_by_tool", + ) + + assert doc["completeness"]["complete"] is False + assert "authz-tester" in doc["completeness"]["caveats"][0] + + def test_failed_run_status_makes_the_record_partial() -> None: doc = _document( run_record={"run_id": "r1", "status": "failed"},