fix: reject finish report schema placeholders

This commit is contained in:
Seongho Bae 2026-09-13 02:32:56 +09:00
parent 0c4364a6a7
commit ae95d0e355
2 changed files with 46 additions and 9 deletions

View file

@ -33,15 +33,26 @@ def _do_finish(
),
}
report_sections = (
("Executive summary", executive_summary, "Business-level summary for leadership."),
("Methodology", methodology, "Frameworks, scope, and approach."),
(
"Technical analysis",
technical_analysis,
"Consolidated findings + systemic themes.",
),
(
"Recommendations",
recommendations,
"Prioritized, actionable remediation.",
),
)
errors: list[str] = []
if not executive_summary.strip():
errors.append("Executive summary cannot be empty")
if not methodology.strip():
errors.append("Methodology cannot be empty")
if not technical_analysis.strip():
errors.append("Technical analysis cannot be empty")
if not recommendations.strip():
errors.append("Recommendations cannot be empty")
for section_name, section_text, schema_description in report_sections:
if not section_text.strip():
errors.append(f"{section_name} cannot be empty")
elif " ".join(section_text.split()) == schema_description:
errors.append(f"{section_name} cannot be a schema description placeholder")
if errors:
return {"success": False, "error": "Validation failed", "errors": errors}

View file

@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
import pytest
from strix.tools.coverage.tools import _record_impl, hydrate_coverage_from_disk
from strix.tools.finish.tool import _coverage_summary
from strix.tools.finish.tool import _coverage_summary, _do_finish
if TYPE_CHECKING:
@ -63,3 +63,29 @@ def test_an_empty_ledger_still_warns_first() -> None:
assert summary["coverage_recorded"] == 0
assert "No coverage was recorded" in summary["coverage_warning"]
@pytest.mark.parametrize(
("field_name", "placeholder"),
[
("executive_summary", "Business-level summary for leadership."),
("methodology", "Frameworks, scope, and approach."),
("technical_analysis", "Consolidated findings + systemic themes."),
("recommendations", "Prioritized, actionable remediation."),
],
)
def test_finish_rejects_schema_description_placeholders(
field_name: str, placeholder: str
) -> None:
report_fields = {
"executive_summary": "No material exposure was confirmed.",
"methodology": "The scoped paths were reviewed and exercised.",
"technical_analysis": "Observed controls held under the tested cases.",
"recommendations": "Retest after material changes.",
}
report_fields[field_name] = f" \n{placeholder}\t"
result = _do_finish(parent_id=None, agent_graph={}, **report_fields)
assert result["success"] is False
assert "placeholder" in " ".join(result["errors"]).lower()