diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index 6499a063b39..e23a012425a 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -597,11 +597,13 @@ def build_issue_prompt(*, title: str, body: str) -> str: that it does not today). - Motivation / use case with a concrete example (config, API call, UI flow, or scenario showing what's blocked today). - - END-TO-END EVIDENCE OF THE DEAD-END: a video, a screenshot, or the - exact command(s) actually run paired with their real output, - showing the point where the flow stops today. Mocked or stubbed - dependencies do NOT count, and an unfilled template scaffold - (bare headings, empty numbered lists) counts as absent. + - END-TO-END EVIDENCE OF THE DEAD-END (set + `has_dead_end_evidence=true` only when this is present): a video, + a screenshot, or the exact command(s) actually run paired with + their real output, showing the point where the flow stops today. + Mocked or stubbed dependencies do NOT count, and an unfilled + template scaffold (bare headings, empty numbered lists) counts as + absent. For an issue that is neither a bug report nor a feature request (a question, support request, or discussion), PASS as long as it has a @@ -615,6 +617,7 @@ def build_issue_prompt(*, title: str, body: str) -> str: "has_repro": boolean, "has_expected_vs_actual": boolean, "has_motivation_example": boolean, + "has_dead_end_evidence": boolean, "missing": ["plain-english strings naming what is missing"], "explanation": "1-2 sentence reasoning for the team to skim" }} @@ -712,6 +715,10 @@ _ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = ( ) _ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = ( ("has_motivation_example", "Motivation and concrete example"), + ( + "has_dead_end_evidence", + "End-to-end evidence of the dead-end (video, screenshot, or command + real output)", + ), ) diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 50b198f62c5..96b77e80457 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -306,6 +306,27 @@ class TestCloseCommentText: assert "Expected vs. actual behavior" in body assert "- ✅ End-to-end evidence of the bug" not in body + def test_issue_close_comment_should_credit_feature_dead_end_evidence( + self, triage_module + ): + # A feature requester who pasted their dead-end run but skipped the + # motivation must see the evidence credited and only the motivation + # listed as a gap — without a dedicated verdict field the praise + # block could never acknowledge the work they did do. + body = triage_module.format_issue_close_comment( + { + "verdict": "fail", + "kind": "feature", + "has_motivation_example": False, + "has_dead_end_evidence": True, + "missing": ["motivation / use case"], + "explanation": "no use case given", + } + ) + assert "What you got right" in body + assert "- ✅ End-to-end evidence of the dead-end" in body + assert "- ✅ Motivation and concrete example" not in body + def test_close_comments_should_use_softer_park_for_later_framing( self, triage_module ): @@ -707,6 +728,11 @@ class TestBuildPrompts: assert "END-TO-END EVIDENCE OF THE DEAD-END" in normalized assert "showing the point where the flow stops today" in normalized assert "unfilled template scaffold" in normalized + # The evidence has its own verdict field so feature requesters who + # provided it get credited in "What you got right", exactly like + # `has_repro` credits bug evidence. + assert "`has_dead_end_evidence=true` only when this is present" in normalized + assert '"has_dead_end_evidence": boolean' in normalized def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module): """User-supplied content with `{` / `}` must NOT be re-parsed by