diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index 0908c3d76cc..6499a063b39 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -843,8 +843,11 @@ def format_issue_close_comment(verdict: dict) -> str: "video, a screenshot, or the exact commands you ran with their real output / " "traceback) plus expected vs. actual behavior. Written steps with no run output, " "video, or screenshot don't count, and mocked or stubbed runs don't count.\n" - " - For **feature requests**: a concrete description of what should change, plus a " - "use case and example (config / API call / UI flow).\n" + " - For **feature requests**: a concrete description of what should change, a " + "use case and example (config / API call / UI flow), plus end-to-end evidence of " + "the dead-end (a video, a screenshot, or the exact commands you ran with their " + "real output showing where the flow stops today). Mocked or stubbed runs don't " + "count.\n" "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it " "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer " "or bot closed, so the comment-based reconsider is the reliable path.)\n" @@ -950,8 +953,10 @@ def format_grace_warning_issue_comment(verdict: dict) -> str: "screenshot, or the exact commands you ran with their real output / traceback) plus " "expected vs. actual behavior. Written steps with no run output don't count, and " "mocked or stubbed runs don't count.\n" - "- For **feature requests**: a concrete description of what should change, plus a use " - "case and example (config / API call / UI flow).\n" + "- For **feature requests**: a concrete description of what should change, a use " + "case and example (config / API call / UI flow), plus end-to-end evidence of the " + "dead-end (a video, a screenshot, or the exact commands you ran with their real " + "output showing where the flow stops today). Mocked or stubbed runs don't count.\n" "\n" "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` " "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n" diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 3172dc93e82..50b198f62c5 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -207,6 +207,23 @@ class TestCloseCommentText: assert "end-to-end qa proof" in body.lower() assert "mock" in body.lower() + def test_issue_recovery_comments_should_name_feature_dead_end_evidence( + self, triage_module + ): + # The feature-request pass bar demands end-to-end evidence of the + # dead-end, so the close and grace-warning recovery bullets must ask + # for it too — otherwise a requester follows those exact instructions + # (description + use case only) and fails `reconsider` again with no + # hint of what else was needed. + verdict = {"verdict": "fail", "missing": [], "explanation": ""} + for body in ( + triage_module.format_issue_close_comment(verdict), + triage_module.format_grace_warning_issue_comment(verdict), + ): + normalized = " ".join(body.split()) + assert "end-to-end evidence of the dead-end" in normalized + assert "showing where the flow stops today" in normalized + def test_all_agent_shin_comments_should_use_bullet_train_emoji(self, triage_module): # The bullet train (🚅) is Agent Shin's symbol, matching the LiteLLM # logo; the previous wave (👋) was generic and didn't match the bot's