From 38a1bec5c2de09e722faaf8a498e204d1f24ecd5 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:56:38 +0000 Subject: [PATCH] fix(triage): require evidence of the dead-end in feature requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/scripts/triage_with_llm.py | 5 +++++ tests/test_litellm/test_github_triage_with_llm.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index a7bd145dbc8..0908c3d76cc 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -597,6 +597,11 @@ def build_issue_prompt(*, title: str, body: str) -> str: that it does not today). - Motivation / use case with a concrete example (config, API call, UI flow, or scenario showing what's blocked today). + - END-TO-END EVIDENCE OF THE DEAD-END: a video, a screenshot, or the + exact command(s) actually run paired with their real output, + showing the point where the flow stops today. Mocked or stubbed + dependencies do NOT count, and an unfilled template scaffold + (bare headings, empty numbered lists) counts as absent. For an issue that is neither a bug report nor a feature request (a question, support request, or discussion), PASS as long as it has a diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 300fd7c0710..3172dc93e82 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -678,6 +678,19 @@ class TestBuildPrompts: assert "unfilled template scaffold" in normalized assert "counts as absent, not as evidence" in normalized + def test_issue_feature_rubric_requires_evidence_of_the_dead_end( + self, triage_module + ): + # The feature form asks the requester to walk the ideal flow against a + # live proxy and paste output up to the step that dead-ends, so the + # judge has to demand that evidence, and must not accept an unedited + # scaffold of bare headings as if it were a real attempt. + prompt = triage_module.build_issue_prompt(title="t", body="x") + normalized = " ".join(prompt.split()) + assert "END-TO-END EVIDENCE OF THE DEAD-END" in normalized + assert "showing the point where the flow stops today" in normalized + assert "unfilled template scaffold" in normalized + def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module): """User-supplied content with `{` / `}` must NOT be re-parsed by `str.format()`. `format` only scans the template literal for