scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix(triage): ask for dead-end evidence in feature request recovery comments

Browse source
This commit is contained in:
mateo-berri 2026-08-11 10:43:16 -07:00
parent d4468ba63a
commit e59da40750
2 changed files with 26 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
.github/scripts/triage_with_llm.py vendored
View file

@ -843,8 +843,11 @@ def format_issue_close_comment(verdict: dict) -> str:
"video, a screenshot, or the exact commands you ran with their real output / "
"traceback) plus expected vs. actual behavior. Written steps with no run output, "
"video, or screenshot don't count, and mocked or stubbed runs don't count.\n"
" - For **feature requests**: a concrete description of what should change, plus a "
"use case and example (config / API call / UI flow).\n"
" - For **feature requests**: a concrete description of what should change, a "
"use case and example (config / API call / UI flow), plus end-to-end evidence of "
"the dead-end (a video, a screenshot, or the exact commands you ran with their "
"real output showing where the flow stops today). Mocked or stubbed runs don't "
"count.\n"
"2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it "
"now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer "
"or bot closed, so the comment-based reconsider is the reliable path.)\n"
@ -950,8 +953,10 @@ def format_grace_warning_issue_comment(verdict: dict) -> str:
"screenshot, or the exact commands you ran with their real output / traceback) plus "
"expected vs. actual behavior. Written steps with no run output don't count, and "
"mocked or stubbed runs don't count.\n"
"- For **feature requests**: a concrete description of what should change, plus a use "
"case and example (config / API call / UI flow).\n"
"- For **feature requests**: a concrete description of what should change, a use "
"case and example (config / API call / UI flow), plus end-to-end evidence of the "
"dead-end (a video, a screenshot, or the exact commands you ran with their real "
"output showing where the flow stops today). Mocked or stubbed runs don't count.\n"
"\n"
"**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` "
"and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n"

17
tests/test_litellm/test_github_triage_with_llm.py
View file

@ -207,6 +207,23 @@ class TestCloseCommentText:
assert "end-to-end qa proof" in body.lower()
assert "mock" in body.lower()
def test_issue_recovery_comments_should_name_feature_dead_end_evidence(
self, triage_module
):
# The feature-request pass bar demands end-to-end evidence of the
# dead-end, so the close and grace-warning recovery bullets must ask
# for it too — otherwise a requester follows those exact instructions
# (description + use case only) and fails `reconsider` again with no
# hint of what else was needed.
verdict = {"verdict": "fail", "missing": [], "explanation": ""}
for body in (
triage_module.format_issue_close_comment(verdict),
triage_module.format_grace_warning_issue_comment(verdict),
):
normalized = " ".join(body.split())
assert "end-to-end evidence of the dead-end" in normalized
assert "showing where the flow stops today" in normalized
def test_all_agent_shin_comments_should_use_bullet_train_emoji(self, triage_module):
# The bullet train (🚅) is Agent Shin's symbol, matching the LiteLLM
# logo; the previous wave (👋) was generic and didn't match the bot's