mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(triage): track and credit feature dead-end evidence in the verdict
This commit is contained in:
parent
e59da40750
commit
812abcc7f5
2 changed files with 38 additions and 5 deletions
17
.github/scripts/triage_with_llm.py
vendored
17
.github/scripts/triage_with_llm.py
vendored
|
|
@ -597,11 +597,13 @@ def build_issue_prompt(*, title: str, body: str) -> str:
|
|||
that it does not today).
|
||||
- Motivation / use case with a concrete example (config, API call,
|
||||
UI flow, or scenario showing what's blocked today).
|
||||
- END-TO-END EVIDENCE OF THE DEAD-END: a video, a screenshot, or the
|
||||
exact command(s) actually run paired with their real output,
|
||||
showing the point where the flow stops today. Mocked or stubbed
|
||||
dependencies do NOT count, and an unfilled template scaffold
|
||||
(bare headings, empty numbered lists) counts as absent.
|
||||
- END-TO-END EVIDENCE OF THE DEAD-END (set
|
||||
`has_dead_end_evidence=true` only when this is present): a video,
|
||||
a screenshot, or the exact command(s) actually run paired with
|
||||
their real output, showing the point where the flow stops today.
|
||||
Mocked or stubbed dependencies do NOT count, and an unfilled
|
||||
template scaffold (bare headings, empty numbered lists) counts as
|
||||
absent.
|
||||
|
||||
For an issue that is neither a bug report nor a feature request (a
|
||||
question, support request, or discussion), PASS as long as it has a
|
||||
|
|
@ -615,6 +617,7 @@ def build_issue_prompt(*, title: str, body: str) -> str:
|
|||
"has_repro": boolean,
|
||||
"has_expected_vs_actual": boolean,
|
||||
"has_motivation_example": boolean,
|
||||
"has_dead_end_evidence": boolean,
|
||||
"missing": ["plain-english strings naming what is missing"],
|
||||
"explanation": "1-2 sentence reasoning for the team to skim"
|
||||
}}
|
||||
|
|
@ -712,6 +715,10 @@ _ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = (
|
|||
)
|
||||
_ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = (
|
||||
("has_motivation_example", "Motivation and concrete example"),
|
||||
(
|
||||
"has_dead_end_evidence",
|
||||
"End-to-end evidence of the dead-end (video, screenshot, or command + real output)",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -306,6 +306,27 @@ class TestCloseCommentText:
|
|||
assert "Expected vs. actual behavior" in body
|
||||
assert "- ✅ End-to-end evidence of the bug" not in body
|
||||
|
||||
def test_issue_close_comment_should_credit_feature_dead_end_evidence(
|
||||
self, triage_module
|
||||
):
|
||||
# A feature requester who pasted their dead-end run but skipped the
|
||||
# motivation must see the evidence credited and only the motivation
|
||||
# listed as a gap — without a dedicated verdict field the praise
|
||||
# block could never acknowledge the work they did do.
|
||||
body = triage_module.format_issue_close_comment(
|
||||
{
|
||||
"verdict": "fail",
|
||||
"kind": "feature",
|
||||
"has_motivation_example": False,
|
||||
"has_dead_end_evidence": True,
|
||||
"missing": ["motivation / use case"],
|
||||
"explanation": "no use case given",
|
||||
}
|
||||
)
|
||||
assert "What you got right" in body
|
||||
assert "- ✅ End-to-end evidence of the dead-end" in body
|
||||
assert "- ✅ Motivation and concrete example" not in body
|
||||
|
||||
def test_close_comments_should_use_softer_park_for_later_framing(
|
||||
self, triage_module
|
||||
):
|
||||
|
|
@ -707,6 +728,11 @@ class TestBuildPrompts:
|
|||
assert "END-TO-END EVIDENCE OF THE DEAD-END" in normalized
|
||||
assert "showing the point where the flow stops today" in normalized
|
||||
assert "unfilled template scaffold" in normalized
|
||||
# The evidence has its own verdict field so feature requesters who
|
||||
# provided it get credited in "What you got right", exactly like
|
||||
# `has_repro` credits bug evidence.
|
||||
assert "`has_dead_end_evidence=true` only when this is present" in normalized
|
||||
assert '"has_dead_end_evidence": boolean' in normalized
|
||||
|
||||
def test_should_not_crash_when_pr_body_contains_curly_braces(self, triage_module):
|
||||
"""User-supplied content with `{` / `}` must NOT be re-parsed by
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue