From 9c0a840122ba503308c9b913035cb24ee5528a62 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 17 Sep 2026 16:31:22 -0700 Subject: [PATCH] ci(issue-classifier): parse only form headings and keep the claude code label Split the issue body only on headings the two forms actually emit, keep the first value when a heading repeats, cap each field on its own so a long config cannot push the repro out of the model's view, and only treat comments from github-actions[bot] as the template notice. The claude code keyword label the deleted component labeler used to add gets its own small workflow. --- .github/workflows/issue_classifier.yml | 2 + .github/workflows/label_claude_code.yml | 21 ++++++++++ scripts/classify-issue.test.ts | 48 +++++++++++++++++++++-- scripts/classify-issue.ts | 51 ++++++++++++++++++++----- scripts/label-issue.test.ts | 18 ++++++++- scripts/label-issue.ts | 3 +- 6 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/label_claude_code.yml diff --git a/.github/workflows/issue_classifier.yml b/.github/workflows/issue_classifier.yml index 8f4a094e75e..8dce1663ef3 100644 --- a/.github/workflows/issue_classifier.yml +++ b/.github/workflows/issue_classifier.yml @@ -14,6 +14,8 @@ on: - .github/prompts/issue-classifier.md - .github/prompts/issue-classifier.schema.json - .github/labels.json + - .github/ISSUE_TEMPLATE/bug_report.yml + - .github/ISSUE_TEMPLATE/feature_request.yml - scripts/classify-issue.ts - scripts/classify-issue.test.ts - scripts/label-issue.ts diff --git a/.github/workflows/label_claude_code.yml b/.github/workflows/label_claude_code.yml new file mode 100644 index 00000000000..aa0c78addbc --- /dev/null +++ b/.github/workflows/label_claude_code.yml @@ -0,0 +1,21 @@ +name: Label Claude Code issues + +on: + issues: + types: [opened] + +permissions: {} + +jobs: + label: + if: github.repository == 'BerriAI/litellm' && contains(github.event.issue.body, 'claude code') + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: + issues: write + steps: + - name: Add the claude code label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_URL: ${{ github.event.issue.html_url }} + run: gh issue edit "$ISSUE_URL" --add-label "claude code" diff --git a/scripts/classify-issue.test.ts b/scripts/classify-issue.test.ts index e7532fc509c..f980496e9c9 100644 --- a/scripts/classify-issue.test.ts +++ b/scripts/classify-issue.test.ts @@ -4,6 +4,8 @@ import type { GitHubApi } from "./auto-close-duplicates"; import { BODY_CAP_CHARS, BUG_SECTIONS, + FORM_HEADINGS, + SECTION_CAP_CHARS, FEATURE_SECTIONS, MIN_SECTION_CHARS, buildRequest, @@ -95,7 +97,7 @@ describe("the schema and the manifest agree", () => { }); describe("sections", () => { - test("splits an issue form body on its headings and trims each block", () => { + test("splits an issue form body on its field headings and trims each block", () => { const found = sections("preamble\n### Description\n\nIt broke.\n\n### Config\n\n_No response_\n"); expect([...found.entries()]).toEqual([ ["Description", "It broke."], @@ -103,8 +105,35 @@ describe("sections", () => { ]); }); + test("a heading the reporter typed inside a field stays inside that field", () => { + const found = sections( + "### Steps to Repro\n\n### Actual response\n\n500 from the proxy\n\n### Expected\n\n200\n\n### LiteLLM Version\n\nv1.100.0\n", + ); + expect(found.get("Steps to Repro")).toBe("### Actual response\n\n500 from the proxy\n\n### Expected\n\n200"); + expect(found.get("LiteLLM Version")).toBe("v1.100.0"); + }); + + test("a repeated field heading does not overwrite the first value", () => { + const found = sections("### Description\n\nreal text\n\n### Config\n\n### Description\n\nnot a field\n"); + expect(found.get("Description")).toBe("real text"); + expect(found.get("Config")).toBe("### Description\n\nnot a field"); + }); + test("a body with no headings has no sections", () => { expect(sections("just some prose with ### inside a line").size).toBe(0); + expect(sections("### Open question for OWNER\n\nnot a form field").size).toBe(0); + }); + + test("the known headings are exactly the field labels of the two issue forms", async () => { + const labels = await Promise.all( + ["bug_report.yml", "feature_request.yml"].map(async (file) => { + const form = Bun.YAML.parse(await Bun.file(`${import.meta.dir}/../.github/ISSUE_TEMPLATE/${file}`).text()) as { + readonly body: readonly { readonly attributes?: { readonly label?: string } }[]; + }; + return form.body.flatMap((field) => (field.attributes?.label === undefined ? [] : [field.attributes.label.trim()])); + }), + ); + expect(new Set(labels.flat())).toEqual(new Set(FORM_HEADINGS)); }); }); @@ -213,8 +242,21 @@ describe("buildRequest", () => { expect(message).toContain("### Steps to Repro"); }); - test("a long body is capped and the version survives the cap", () => { - const body = `${bugBody()}${"x".repeat(BODY_CAP_CHARS * 2)}`; + test("each field is capped on its own, so a huge config cannot push the repro out of the message", () => { + const message = userMessage(issue({ body: bugBody({ Config: "y".repeat(SECTION_CAP_CHARS * 3) }) }), passed); + expect(message).toContain(`[section truncated at ${SECTION_CAP_CHARS} characters]`); + expect(message).toContain("### Steps to Repro\n\n1. curl -X POST http://localhost:4000/v1/chat/completions"); + expect(message.length).toBeLessThan(SECTION_CAP_CHARS + 1500); + }); + + test("the hiring, contact and duplicate-check fields are left out of the message", () => { + const message = userMessage(issue({ title: "[Feature]: scope guardrails", body: featureBody() }), passed); + expect(message).toContain("### The Feature"); + expect(message).not.toContain("Check for existing issues"); + }); + + test("a body without form fields is sent whole, capped, and the version survives the cap", () => { + const body = "x".repeat(BODY_CAP_CHARS * 2); const message = userMessage(issue({ body }), passed); expect(message.length).toBeLessThan(BODY_CAP_CHARS + 500); expect(message).toContain(`[body truncated at ${BODY_CAP_CHARS} characters]`); diff --git a/scripts/classify-issue.ts b/scripts/classify-issue.ts index 5ee0fff388b..fb6540cff0d 100644 --- a/scripts/classify-issue.ts +++ b/scripts/classify-issue.ts @@ -84,18 +84,39 @@ export const BUG_SECTIONS = ["Description", "Config", "LiteLLM Version", "Steps export const FEATURE_SECTIONS = ["The Feature", "User Flow", "How far you got"] as const; export const DOMAIN_HEADING = "Which part of LiteLLM is this about?"; export const VERSION_HEADING = "LiteLLM Version"; +export const DEPLOYMENT_HEADING = "How are you deploying?"; +export const NOISE_HEADINGS = [ + "Check for existing issues", + "LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?", + "Twitter / LinkedIn details", +] as const; +export const FORM_HEADINGS: readonly string[] = [ + ...BUG_SECTIONS, + ...FEATURE_SECTIONS, + DOMAIN_HEADING, + DEPLOYMENT_HEADING, + ...NOISE_HEADINGS, +]; export const MIN_SECTION_CHARS = 20; +export const SECTION_CAP_CHARS = 4000; export const BODY_CAP_CHARS = 8000; export const MAINTAINER_ASSOCIATIONS: readonly string[] = ["OWNER", "MEMBER", "COLLABORATOR"]; const EMPTY_FIELD = "_No response_"; const NOT_SURE = "Not sure"; +type Block = readonly [heading: string, lines: readonly string[]]; + export function sections(body: string): ReadonlyMap { - const parts = body.split(/^### (.+)$/m).slice(1); - const pairs = parts.flatMap((part, index): readonly (readonly [string, string])[] => - index % 2 === 0 ? [[part.trim(), (parts[index + 1] ?? "").trim()]] : [], - ); - return new Map(pairs); + const blocks = body.split("\n").reduce((acc, line) => { + const heading = /^### (.+?)\s*$/.exec(line)?.[1]; + const opensField = heading !== undefined && FORM_HEADINGS.includes(heading) && !acc.some(([name]) => name === heading); + if (opensField) { + return [...acc, [heading, []]]; + } + const current = acc.at(-1); + return current === undefined ? acc : [...acc.slice(0, -1), [current[0], [...current[1], line]]]; + }, []); + return new Map(blocks.map(([heading, lines]) => [heading, lines.join("\n").trim()])); } export function templateFor(title: string, found: ReadonlyMap): Template { @@ -136,12 +157,22 @@ export function gate(issue: Pick + text.length > cap ? `${text.slice(0, cap)}\n\n[${what} truncated at ${cap} characters]` : text; + +export function issueText(body: string): string { + const found = sections(body); + if (found.size === 0) { + return clip(body, BODY_CAP_CHARS, "body"); + } + return [...found] + .filter(([heading]) => !NOISE_HEADINGS.some((noise) => noise === heading)) + .map(([heading, text]) => `### ${heading}\n\n${clip(text, SECTION_CAP_CHARS, "section")}`) + .join("\n\n"); +} + export function userMessage(issue: Pick, passed: Gate & { kind: "pass" }): string { - const body = issue.body ?? ""; - const capped = - body.length > BODY_CAP_CHARS - ? `${body.slice(0, BODY_CAP_CHARS)}\n\n[body truncated at ${BODY_CAP_CHARS} characters]` - : body; + const capped = issueText(issue.body ?? ""); const versionLine = passed.version === null ? "" : `\nLiteLLM Version (from the template): ${passed.version}`; return [ `Title: ${issue.title}`, diff --git a/scripts/label-issue.test.ts b/scripts/label-issue.test.ts index 89397b82d32..e24d7728a61 100644 --- a/scripts/label-issue.test.ts +++ b/scripts/label-issue.test.ts @@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test"; import type { Comment, GitHubApi } from "./auto-close-duplicates"; import type { Classification, GateVerdict } from "./classify-issue"; import { + BOT_LOGIN, TEMPLATE_MARKER, desiredLabels, labelIssue, @@ -122,8 +123,9 @@ describe("labelIssue", () => { id: 77, body: templateComment(gated), created_at: "2026-09-10T00:00:00Z", - user: { type: "Bot", login: "github-actions[bot]" }, + user: { type: "Bot", login: BOT_LOGIN }, }; + const impostor: Comment = { ...notice, id: 78, user: { type: "User", login: "someone" } }; function fakeApi( labels: readonly string[], @@ -179,6 +181,20 @@ describe("labelIssue", () => { expect(outcome).toEqual({ plan: { add: [], remove: [] }, comment: null, removedNotices: 0 }); }); + test("someone else's comment carrying the marker is neither the notice nor deleted", async () => { + const gatedRun = fakeApi(["bug"], [impostor]); + const outcome = await labelIssue(gatedRun.api, config, gated); + expect(outcome.comment).toContain(TEMPLATE_MARKER); + expect(gatedRun.writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([ + "POST /repos/BerriAI/litellm/issues/41700/labels", + "POST /repos/BerriAI/litellm/issues/41700/comments", + ]); + + const passedRun = fakeApi(["needs:template"], [impostor]); + await labelIssue(passedRun.api, config, classified()); + expect(passedRun.writes).not.toContain("DELETE /repos/BerriAI/litellm/issues/comments/78"); + }); + test("a dry run reports the plan and the comment and touches nothing", async () => { const { api, writes } = fakeApi(["bug"]); const outcome = await labelIssue(api, { ...config, dryRun: true }, gated); diff --git a/scripts/label-issue.ts b/scripts/label-issue.ts index 1eb5b51e3da..05a933755df 100644 --- a/scripts/label-issue.ts +++ b/scripts/label-issue.ts @@ -28,6 +28,7 @@ export type ParsedVerdict = | { readonly kind: "invalid"; readonly reason: string }; export const TEMPLATE_MARKER = ""; +export const BOT_LOGIN = "github-actions[bot]"; const TEMPLATE_URLS: Readonly> = { bug: "https://github.com/BerriAI/litellm/issues/new?template=bug_report.yml", feature: "https://github.com/BerriAI/litellm/issues/new?template=feature_request.yml", @@ -109,7 +110,7 @@ export async function labelIssue(api: GitHubApi, config: LabelConfig, verdict: V verdict, ); const comments = await listAll(api, `${issuePath}/comments`); - const notices = comments.filter((comment) => comment.body.includes(TEMPLATE_MARKER)); + const notices = comments.filter((comment) => comment.user.login === BOT_LOGIN && comment.body.includes(TEMPLATE_MARKER)); const comment = verdict.gate === "template" && notices.length === 0 ? templateComment(verdict) : null; const staleNotices = verdict.gate === "pass" ? notices : []; if (config.dryRun) {