ci(issue-classifier): parse only form headings and keep the claude code label

Split the issue body only on headings the two forms actually emit, keep the
first value when a heading repeats, cap each field on its own so a long config
cannot push the repro out of the model's view, and only treat comments from
github-actions[bot] as the template notice. The claude code keyword label the
deleted component labeler used to add gets its own small workflow.
This commit is contained in:
ryan-crabbe-berri 2026-09-17 16:31:22 -07:00
parent 294a9e15de
commit 9c0a840122
6 changed files with 128 additions and 15 deletions

View file

@ -14,6 +14,8 @@ on:
- .github/prompts/issue-classifier.md
- .github/prompts/issue-classifier.schema.json
- .github/labels.json
- .github/ISSUE_TEMPLATE/bug_report.yml
- .github/ISSUE_TEMPLATE/feature_request.yml
- scripts/classify-issue.ts
- scripts/classify-issue.test.ts
- scripts/label-issue.ts

21
.github/workflows/label_claude_code.yml vendored Normal file
View file

@ -0,0 +1,21 @@
name: Label Claude Code issues
on:
issues:
types: [opened]
permissions: {}
jobs:
label:
if: github.repository == 'BerriAI/litellm' && contains(github.event.issue.body, 'claude code')
runs-on: ubuntu-latest
timeout-minutes: 2
permissions:
issues: write
steps:
- name: Add the claude code label
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ISSUE_URL: ${{ github.event.issue.html_url }}
run: gh issue edit "$ISSUE_URL" --add-label "claude code"

View file

@ -4,6 +4,8 @@ import type { GitHubApi } from "./auto-close-duplicates";
import {
BODY_CAP_CHARS,
BUG_SECTIONS,
FORM_HEADINGS,
SECTION_CAP_CHARS,
FEATURE_SECTIONS,
MIN_SECTION_CHARS,
buildRequest,
@ -95,7 +97,7 @@ describe("the schema and the manifest agree", () => {
});
describe("sections", () => {
test("splits an issue form body on its headings and trims each block", () => {
test("splits an issue form body on its field headings and trims each block", () => {
const found = sections("preamble\n### Description\n\nIt broke.\n\n### Config\n\n_No response_\n");
expect([...found.entries()]).toEqual([
["Description", "It broke."],
@ -103,8 +105,35 @@ describe("sections", () => {
]);
});
test("a heading the reporter typed inside a field stays inside that field", () => {
const found = sections(
"### Steps to Repro\n\n### Actual response\n\n500 from the proxy\n\n### Expected\n\n200\n\n### LiteLLM Version\n\nv1.100.0\n",
);
expect(found.get("Steps to Repro")).toBe("### Actual response\n\n500 from the proxy\n\n### Expected\n\n200");
expect(found.get("LiteLLM Version")).toBe("v1.100.0");
});
test("a repeated field heading does not overwrite the first value", () => {
const found = sections("### Description\n\nreal text\n\n### Config\n\n### Description\n\nnot a field\n");
expect(found.get("Description")).toBe("real text");
expect(found.get("Config")).toBe("### Description\n\nnot a field");
});
test("a body with no headings has no sections", () => {
expect(sections("just some prose with ### inside a line").size).toBe(0);
expect(sections("### Open question for OWNER\n\nnot a form field").size).toBe(0);
});
test("the known headings are exactly the field labels of the two issue forms", async () => {
const labels = await Promise.all(
["bug_report.yml", "feature_request.yml"].map(async (file) => {
const form = Bun.YAML.parse(await Bun.file(`${import.meta.dir}/../.github/ISSUE_TEMPLATE/${file}`).text()) as {
readonly body: readonly { readonly attributes?: { readonly label?: string } }[];
};
return form.body.flatMap((field) => (field.attributes?.label === undefined ? [] : [field.attributes.label.trim()]));
}),
);
expect(new Set(labels.flat())).toEqual(new Set(FORM_HEADINGS));
});
});
@ -213,8 +242,21 @@ describe("buildRequest", () => {
expect(message).toContain("### Steps to Repro");
});
test("a long body is capped and the version survives the cap", () => {
const body = `${bugBody()}${"x".repeat(BODY_CAP_CHARS * 2)}`;
test("each field is capped on its own, so a huge config cannot push the repro out of the message", () => {
const message = userMessage(issue({ body: bugBody({ Config: "y".repeat(SECTION_CAP_CHARS * 3) }) }), passed);
expect(message).toContain(`[section truncated at ${SECTION_CAP_CHARS} characters]`);
expect(message).toContain("### Steps to Repro\n\n1. curl -X POST http://localhost:4000/v1/chat/completions");
expect(message.length).toBeLessThan(SECTION_CAP_CHARS + 1500);
});
test("the hiring, contact and duplicate-check fields are left out of the message", () => {
const message = userMessage(issue({ title: "[Feature]: scope guardrails", body: featureBody() }), passed);
expect(message).toContain("### The Feature");
expect(message).not.toContain("Check for existing issues");
});
test("a body without form fields is sent whole, capped, and the version survives the cap", () => {
const body = "x".repeat(BODY_CAP_CHARS * 2);
const message = userMessage(issue({ body }), passed);
expect(message.length).toBeLessThan(BODY_CAP_CHARS + 500);
expect(message).toContain(`[body truncated at ${BODY_CAP_CHARS} characters]`);

View file

@ -84,18 +84,39 @@ export const BUG_SECTIONS = ["Description", "Config", "LiteLLM Version", "Steps
export const FEATURE_SECTIONS = ["The Feature", "User Flow", "How far you got"] as const;
export const DOMAIN_HEADING = "Which part of LiteLLM is this about?";
export const VERSION_HEADING = "LiteLLM Version";
export const DEPLOYMENT_HEADING = "How are you deploying?";
export const NOISE_HEADINGS = [
"Check for existing issues",
"LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?",
"Twitter / LinkedIn details",
] as const;
export const FORM_HEADINGS: readonly string[] = [
...BUG_SECTIONS,
...FEATURE_SECTIONS,
DOMAIN_HEADING,
DEPLOYMENT_HEADING,
...NOISE_HEADINGS,
];
export const MIN_SECTION_CHARS = 20;
export const SECTION_CAP_CHARS = 4000;
export const BODY_CAP_CHARS = 8000;
export const MAINTAINER_ASSOCIATIONS: readonly string[] = ["OWNER", "MEMBER", "COLLABORATOR"];
const EMPTY_FIELD = "_No response_";
const NOT_SURE = "Not sure";
type Block = readonly [heading: string, lines: readonly string[]];
export function sections(body: string): ReadonlyMap<string, string> {
const parts = body.split(/^### (.+)$/m).slice(1);
const pairs = parts.flatMap((part, index): readonly (readonly [string, string])[] =>
index % 2 === 0 ? [[part.trim(), (parts[index + 1] ?? "").trim()]] : [],
);
return new Map(pairs);
const blocks = body.split("\n").reduce<readonly Block[]>((acc, line) => {
const heading = /^### (.+?)\s*$/.exec(line)?.[1];
const opensField = heading !== undefined && FORM_HEADINGS.includes(heading) && !acc.some(([name]) => name === heading);
if (opensField) {
return [...acc, [heading, []]];
}
const current = acc.at(-1);
return current === undefined ? acc : [...acc.slice(0, -1), [current[0], [...current[1], line]]];
}, []);
return new Map(blocks.map(([heading, lines]) => [heading, lines.join("\n").trim()]));
}
export function templateFor(title: string, found: ReadonlyMap<string, string>): Template {
@ -136,12 +157,22 @@ export function gate(issue: Pick<IssueForClassification, "title" | "body" | "aut
};
}
const clip = (text: string, cap: number, what: string): string =>
text.length > cap ? `${text.slice(0, cap)}\n\n[${what} truncated at ${cap} characters]` : text;
export function issueText(body: string): string {
const found = sections(body);
if (found.size === 0) {
return clip(body, BODY_CAP_CHARS, "body");
}
return [...found]
.filter(([heading]) => !NOISE_HEADINGS.some((noise) => noise === heading))
.map(([heading, text]) => `### ${heading}\n\n${clip(text, SECTION_CAP_CHARS, "section")}`)
.join("\n\n");
}
export function userMessage(issue: Pick<IssueForClassification, "title" | "body">, passed: Gate & { kind: "pass" }): string {
const body = issue.body ?? "";
const capped =
body.length > BODY_CAP_CHARS
? `${body.slice(0, BODY_CAP_CHARS)}\n\n[body truncated at ${BODY_CAP_CHARS} characters]`
: body;
const capped = issueText(issue.body ?? "");
const versionLine = passed.version === null ? "" : `\nLiteLLM Version (from the template): ${passed.version}`;
return [
`Title: ${issue.title}`,

View file

@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test";
import type { Comment, GitHubApi } from "./auto-close-duplicates";
import type { Classification, GateVerdict } from "./classify-issue";
import {
BOT_LOGIN,
TEMPLATE_MARKER,
desiredLabels,
labelIssue,
@ -122,8 +123,9 @@ describe("labelIssue", () => {
id: 77,
body: templateComment(gated),
created_at: "2026-09-10T00:00:00Z",
user: { type: "Bot", login: "github-actions[bot]" },
user: { type: "Bot", login: BOT_LOGIN },
};
const impostor: Comment = { ...notice, id: 78, user: { type: "User", login: "someone" } };
function fakeApi(
labels: readonly string[],
@ -179,6 +181,20 @@ describe("labelIssue", () => {
expect(outcome).toEqual({ plan: { add: [], remove: [] }, comment: null, removedNotices: 0 });
});
test("someone else's comment carrying the marker is neither the notice nor deleted", async () => {
const gatedRun = fakeApi(["bug"], [impostor]);
const outcome = await labelIssue(gatedRun.api, config, gated);
expect(outcome.comment).toContain(TEMPLATE_MARKER);
expect(gatedRun.writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([
"POST /repos/BerriAI/litellm/issues/41700/labels",
"POST /repos/BerriAI/litellm/issues/41700/comments",
]);
const passedRun = fakeApi(["needs:template"], [impostor]);
await labelIssue(passedRun.api, config, classified());
expect(passedRun.writes).not.toContain("DELETE /repos/BerriAI/litellm/issues/comments/78");
});
test("a dry run reports the plan and the comment and touches nothing", async () => {
const { api, writes } = fakeApi(["bug"]);
const outcome = await labelIssue(api, { ...config, dryRun: true }, gated);

View file

@ -28,6 +28,7 @@ export type ParsedVerdict =
| { readonly kind: "invalid"; readonly reason: string };
export const TEMPLATE_MARKER = "<!-- litellm:needs-template -->";
export const BOT_LOGIN = "github-actions[bot]";
const TEMPLATE_URLS: Readonly<Record<GateVerdict["template"], string>> = {
bug: "https://github.com/BerriAI/litellm/issues/new?template=bug_report.yml",
feature: "https://github.com/BerriAI/litellm/issues/new?template=feature_request.yml",
@ -109,7 +110,7 @@ export async function labelIssue(api: GitHubApi, config: LabelConfig, verdict: V
verdict,
);
const comments = await listAll<Comment>(api, `${issuePath}/comments`);
const notices = comments.filter((comment) => comment.body.includes(TEMPLATE_MARKER));
const notices = comments.filter((comment) => comment.user.login === BOT_LOGIN && comment.body.includes(TEMPLATE_MARKER));
const comment = verdict.gate === "template" && notices.length === 0 ? templateComment(verdict) : null;
const staleNotices = verdict.gate === "pass" ? notices : [];
if (config.dryRun) {