mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-20 00:11:50 +00:00
ci(issue-classifier): parse only form headings and keep the claude code label
Split the issue body only on headings the two forms actually emit, keep the first value when a heading repeats, cap each field on its own so a long config cannot push the repro out of the model's view, and only treat comments from github-actions[bot] as the template notice. The claude code keyword label the deleted component labeler used to add gets its own small workflow.
This commit is contained in:
parent
294a9e15de
commit
9c0a840122
6 changed files with 128 additions and 15 deletions
2
.github/workflows/issue_classifier.yml
vendored
2
.github/workflows/issue_classifier.yml
vendored
|
|
@ -14,6 +14,8 @@ on:
|
|||
- .github/prompts/issue-classifier.md
|
||||
- .github/prompts/issue-classifier.schema.json
|
||||
- .github/labels.json
|
||||
- .github/ISSUE_TEMPLATE/bug_report.yml
|
||||
- .github/ISSUE_TEMPLATE/feature_request.yml
|
||||
- scripts/classify-issue.ts
|
||||
- scripts/classify-issue.test.ts
|
||||
- scripts/label-issue.ts
|
||||
|
|
|
|||
21
.github/workflows/label_claude_code.yml
vendored
Normal file
21
.github/workflows/label_claude_code.yml
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
name: Label Claude Code issues
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
label:
|
||||
if: github.repository == 'BerriAI/litellm' && contains(github.event.issue.body, 'claude code')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Add the claude code label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ISSUE_URL: ${{ github.event.issue.html_url }}
|
||||
run: gh issue edit "$ISSUE_URL" --add-label "claude code"
|
||||
|
|
@ -4,6 +4,8 @@ import type { GitHubApi } from "./auto-close-duplicates";
|
|||
import {
|
||||
BODY_CAP_CHARS,
|
||||
BUG_SECTIONS,
|
||||
FORM_HEADINGS,
|
||||
SECTION_CAP_CHARS,
|
||||
FEATURE_SECTIONS,
|
||||
MIN_SECTION_CHARS,
|
||||
buildRequest,
|
||||
|
|
@ -95,7 +97,7 @@ describe("the schema and the manifest agree", () => {
|
|||
});
|
||||
|
||||
describe("sections", () => {
|
||||
test("splits an issue form body on its headings and trims each block", () => {
|
||||
test("splits an issue form body on its field headings and trims each block", () => {
|
||||
const found = sections("preamble\n### Description\n\nIt broke.\n\n### Config\n\n_No response_\n");
|
||||
expect([...found.entries()]).toEqual([
|
||||
["Description", "It broke."],
|
||||
|
|
@ -103,8 +105,35 @@ describe("sections", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
test("a heading the reporter typed inside a field stays inside that field", () => {
|
||||
const found = sections(
|
||||
"### Steps to Repro\n\n### Actual response\n\n500 from the proxy\n\n### Expected\n\n200\n\n### LiteLLM Version\n\nv1.100.0\n",
|
||||
);
|
||||
expect(found.get("Steps to Repro")).toBe("### Actual response\n\n500 from the proxy\n\n### Expected\n\n200");
|
||||
expect(found.get("LiteLLM Version")).toBe("v1.100.0");
|
||||
});
|
||||
|
||||
test("a repeated field heading does not overwrite the first value", () => {
|
||||
const found = sections("### Description\n\nreal text\n\n### Config\n\n### Description\n\nnot a field\n");
|
||||
expect(found.get("Description")).toBe("real text");
|
||||
expect(found.get("Config")).toBe("### Description\n\nnot a field");
|
||||
});
|
||||
|
||||
test("a body with no headings has no sections", () => {
|
||||
expect(sections("just some prose with ### inside a line").size).toBe(0);
|
||||
expect(sections("### Open question for OWNER\n\nnot a form field").size).toBe(0);
|
||||
});
|
||||
|
||||
test("the known headings are exactly the field labels of the two issue forms", async () => {
|
||||
const labels = await Promise.all(
|
||||
["bug_report.yml", "feature_request.yml"].map(async (file) => {
|
||||
const form = Bun.YAML.parse(await Bun.file(`${import.meta.dir}/../.github/ISSUE_TEMPLATE/${file}`).text()) as {
|
||||
readonly body: readonly { readonly attributes?: { readonly label?: string } }[];
|
||||
};
|
||||
return form.body.flatMap((field) => (field.attributes?.label === undefined ? [] : [field.attributes.label.trim()]));
|
||||
}),
|
||||
);
|
||||
expect(new Set(labels.flat())).toEqual(new Set(FORM_HEADINGS));
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -213,8 +242,21 @@ describe("buildRequest", () => {
|
|||
expect(message).toContain("### Steps to Repro");
|
||||
});
|
||||
|
||||
test("a long body is capped and the version survives the cap", () => {
|
||||
const body = `${bugBody()}${"x".repeat(BODY_CAP_CHARS * 2)}`;
|
||||
test("each field is capped on its own, so a huge config cannot push the repro out of the message", () => {
|
||||
const message = userMessage(issue({ body: bugBody({ Config: "y".repeat(SECTION_CAP_CHARS * 3) }) }), passed);
|
||||
expect(message).toContain(`[section truncated at ${SECTION_CAP_CHARS} characters]`);
|
||||
expect(message).toContain("### Steps to Repro\n\n1. curl -X POST http://localhost:4000/v1/chat/completions");
|
||||
expect(message.length).toBeLessThan(SECTION_CAP_CHARS + 1500);
|
||||
});
|
||||
|
||||
test("the hiring, contact and duplicate-check fields are left out of the message", () => {
|
||||
const message = userMessage(issue({ title: "[Feature]: scope guardrails", body: featureBody() }), passed);
|
||||
expect(message).toContain("### The Feature");
|
||||
expect(message).not.toContain("Check for existing issues");
|
||||
});
|
||||
|
||||
test("a body without form fields is sent whole, capped, and the version survives the cap", () => {
|
||||
const body = "x".repeat(BODY_CAP_CHARS * 2);
|
||||
const message = userMessage(issue({ body }), passed);
|
||||
expect(message.length).toBeLessThan(BODY_CAP_CHARS + 500);
|
||||
expect(message).toContain(`[body truncated at ${BODY_CAP_CHARS} characters]`);
|
||||
|
|
|
|||
|
|
@ -84,18 +84,39 @@ export const BUG_SECTIONS = ["Description", "Config", "LiteLLM Version", "Steps
|
|||
export const FEATURE_SECTIONS = ["The Feature", "User Flow", "How far you got"] as const;
|
||||
export const DOMAIN_HEADING = "Which part of LiteLLM is this about?";
|
||||
export const VERSION_HEADING = "LiteLLM Version";
|
||||
export const DEPLOYMENT_HEADING = "How are you deploying?";
|
||||
export const NOISE_HEADINGS = [
|
||||
"Check for existing issues",
|
||||
"LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?",
|
||||
"Twitter / LinkedIn details",
|
||||
] as const;
|
||||
export const FORM_HEADINGS: readonly string[] = [
|
||||
...BUG_SECTIONS,
|
||||
...FEATURE_SECTIONS,
|
||||
DOMAIN_HEADING,
|
||||
DEPLOYMENT_HEADING,
|
||||
...NOISE_HEADINGS,
|
||||
];
|
||||
export const MIN_SECTION_CHARS = 20;
|
||||
export const SECTION_CAP_CHARS = 4000;
|
||||
export const BODY_CAP_CHARS = 8000;
|
||||
export const MAINTAINER_ASSOCIATIONS: readonly string[] = ["OWNER", "MEMBER", "COLLABORATOR"];
|
||||
const EMPTY_FIELD = "_No response_";
|
||||
const NOT_SURE = "Not sure";
|
||||
|
||||
type Block = readonly [heading: string, lines: readonly string[]];
|
||||
|
||||
export function sections(body: string): ReadonlyMap<string, string> {
|
||||
const parts = body.split(/^### (.+)$/m).slice(1);
|
||||
const pairs = parts.flatMap((part, index): readonly (readonly [string, string])[] =>
|
||||
index % 2 === 0 ? [[part.trim(), (parts[index + 1] ?? "").trim()]] : [],
|
||||
);
|
||||
return new Map(pairs);
|
||||
const blocks = body.split("\n").reduce<readonly Block[]>((acc, line) => {
|
||||
const heading = /^### (.+?)\s*$/.exec(line)?.[1];
|
||||
const opensField = heading !== undefined && FORM_HEADINGS.includes(heading) && !acc.some(([name]) => name === heading);
|
||||
if (opensField) {
|
||||
return [...acc, [heading, []]];
|
||||
}
|
||||
const current = acc.at(-1);
|
||||
return current === undefined ? acc : [...acc.slice(0, -1), [current[0], [...current[1], line]]];
|
||||
}, []);
|
||||
return new Map(blocks.map(([heading, lines]) => [heading, lines.join("\n").trim()]));
|
||||
}
|
||||
|
||||
export function templateFor(title: string, found: ReadonlyMap<string, string>): Template {
|
||||
|
|
@ -136,12 +157,22 @@ export function gate(issue: Pick<IssueForClassification, "title" | "body" | "aut
|
|||
};
|
||||
}
|
||||
|
||||
const clip = (text: string, cap: number, what: string): string =>
|
||||
text.length > cap ? `${text.slice(0, cap)}\n\n[${what} truncated at ${cap} characters]` : text;
|
||||
|
||||
export function issueText(body: string): string {
|
||||
const found = sections(body);
|
||||
if (found.size === 0) {
|
||||
return clip(body, BODY_CAP_CHARS, "body");
|
||||
}
|
||||
return [...found]
|
||||
.filter(([heading]) => !NOISE_HEADINGS.some((noise) => noise === heading))
|
||||
.map(([heading, text]) => `### ${heading}\n\n${clip(text, SECTION_CAP_CHARS, "section")}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
export function userMessage(issue: Pick<IssueForClassification, "title" | "body">, passed: Gate & { kind: "pass" }): string {
|
||||
const body = issue.body ?? "";
|
||||
const capped =
|
||||
body.length > BODY_CAP_CHARS
|
||||
? `${body.slice(0, BODY_CAP_CHARS)}\n\n[body truncated at ${BODY_CAP_CHARS} characters]`
|
||||
: body;
|
||||
const capped = issueText(issue.body ?? "");
|
||||
const versionLine = passed.version === null ? "" : `\nLiteLLM Version (from the template): ${passed.version}`;
|
||||
return [
|
||||
`Title: ${issue.title}`,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test";
|
|||
import type { Comment, GitHubApi } from "./auto-close-duplicates";
|
||||
import type { Classification, GateVerdict } from "./classify-issue";
|
||||
import {
|
||||
BOT_LOGIN,
|
||||
TEMPLATE_MARKER,
|
||||
desiredLabels,
|
||||
labelIssue,
|
||||
|
|
@ -122,8 +123,9 @@ describe("labelIssue", () => {
|
|||
id: 77,
|
||||
body: templateComment(gated),
|
||||
created_at: "2026-09-10T00:00:00Z",
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
user: { type: "Bot", login: BOT_LOGIN },
|
||||
};
|
||||
const impostor: Comment = { ...notice, id: 78, user: { type: "User", login: "someone" } };
|
||||
|
||||
function fakeApi(
|
||||
labels: readonly string[],
|
||||
|
|
@ -179,6 +181,20 @@ describe("labelIssue", () => {
|
|||
expect(outcome).toEqual({ plan: { add: [], remove: [] }, comment: null, removedNotices: 0 });
|
||||
});
|
||||
|
||||
test("someone else's comment carrying the marker is neither the notice nor deleted", async () => {
|
||||
const gatedRun = fakeApi(["bug"], [impostor]);
|
||||
const outcome = await labelIssue(gatedRun.api, config, gated);
|
||||
expect(outcome.comment).toContain(TEMPLATE_MARKER);
|
||||
expect(gatedRun.writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([
|
||||
"POST /repos/BerriAI/litellm/issues/41700/labels",
|
||||
"POST /repos/BerriAI/litellm/issues/41700/comments",
|
||||
]);
|
||||
|
||||
const passedRun = fakeApi(["needs:template"], [impostor]);
|
||||
await labelIssue(passedRun.api, config, classified());
|
||||
expect(passedRun.writes).not.toContain("DELETE /repos/BerriAI/litellm/issues/comments/78");
|
||||
});
|
||||
|
||||
test("a dry run reports the plan and the comment and touches nothing", async () => {
|
||||
const { api, writes } = fakeApi(["bug"]);
|
||||
const outcome = await labelIssue(api, { ...config, dryRun: true }, gated);
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ export type ParsedVerdict =
|
|||
| { readonly kind: "invalid"; readonly reason: string };
|
||||
|
||||
export const TEMPLATE_MARKER = "<!-- litellm:needs-template -->";
|
||||
export const BOT_LOGIN = "github-actions[bot]";
|
||||
const TEMPLATE_URLS: Readonly<Record<GateVerdict["template"], string>> = {
|
||||
bug: "https://github.com/BerriAI/litellm/issues/new?template=bug_report.yml",
|
||||
feature: "https://github.com/BerriAI/litellm/issues/new?template=feature_request.yml",
|
||||
|
|
@ -109,7 +110,7 @@ export async function labelIssue(api: GitHubApi, config: LabelConfig, verdict: V
|
|||
verdict,
|
||||
);
|
||||
const comments = await listAll<Comment>(api, `${issuePath}/comments`);
|
||||
const notices = comments.filter((comment) => comment.body.includes(TEMPLATE_MARKER));
|
||||
const notices = comments.filter((comment) => comment.user.login === BOT_LOGIN && comment.body.includes(TEMPLATE_MARKER));
|
||||
const comment = verdict.gate === "template" && notices.length === 0 ? templateComment(verdict) : null;
|
||||
const staleNotices = verdict.gate === "pass" ? notices : [];
|
||||
if (config.dryRun) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue