From e37bd7c60b063b5b9c84b7cd0d8ec45d841f51d5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 17 Sep 2026 16:45:50 -0700 Subject: [PATCH] fix(issue-classifier): queue runs per issue and judge edits by live labels An edit during the first run used to cancel it, and the edited run then skipped because the webhook payload had no needs:template yet, so a well-formed issue edited within the first minute was never labelled. Runs for one issue now queue, and the script decides an edited event against the live labels: a domain label means leave it alone, a gated issue is re-run, and an unlabelled issue is re-run for its first hour --- .github/workflows/issue_classifier.yml | 12 ++-- scripts/classify-issue.test.ts | 90 +++++++++++++++++++++++--- scripts/classify-issue.ts | 32 +++++++-- 3 files changed, 113 insertions(+), 21 deletions(-) diff --git a/.github/workflows/issue_classifier.yml b/.github/workflows/issue_classifier.yml index 8dce1663ef3..f84934aae4b 100644 --- a/.github/workflows/issue_classifier.yml +++ b/.github/workflows/issue_classifier.yml @@ -25,9 +25,10 @@ on: permissions: {} +# Runs for one issue queue instead of cancelling, so an edit during the first run never cuts the label step short concurrency: group: issue-classifier-${{ github.event.issue.number || github.event.inputs.issue_number || github.run_id }} - cancel-in-progress: true + cancel-in-progress: false jobs: classifier-tests: @@ -51,16 +52,13 @@ jobs: run: bun test scripts/classify-issue.test.ts scripts/label-issue.test.ts classify: - # An edit only re-runs while the issue is still gated and no domain label has been applied by hand + # An edit to a labelled issue is dropped here; the script decides the rest against the live labels if: >- github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && ( github.event.action != 'edited' - || ( - contains(github.event.issue.labels.*.name, 'needs:template') - && !contains(join(github.event.issue.labels.*.name, ','), 'domain:') - ) + || !contains(join(github.event.issue.labels.*.name, ','), 'domain:') ) runs-on: ubuntu-latest timeout-minutes: 10 @@ -104,6 +102,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }} + GITHUB_EVENT_ACTION: ${{ github.event.action }} LITELLM_API_BASE: ${{ vars.LITELLM_API_BASE }} LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} ISSUE_CLASSIFIER_MODEL: ${{ vars.ISSUE_CLASSIFIER_MODEL }} @@ -123,6 +122,7 @@ jobs: } >> "${GITHUB_STEP_SUMMARY}" - name: Keep the verdict + if: steps.classify.outputs.verdict != '' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: classification-${{ github.event.issue.number || github.event.inputs.issue_number }} diff --git a/scripts/classify-issue.test.ts b/scripts/classify-issue.test.ts index f980496e9c9..96236dde4da 100644 --- a/scripts/classify-issue.test.ts +++ b/scripts/classify-issue.test.ts @@ -4,6 +4,7 @@ import type { GitHubApi } from "./auto-close-duplicates"; import { BODY_CAP_CHARS, BUG_SECTIONS, + EDIT_WINDOW_MS, FORM_HEADINGS, SECTION_CAP_CHARS, FEATURE_SECTIONS, @@ -15,6 +16,7 @@ import { readConfig, routesOf, sections, + shouldReclassify, userMessage, type ChatRequest, type IssueForClassification, @@ -53,9 +55,13 @@ const issue = (overrides: Partial = {}): IssueForClassif title: "[Bug]: Bedrock streaming drops the last chunk with tools", body: bugBody(), author_association: "NONE", + labels: [], + created_at: "2026-09-17T12:00:00Z", ...overrides, }); +const label = (...names: readonly string[]): readonly { readonly name: string }[] => names.map((name) => ({ name })); + const modelAnswer = (overrides: Record = {}): string => JSON.stringify({ domain: "llm-translation", @@ -322,8 +328,34 @@ describe("parseClassification", () => { }); }); +describe("shouldReclassify", () => { + const now = new Date("2026-09-17T12:10:00Z"); + + test("an issue that already carries a domain label is left alone, whatever else it has", () => { + expect(shouldReclassify(issue({ labels: label("domain:caching", "kind:bug") }), now)).toBe(false); + expect(shouldReclassify(issue({ labels: label("needs:template", "domain:caching") }), now)).toBe(false); + }); + + test("a gated issue is re-run however old it is", () => { + const old = new Date(Date.parse("2026-09-17T12:00:00Z") + EDIT_WINDOW_MS * 48); + expect(shouldReclassify(issue({ labels: label("bug", "needs:template") }), old)).toBe(true); + }); + + test("an unlabelled issue is re-run inside the edit window and ignored after it", () => { + expect(shouldReclassify(issue({ labels: label("bug") }), now)).toBe(true); + const later = new Date(Date.parse("2026-09-17T12:00:00Z") + EDIT_WINDOW_MS); + expect(shouldReclassify(issue({ labels: label("bug") }), later)).toBe(false); + }); +}); + describe("classifyIssue", () => { - const config = { repo: "BerriAI/litellm", issueNumber: 41700, model: "gpt-5.6-luna" }; + const config = { + repo: "BerriAI/litellm", + issueNumber: 41700, + model: "gpt-5.6-luna", + action: "opened", + now: new Date("2026-09-17T12:10:00Z"), + }; function fakeApi(fetched: IssueForClassification): GitHubApi { return { @@ -377,6 +409,39 @@ describe("classifyIssue", () => { ); expect(requests).toEqual([]); }); + + test("an edit to an issue that was classified while the edit was pending is ignored", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const edited = { ...config, action: "edited" }; + const labelled = issue({ labels: label("domain:llm-translation", "kind:bug", "priority:p1", "lift:small") }); + expect(await classifyIssue(fakeApi(labelled), llm, edited, "PROMPT", schema)).toBeNull(); + expect(requests).toEqual([]); + }); + + test("an edit that fixes a gated issue is classified against the new body", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const edited = { ...config, action: "edited" }; + const verdict = await classifyIssue(fakeApi(issue({ labels: label("bug", "needs:template") })), llm, edited, "PROMPT", schema); + expect(verdict).toMatchObject({ gate: "pass", domain: "llm-translation" }); + expect(requests).toHaveLength(1); + }); + + test("an edit during the first run, before any label landed, is classified instead of dropped", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const edited = { ...config, action: "edited" }; + expect(await classifyIssue(fakeApi(issue({ labels: label("bug") })), llm, edited, "PROMPT", schema)).toMatchObject({ + gate: "pass", + }); + expect(requests).toHaveLength(1); + }); + + test("a manual run classifies an old unlabelled issue that an edit would ignore", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const old = issue({ labels: label("bug"), created_at: "2020-01-01T00:00:00Z" }); + expect(await classifyIssue(fakeApi(old), llm, { ...config, action: "edited" }, "PROMPT", schema)).toBeNull(); + expect(await classifyIssue(fakeApi(old), llm, { ...config, action: "" }, "PROMPT", schema)).toMatchObject({ gate: "pass" }); + expect(requests).toHaveLength(1); + }); }); describe("readConfig", () => { @@ -389,24 +454,29 @@ describe("readConfig", () => { ISSUE_CLASSIFIER_MODEL: "gpt-5.6-luna", }; - test("reads the six settings", () => { - expect(readConfig(env)).toEqual({ + const now = new Date("2026-09-17T12:10:00Z"); + + test("reads the six settings, and the event action when the workflow passes one", () => { + expect(readConfig(env, now)).toEqual({ token: "t", repo: "BerriAI/litellm", issueNumber: 41700, apiBase: "https://llm.example.com", apiKey: "sk-test", model: "gpt-5.6-luna", + action: "", + now, }); + expect(readConfig({ ...env, GITHUB_EVENT_ACTION: "edited" }, now)).toMatchObject({ action: "edited" }); }); test("refuses a missing or malformed setting by name", () => { - expect(() => readConfig({ ...env, GITHUB_TOKEN: undefined })).toThrow("GITHUB_TOKEN"); - expect(() => readConfig({ ...env, GITHUB_REPOSITORY: "nope" })).toThrow("GITHUB_REPOSITORY"); - expect(() => readConfig({ ...env, ISSUE_NUMBER: "0" })).toThrow("ISSUE_NUMBER"); - expect(() => readConfig({ ...env, LITELLM_API_BASE: "" })).toThrow("LITELLM_API_BASE"); - expect(() => readConfig({ ...env, LITELLM_API_BASE: "llm.example.com" })).toThrow("LITELLM_API_BASE"); - expect(() => readConfig({ ...env, LITELLM_API_KEY: "" })).toThrow("LITELLM_API_KEY"); - expect(() => readConfig({ ...env, ISSUE_CLASSIFIER_MODEL: undefined })).toThrow("ISSUE_CLASSIFIER_MODEL"); + expect(() => readConfig({ ...env, GITHUB_TOKEN: undefined }, now)).toThrow("GITHUB_TOKEN"); + expect(() => readConfig({ ...env, GITHUB_REPOSITORY: "nope" }, now)).toThrow("GITHUB_REPOSITORY"); + expect(() => readConfig({ ...env, ISSUE_NUMBER: "0" }, now)).toThrow("ISSUE_NUMBER"); + expect(() => readConfig({ ...env, LITELLM_API_BASE: "" }, now)).toThrow("LITELLM_API_BASE"); + expect(() => readConfig({ ...env, LITELLM_API_BASE: "llm.example.com" }, now)).toThrow("LITELLM_API_BASE"); + expect(() => readConfig({ ...env, LITELLM_API_KEY: "" }, now)).toThrow("LITELLM_API_KEY"); + expect(() => readConfig({ ...env, ISSUE_CLASSIFIER_MODEL: undefined }, now)).toThrow("ISSUE_CLASSIFIER_MODEL"); }); }); diff --git a/scripts/classify-issue.ts b/scripts/classify-issue.ts index fb6540cff0d..7b72b29e711 100644 --- a/scripts/classify-issue.ts +++ b/scripts/classify-issue.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { githubApi, type GitHubApi } from "./auto-close-duplicates"; -import { MANIFEST, type Manifest } from "./issue-labels"; +import { MANIFEST, labelName, namespaceOf, type Manifest } from "./issue-labels"; declare const process: { readonly env: Readonly> }; declare const Bun: { @@ -13,6 +13,8 @@ export interface IssueForClassification { readonly title: string; readonly body: string | null; readonly author_association: string; + readonly labels: readonly { readonly name: string }[]; + readonly created_at: string; readonly pull_request?: unknown; } @@ -74,6 +76,8 @@ export interface ClassifyConfig { readonly repo: string; readonly issueNumber: number; readonly model: string; + readonly action: string; + readonly now: Date; } export interface Schema { @@ -273,17 +277,30 @@ export function parseClassification(raw: string, manifest: Manifest, routes: rea }; } +export const EDIT_WINDOW_MS = 60 * 60 * 1000; + +export function shouldReclassify(issue: Pick, now: Date): boolean { + const names = issue.labels.map((label) => label.name); + if (names.some((name) => namespaceOf(name) === "domain")) { + return false; + } + return names.includes(labelName("needs", "template")) || now.getTime() - Date.parse(issue.created_at) < EDIT_WINDOW_MS; +} + export async function classifyIssue( api: GitHubApi, llm: LlmClient, config: ClassifyConfig, prompt: string, schema: Schema, -): Promise { +): Promise { const issue = await api.request("GET", `/repos/${config.repo}/issues/${config.issueNumber}`); if (issue.pull_request !== undefined) { throw new Error(`#${config.issueNumber} is a pull request`); } + if (config.action === "edited" && !shouldReclassify(issue, config.now)) { + return null; + } const passed = gate(issue); if (passed.kind === "template") { return { gate: "template", template: passed.template, missing: passed.missing }; @@ -331,6 +348,7 @@ export function litellmClient(apiBase: string, apiKey: string): LlmClient { export function readConfig( env: Readonly>, + now: Date, ): ClassifyConfig & { readonly token: string; readonly apiBase: string; readonly apiKey: string } { const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; @@ -353,13 +371,17 @@ export function readConfig( if (!model) { throw new Error("ISSUE_CLASSIFIER_MODEL must name a model the LiteLLM deployment serves"); } - return { token, repo, issueNumber, apiBase, apiKey, model }; + return { token, repo, issueNumber, apiBase, apiKey, model, action: env.GITHUB_EVENT_ACTION ?? "", now }; } if (import.meta.main) { - const { token, apiBase, apiKey, ...config } = readConfig(process.env); + const { token, apiBase, apiKey, ...config } = readConfig(process.env, new Date()); const prompt = await Bun.file(`${import.meta.dir}/../.github/prompts/issue-classifier.md`).text(); const schema = (await Bun.file(`${import.meta.dir}/../.github/prompts/issue-classifier.schema.json`).json()) as Schema; const verdict = await classifyIssue(githubApi(token), litellmClient(apiBase, apiKey), config, prompt, schema); - console.log(JSON.stringify(verdict)); + if (verdict === null) { + console.error(`#${config.issueNumber}: edit ignored, the issue is already classified or older than the edit window`); + } else { + console.log(JSON.stringify(verdict)); + } }