diff --git a/.github/workflows/issue_classifier.yml b/.github/workflows/issue_classifier.yml index 8dce1663ef3..f84934aae4b 100644 --- a/.github/workflows/issue_classifier.yml +++ b/.github/workflows/issue_classifier.yml @@ -25,9 +25,10 @@ on: permissions: {} +# Runs for one issue queue instead of cancelling, so an edit during the first run never cuts the label step short concurrency: group: issue-classifier-${{ github.event.issue.number || github.event.inputs.issue_number || github.run_id }} - cancel-in-progress: true + cancel-in-progress: false jobs: classifier-tests: @@ -51,16 +52,13 @@ jobs: run: bun test scripts/classify-issue.test.ts scripts/label-issue.test.ts classify: - # An edit only re-runs while the issue is still gated and no domain label has been applied by hand + # An edit to a labelled issue is dropped here; the script decides the rest against the live labels if: >- github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && ( github.event.action != 'edited' - || ( - contains(github.event.issue.labels.*.name, 'needs:template') - && !contains(join(github.event.issue.labels.*.name, ','), 'domain:') - ) + || !contains(join(github.event.issue.labels.*.name, ','), 'domain:') ) runs-on: ubuntu-latest timeout-minutes: 10 @@ -104,6 +102,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }} + GITHUB_EVENT_ACTION: ${{ github.event.action }} LITELLM_API_BASE: ${{ vars.LITELLM_API_BASE }} LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} ISSUE_CLASSIFIER_MODEL: ${{ vars.ISSUE_CLASSIFIER_MODEL }} @@ -123,6 +122,7 @@ jobs: } >> "${GITHUB_STEP_SUMMARY}" - name: Keep the verdict + if: steps.classify.outputs.verdict != '' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: classification-${{ github.event.issue.number || github.event.inputs.issue_number }} diff --git a/scripts/classify-issue.test.ts b/scripts/classify-issue.test.ts index f980496e9c9..96236dde4da 100644 --- a/scripts/classify-issue.test.ts +++ b/scripts/classify-issue.test.ts @@ -4,6 +4,7 @@ import type { GitHubApi } from "./auto-close-duplicates"; import { BODY_CAP_CHARS, BUG_SECTIONS, + EDIT_WINDOW_MS, FORM_HEADINGS, SECTION_CAP_CHARS, FEATURE_SECTIONS, @@ -15,6 +16,7 @@ import { readConfig, routesOf, sections, + shouldReclassify, userMessage, type ChatRequest, type IssueForClassification, @@ -53,9 +55,13 @@ const issue = (overrides: Partial = {}): IssueForClassif title: "[Bug]: Bedrock streaming drops the last chunk with tools", body: bugBody(), author_association: "NONE", + labels: [], + created_at: "2026-09-17T12:00:00Z", ...overrides, }); +const label = (...names: readonly string[]): readonly { readonly name: string }[] => names.map((name) => ({ name })); + const modelAnswer = (overrides: Record = {}): string => JSON.stringify({ domain: "llm-translation", @@ -322,8 +328,34 @@ describe("parseClassification", () => { }); }); +describe("shouldReclassify", () => { + const now = new Date("2026-09-17T12:10:00Z"); + + test("an issue that already carries a domain label is left alone, whatever else it has", () => { + expect(shouldReclassify(issue({ labels: label("domain:caching", "kind:bug") }), now)).toBe(false); + expect(shouldReclassify(issue({ labels: label("needs:template", "domain:caching") }), now)).toBe(false); + }); + + test("a gated issue is re-run however old it is", () => { + const old = new Date(Date.parse("2026-09-17T12:00:00Z") + EDIT_WINDOW_MS * 48); + expect(shouldReclassify(issue({ labels: label("bug", "needs:template") }), old)).toBe(true); + }); + + test("an unlabelled issue is re-run inside the edit window and ignored after it", () => { + expect(shouldReclassify(issue({ labels: label("bug") }), now)).toBe(true); + const later = new Date(Date.parse("2026-09-17T12:00:00Z") + EDIT_WINDOW_MS); + expect(shouldReclassify(issue({ labels: label("bug") }), later)).toBe(false); + }); +}); + describe("classifyIssue", () => { - const config = { repo: "BerriAI/litellm", issueNumber: 41700, model: "gpt-5.6-luna" }; + const config = { + repo: "BerriAI/litellm", + issueNumber: 41700, + model: "gpt-5.6-luna", + action: "opened", + now: new Date("2026-09-17T12:10:00Z"), + }; function fakeApi(fetched: IssueForClassification): GitHubApi { return { @@ -377,6 +409,39 @@ describe("classifyIssue", () => { ); expect(requests).toEqual([]); }); + + test("an edit to an issue that was classified while the edit was pending is ignored", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const edited = { ...config, action: "edited" }; + const labelled = issue({ labels: label("domain:llm-translation", "kind:bug", "priority:p1", "lift:small") }); + expect(await classifyIssue(fakeApi(labelled), llm, edited, "PROMPT", schema)).toBeNull(); + expect(requests).toEqual([]); + }); + + test("an edit that fixes a gated issue is classified against the new body", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const edited = { ...config, action: "edited" }; + const verdict = await classifyIssue(fakeApi(issue({ labels: label("bug", "needs:template") })), llm, edited, "PROMPT", schema); + expect(verdict).toMatchObject({ gate: "pass", domain: "llm-translation" }); + expect(requests).toHaveLength(1); + }); + + test("an edit during the first run, before any label landed, is classified instead of dropped", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const edited = { ...config, action: "edited" }; + expect(await classifyIssue(fakeApi(issue({ labels: label("bug") })), llm, edited, "PROMPT", schema)).toMatchObject({ + gate: "pass", + }); + expect(requests).toHaveLength(1); + }); + + test("a manual run classifies an old unlabelled issue that an edit would ignore", async () => { + const { llm, requests } = fakeLlm(modelAnswer()); + const old = issue({ labels: label("bug"), created_at: "2020-01-01T00:00:00Z" }); + expect(await classifyIssue(fakeApi(old), llm, { ...config, action: "edited" }, "PROMPT", schema)).toBeNull(); + expect(await classifyIssue(fakeApi(old), llm, { ...config, action: "" }, "PROMPT", schema)).toMatchObject({ gate: "pass" }); + expect(requests).toHaveLength(1); + }); }); describe("readConfig", () => { @@ -389,24 +454,29 @@ describe("readConfig", () => { ISSUE_CLASSIFIER_MODEL: "gpt-5.6-luna", }; - test("reads the six settings", () => { - expect(readConfig(env)).toEqual({ + const now = new Date("2026-09-17T12:10:00Z"); + + test("reads the six settings, and the event action when the workflow passes one", () => { + expect(readConfig(env, now)).toEqual({ token: "t", repo: "BerriAI/litellm", issueNumber: 41700, apiBase: "https://llm.example.com", apiKey: "sk-test", model: "gpt-5.6-luna", + action: "", + now, }); + expect(readConfig({ ...env, GITHUB_EVENT_ACTION: "edited" }, now)).toMatchObject({ action: "edited" }); }); test("refuses a missing or malformed setting by name", () => { - expect(() => readConfig({ ...env, GITHUB_TOKEN: undefined })).toThrow("GITHUB_TOKEN"); - expect(() => readConfig({ ...env, GITHUB_REPOSITORY: "nope" })).toThrow("GITHUB_REPOSITORY"); - expect(() => readConfig({ ...env, ISSUE_NUMBER: "0" })).toThrow("ISSUE_NUMBER"); - expect(() => readConfig({ ...env, LITELLM_API_BASE: "" })).toThrow("LITELLM_API_BASE"); - expect(() => readConfig({ ...env, LITELLM_API_BASE: "llm.example.com" })).toThrow("LITELLM_API_BASE"); - expect(() => readConfig({ ...env, LITELLM_API_KEY: "" })).toThrow("LITELLM_API_KEY"); - expect(() => readConfig({ ...env, ISSUE_CLASSIFIER_MODEL: undefined })).toThrow("ISSUE_CLASSIFIER_MODEL"); + expect(() => readConfig({ ...env, GITHUB_TOKEN: undefined }, now)).toThrow("GITHUB_TOKEN"); + expect(() => readConfig({ ...env, GITHUB_REPOSITORY: "nope" }, now)).toThrow("GITHUB_REPOSITORY"); + expect(() => readConfig({ ...env, ISSUE_NUMBER: "0" }, now)).toThrow("ISSUE_NUMBER"); + expect(() => readConfig({ ...env, LITELLM_API_BASE: "" }, now)).toThrow("LITELLM_API_BASE"); + expect(() => readConfig({ ...env, LITELLM_API_BASE: "llm.example.com" }, now)).toThrow("LITELLM_API_BASE"); + expect(() => readConfig({ ...env, LITELLM_API_KEY: "" }, now)).toThrow("LITELLM_API_KEY"); + expect(() => readConfig({ ...env, ISSUE_CLASSIFIER_MODEL: undefined }, now)).toThrow("ISSUE_CLASSIFIER_MODEL"); }); }); diff --git a/scripts/classify-issue.ts b/scripts/classify-issue.ts index fb6540cff0d..7b72b29e711 100644 --- a/scripts/classify-issue.ts +++ b/scripts/classify-issue.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { githubApi, type GitHubApi } from "./auto-close-duplicates"; -import { MANIFEST, type Manifest } from "./issue-labels"; +import { MANIFEST, labelName, namespaceOf, type Manifest } from "./issue-labels"; declare const process: { readonly env: Readonly> }; declare const Bun: { @@ -13,6 +13,8 @@ export interface IssueForClassification { readonly title: string; readonly body: string | null; readonly author_association: string; + readonly labels: readonly { readonly name: string }[]; + readonly created_at: string; readonly pull_request?: unknown; } @@ -74,6 +76,8 @@ export interface ClassifyConfig { readonly repo: string; readonly issueNumber: number; readonly model: string; + readonly action: string; + readonly now: Date; } export interface Schema { @@ -273,17 +277,30 @@ export function parseClassification(raw: string, manifest: Manifest, routes: rea }; } +export const EDIT_WINDOW_MS = 60 * 60 * 1000; + +export function shouldReclassify(issue: Pick, now: Date): boolean { + const names = issue.labels.map((label) => label.name); + if (names.some((name) => namespaceOf(name) === "domain")) { + return false; + } + return names.includes(labelName("needs", "template")) || now.getTime() - Date.parse(issue.created_at) < EDIT_WINDOW_MS; +} + export async function classifyIssue( api: GitHubApi, llm: LlmClient, config: ClassifyConfig, prompt: string, schema: Schema, -): Promise { +): Promise { const issue = await api.request("GET", `/repos/${config.repo}/issues/${config.issueNumber}`); if (issue.pull_request !== undefined) { throw new Error(`#${config.issueNumber} is a pull request`); } + if (config.action === "edited" && !shouldReclassify(issue, config.now)) { + return null; + } const passed = gate(issue); if (passed.kind === "template") { return { gate: "template", template: passed.template, missing: passed.missing }; @@ -331,6 +348,7 @@ export function litellmClient(apiBase: string, apiKey: string): LlmClient { export function readConfig( env: Readonly>, + now: Date, ): ClassifyConfig & { readonly token: string; readonly apiBase: string; readonly apiKey: string } { const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; @@ -353,13 +371,17 @@ export function readConfig( if (!model) { throw new Error("ISSUE_CLASSIFIER_MODEL must name a model the LiteLLM deployment serves"); } - return { token, repo, issueNumber, apiBase, apiKey, model }; + return { token, repo, issueNumber, apiBase, apiKey, model, action: env.GITHUB_EVENT_ACTION ?? "", now }; } if (import.meta.main) { - const { token, apiBase, apiKey, ...config } = readConfig(process.env); + const { token, apiBase, apiKey, ...config } = readConfig(process.env, new Date()); const prompt = await Bun.file(`${import.meta.dir}/../.github/prompts/issue-classifier.md`).text(); const schema = (await Bun.file(`${import.meta.dir}/../.github/prompts/issue-classifier.schema.json`).json()) as Schema; const verdict = await classifyIssue(githubApi(token), litellmClient(apiBase, apiKey), config, prompt, schema); - console.log(JSON.stringify(verdict)); + if (verdict === null) { + console.error(`#${config.issueNumber}: edit ignored, the issue is already classified or older than the edit window`); + } else { + console.log(JSON.stringify(verdict)); + } }