fix(issue-classifier): queue runs per issue and judge edits by live labels

An edit during the first run used to cancel it, and the edited run then
skipped because the webhook payload had no needs:template yet, so a
well-formed issue edited within the first minute was never labelled.
Runs for one issue now queue, and the script decides an edited event
against the live labels: a domain label means leave it alone, a gated
issue is re-run, and an unlabelled issue is re-run for its first hour

.github/workflows/issue_classifier.yml

@@ -25,9 +25,10 @@ on:
 
 permissions: {}
 
+# Runs for one issue queue instead of cancelling, so an edit during the first run never cuts the label step short
 concurrency:
   group: issue-classifier-${{ github.event.issue.number || github.event.inputs.issue_number || github.run_id }}
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 jobs:
   classifier-tests:
@@ -51,16 +52,13 @@ jobs:
         run: bun test scripts/classify-issue.test.ts scripts/label-issue.test.ts
 
   classify:
-    # An edit only re-runs while the issue is still gated and no domain label has been applied by hand
+    # An edit to a labelled issue is dropped here; the script decides the rest against the live labels
     if: >-
       github.event_name != 'pull_request'
       && github.repository == 'BerriAI/litellm'
       && (
         github.event.action != 'edited'
-        || (
-          contains(github.event.issue.labels.*.name, 'needs:template')
-          && !contains(join(github.event.issue.labels.*.name, ','), 'domain:')
-        )
+        || !contains(join(github.event.issue.labels.*.name, ','), 'domain:')
       )
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -104,6 +102,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }}
+          GITHUB_EVENT_ACTION: ${{ github.event.action }}
           LITELLM_API_BASE: ${{ vars.LITELLM_API_BASE }}
           LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }}
           ISSUE_CLASSIFIER_MODEL: ${{ vars.ISSUE_CLASSIFIER_MODEL }}
@@ -123,6 +122,7 @@ jobs:
           } >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Keep the verdict
+        if: steps.classify.outputs.verdict != ''
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: classification-${{ github.event.issue.number || github.event.inputs.issue_number }}

scripts/classify-issue.test.ts

@@ -4,6 +4,7 @@ import type { GitHubApi } from "./auto-close-duplicates";
 import {
   BODY_CAP_CHARS,
   BUG_SECTIONS,
+  EDIT_WINDOW_MS,
   FORM_HEADINGS,
   SECTION_CAP_CHARS,
   FEATURE_SECTIONS,
@@ -15,6 +16,7 @@ import {
   readConfig,
   routesOf,
   sections,
+  shouldReclassify,
   userMessage,
   type ChatRequest,
   type IssueForClassification,
@@ -53,9 +55,13 @@ const issue = (overrides: Partial<IssueForClassification> = {}): IssueForClassif
   title: "[Bug]: Bedrock streaming drops the last chunk with tools",
   body: bugBody(),
   author_association: "NONE",
+  labels: [],
+  created_at: "2026-09-17T12:00:00Z",
   ...overrides,
 });
 
+const label = (...names: readonly string[]): readonly { readonly name: string }[] => names.map((name) => ({ name }));
+
 const modelAnswer = (overrides: Record<string, unknown> = {}): string =>
   JSON.stringify({
     domain: "llm-translation",
@@ -322,8 +328,34 @@ describe("parseClassification", () => {
   });
 });
 
+describe("shouldReclassify", () => {
+  const now = new Date("2026-09-17T12:10:00Z");
+
+  test("an issue that already carries a domain label is left alone, whatever else it has", () => {
+    expect(shouldReclassify(issue({ labels: label("domain:caching", "kind:bug") }), now)).toBe(false);
+    expect(shouldReclassify(issue({ labels: label("needs:template", "domain:caching") }), now)).toBe(false);
+  });
+
+  test("a gated issue is re-run however old it is", () => {
+    const old = new Date(Date.parse("2026-09-17T12:00:00Z") + EDIT_WINDOW_MS * 48);
+    expect(shouldReclassify(issue({ labels: label("bug", "needs:template") }), old)).toBe(true);
+  });
+
+  test("an unlabelled issue is re-run inside the edit window and ignored after it", () => {
+    expect(shouldReclassify(issue({ labels: label("bug") }), now)).toBe(true);
+    const later = new Date(Date.parse("2026-09-17T12:00:00Z") + EDIT_WINDOW_MS);
+    expect(shouldReclassify(issue({ labels: label("bug") }), later)).toBe(false);
+  });
+});
+
 describe("classifyIssue", () => {
-  const config = { repo: "BerriAI/litellm", issueNumber: 41700, model: "gpt-5.6-luna" };
+  const config = {
+    repo: "BerriAI/litellm",
+    issueNumber: 41700,
+    model: "gpt-5.6-luna",
+    action: "opened",
+    now: new Date("2026-09-17T12:10:00Z"),
+  };
 
   function fakeApi(fetched: IssueForClassification): GitHubApi {
     return {
@@ -377,6 +409,39 @@ describe("classifyIssue", () => {
     );
     expect(requests).toEqual([]);
   });
+
+  test("an edit to an issue that was classified while the edit was pending is ignored", async () => {
+    const { llm, requests } = fakeLlm(modelAnswer());
+    const edited = { ...config, action: "edited" };
+    const labelled = issue({ labels: label("domain:llm-translation", "kind:bug", "priority:p1", "lift:small") });
+    expect(await classifyIssue(fakeApi(labelled), llm, edited, "PROMPT", schema)).toBeNull();
+    expect(requests).toEqual([]);
+  });
+
+  test("an edit that fixes a gated issue is classified against the new body", async () => {
+    const { llm, requests } = fakeLlm(modelAnswer());
+    const edited = { ...config, action: "edited" };
+    const verdict = await classifyIssue(fakeApi(issue({ labels: label("bug", "needs:template") })), llm, edited, "PROMPT", schema);
+    expect(verdict).toMatchObject({ gate: "pass", domain: "llm-translation" });
+    expect(requests).toHaveLength(1);
+  });
+
+  test("an edit during the first run, before any label landed, is classified instead of dropped", async () => {
+    const { llm, requests } = fakeLlm(modelAnswer());
+    const edited = { ...config, action: "edited" };
+    expect(await classifyIssue(fakeApi(issue({ labels: label("bug") })), llm, edited, "PROMPT", schema)).toMatchObject({
+      gate: "pass",
+    });
+    expect(requests).toHaveLength(1);
+  });
+
+  test("a manual run classifies an old unlabelled issue that an edit would ignore", async () => {
+    const { llm, requests } = fakeLlm(modelAnswer());
+    const old = issue({ labels: label("bug"), created_at: "2020-01-01T00:00:00Z" });
+    expect(await classifyIssue(fakeApi(old), llm, { ...config, action: "edited" }, "PROMPT", schema)).toBeNull();
+    expect(await classifyIssue(fakeApi(old), llm, { ...config, action: "" }, "PROMPT", schema)).toMatchObject({ gate: "pass" });
+    expect(requests).toHaveLength(1);
+  });
 });
 
 describe("readConfig", () => {
@@ -389,24 +454,29 @@ describe("readConfig", () => {
     ISSUE_CLASSIFIER_MODEL: "gpt-5.6-luna",
   };
 
-  test("reads the six settings", () => {
-    expect(readConfig(env)).toEqual({
+  const now = new Date("2026-09-17T12:10:00Z");
+
+  test("reads the six settings, and the event action when the workflow passes one", () => {
+    expect(readConfig(env, now)).toEqual({
       token: "t",
       repo: "BerriAI/litellm",
       issueNumber: 41700,
       apiBase: "https://llm.example.com",
       apiKey: "sk-test",
       model: "gpt-5.6-luna",
+      action: "",
+      now,
     });
+    expect(readConfig({ ...env, GITHUB_EVENT_ACTION: "edited" }, now)).toMatchObject({ action: "edited" });
   });
 
   test("refuses a missing or malformed setting by name", () => {
-    expect(() => readConfig({ ...env, GITHUB_TOKEN: undefined })).toThrow("GITHUB_TOKEN");
-    expect(() => readConfig({ ...env, GITHUB_REPOSITORY: "nope" })).toThrow("GITHUB_REPOSITORY");
-    expect(() => readConfig({ ...env, ISSUE_NUMBER: "0" })).toThrow("ISSUE_NUMBER");
-    expect(() => readConfig({ ...env, LITELLM_API_BASE: "" })).toThrow("LITELLM_API_BASE");
-    expect(() => readConfig({ ...env, LITELLM_API_BASE: "llm.example.com" })).toThrow("LITELLM_API_BASE");
-    expect(() => readConfig({ ...env, LITELLM_API_KEY: "" })).toThrow("LITELLM_API_KEY");
-    expect(() => readConfig({ ...env, ISSUE_CLASSIFIER_MODEL: undefined })).toThrow("ISSUE_CLASSIFIER_MODEL");
+    expect(() => readConfig({ ...env, GITHUB_TOKEN: undefined }, now)).toThrow("GITHUB_TOKEN");
+    expect(() => readConfig({ ...env, GITHUB_REPOSITORY: "nope" }, now)).toThrow("GITHUB_REPOSITORY");
+    expect(() => readConfig({ ...env, ISSUE_NUMBER: "0" }, now)).toThrow("ISSUE_NUMBER");
+    expect(() => readConfig({ ...env, LITELLM_API_BASE: "" }, now)).toThrow("LITELLM_API_BASE");
+    expect(() => readConfig({ ...env, LITELLM_API_BASE: "llm.example.com" }, now)).toThrow("LITELLM_API_BASE");
+    expect(() => readConfig({ ...env, LITELLM_API_KEY: "" }, now)).toThrow("LITELLM_API_KEY");
+    expect(() => readConfig({ ...env, ISSUE_CLASSIFIER_MODEL: undefined }, now)).toThrow("ISSUE_CLASSIFIER_MODEL");
   });
 });

scripts/classify-issue.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env bun
 
 import { githubApi, type GitHubApi } from "./auto-close-duplicates";
-import { MANIFEST, type Manifest } from "./issue-labels";
+import { MANIFEST, labelName, namespaceOf, type Manifest } from "./issue-labels";
 
 declare const process: { readonly env: Readonly<Record<string, string | undefined>> };
 declare const Bun: {
@@ -13,6 +13,8 @@ export interface IssueForClassification {
   readonly title: string;
   readonly body: string | null;
   readonly author_association: string;
+  readonly labels: readonly { readonly name: string }[];
+  readonly created_at: string;
   readonly pull_request?: unknown;
 }
 
@@ -74,6 +76,8 @@ export interface ClassifyConfig {
   readonly repo: string;
   readonly issueNumber: number;
   readonly model: string;
+  readonly action: string;
+  readonly now: Date;
 }
 
 export interface Schema {
@@ -273,17 +277,30 @@ export function parseClassification(raw: string, manifest: Manifest, routes: rea
   };
 }
 
+export const EDIT_WINDOW_MS = 60 * 60 * 1000;
+
+export function shouldReclassify(issue: Pick<IssueForClassification, "labels" | "created_at">, now: Date): boolean {
+  const names = issue.labels.map((label) => label.name);
+  if (names.some((name) => namespaceOf(name) === "domain")) {
+    return false;
+  }
+  return names.includes(labelName("needs", "template")) || now.getTime() - Date.parse(issue.created_at) < EDIT_WINDOW_MS;
+}
+
 export async function classifyIssue(
   api: GitHubApi,
   llm: LlmClient,
   config: ClassifyConfig,
   prompt: string,
   schema: Schema,
-): Promise<Verdict> {
+): Promise<Verdict | null> {
   const issue = await api.request<IssueForClassification>("GET", `/repos/${config.repo}/issues/${config.issueNumber}`);
   if (issue.pull_request !== undefined) {
     throw new Error(`#${config.issueNumber} is a pull request`);
   }
+  if (config.action === "edited" && !shouldReclassify(issue, config.now)) {
+    return null;
+  }
   const passed = gate(issue);
   if (passed.kind === "template") {
     return { gate: "template", template: passed.template, missing: passed.missing };
@@ -331,6 +348,7 @@ export function litellmClient(apiBase: string, apiKey: string): LlmClient {
 
 export function readConfig(
   env: Readonly<Record<string, string | undefined>>,
+  now: Date,
 ): ClassifyConfig & { readonly token: string; readonly apiBase: string; readonly apiKey: string } {
   const token = env.GITHUB_TOKEN;
   const repo = env.GITHUB_REPOSITORY;
@@ -353,13 +371,17 @@ export function readConfig(
   if (!model) {
     throw new Error("ISSUE_CLASSIFIER_MODEL must name a model the LiteLLM deployment serves");
   }
-  return { token, repo, issueNumber, apiBase, apiKey, model };
+  return { token, repo, issueNumber, apiBase, apiKey, model, action: env.GITHUB_EVENT_ACTION ?? "", now };
 }
 
 if (import.meta.main) {
-  const { token, apiBase, apiKey, ...config } = readConfig(process.env);
+  const { token, apiBase, apiKey, ...config } = readConfig(process.env, new Date());
   const prompt = await Bun.file(`${import.meta.dir}/../.github/prompts/issue-classifier.md`).text();
   const schema = (await Bun.file(`${import.meta.dir}/../.github/prompts/issue-classifier.schema.json`).json()) as Schema;
   const verdict = await classifyIssue(githubApi(token), litellmClient(apiBase, apiKey), config, prompt, schema);
-  console.log(JSON.stringify(verdict));
+  if (verdict === null) {
+    console.error(`#${config.issueNumber}: edit ignored, the issue is already classified or older than the edit window`);
+  } else {
+    console.log(JSON.stringify(verdict));
+  }
 }