diff --git a/.github/issue-labels.json b/.github/issue-labels.json index 2b99faf2e4f..a9b583007a4 100644 --- a/.github/issue-labels.json +++ b/.github/issue-labels.json @@ -54,5 +54,15 @@ "template": { "color": "E99695", "description": "Required sections of the issue template are missing or empty" }, "version": { "color": "E99695", "description": "No LiteLLM version anywhere in the issue" }, "repro": { "color": "E99695", "description": "A bug with no command, output or screenshot to reproduce it" } + }, + "dup": { + "clear": { "color": "0E8A16", "description": "The duplicate check ran and named no earlier issue as the original" } + }, + "repro": { + "running": { "color": "1D76DB", "description": "Devin is reproducing this now; the lock that keeps a second run from starting" }, + "success": { "color": "0E8A16", "description": "Devin reproduced it; the PR or Linear ticket is linked in a comment" }, + "fail": { "color": "E99695", "description": "Devin could not reproduce it with what the issue provides" }, + "blocked": { "color": "D93F0B", "description": "Devin needs something only a human can supply before it can reproduce this" }, + "skip": { "color": "C5DEF5", "description": "A human opted this issue out of automatic reproduction" } } } diff --git a/.github/workflows/duplicate_issue_check.yml b/.github/workflows/duplicate_issue_check.yml index 91eee3ca383..298775da821 100644 --- a/.github/workflows/duplicate_issue_check.yml +++ b/.github/workflows/duplicate_issue_check.yml @@ -13,8 +13,10 @@ on: - .github/workflows/duplicate_issue_check.yml - .github/prompts/duplicate-issue-check.md - .github/prompts/duplicate-issue-check.schema.json + - .github/issue-labels.json - scripts/flag-duplicate-issue.ts - scripts/flag-duplicate-issue.test.ts + - scripts/issue-labels.ts - scripts/auto-close-duplicates.ts permissions: {} @@ -125,7 +127,9 @@ jobs: - name: Checkout scripts uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - sparse-checkout: scripts + sparse-checkout: | + .github + scripts persist-credentials: false - name: Setup Bun diff --git a/.github/workflows/issue_triage_dispatch.yml b/.github/workflows/issue_triage_dispatch.yml new file mode 100644 index 00000000000..84e58a0b303 --- /dev/null +++ b/.github/workflows/issue_triage_dispatch.yml @@ -0,0 +1,85 @@ +name: Issue triage dispatch + +on: + issues: + types: [labeled, unlabeled] + workflow_dispatch: + inputs: + issue_number: + description: "Issue number to run the gate on manually." + required: true + pull_request: + paths: + - .github/workflows/issue_triage_dispatch.yml + - .github/issue-labels.json + - scripts/dispatch-issue-triage.ts + - scripts/dispatch-issue-triage.test.ts + - scripts/flag-duplicate-issue.ts + - scripts/issue-labels.ts + - scripts/auto-close-duplicates.ts + +permissions: {} + +# Runs for one issue queue instead of cancelling, so two label events land one after the other and the second sees the lock the first added +concurrency: + group: issue-triage-dispatch-${{ github.event.issue.number || github.event.inputs.issue_number || github.run_id }} + cancel-in-progress: false + +jobs: + dispatch-issue-triage-tests: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.4.0" + + - name: Test the gate + run: bun test scripts/dispatch-issue-triage.test.ts + + dispatch-issue-triage: + # Only the labels the gate reads can change its answer; every other label event is dropped here without a checkout + if: >- + github.event_name != 'pull_request' + && github.repository == 'BerriAI/litellm' + && ( + github.event_name == 'workflow_dispatch' + || (github.event.action == 'labeled' && (github.event.label.name == 'kind:bug' || github.event.label.name == 'dup:clear')) + || (github.event.action == 'unlabeled' && (startsWith(github.event.label.name, 'needs:') || startsWith(github.event.label.name, 'repro:') || github.event.label.name == 'potential-duplicate')) + ) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + steps: + - name: Checkout scripts + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: | + .github + scripts + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + # Exact version, never latest: the next step holds an issues: write token + bun-version: "1.4.0" + + # The issue is read through the API inside the script, so its text never reaches a shell + - name: Gate the issue and add the repro:running lock when it passes + run: bun run scripts/dispatch-issue-triage.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }} + DRY_RUN: ${{ vars.TRIAGE_DISPATCH_ENABLED != 'true' }} diff --git a/scripts/dispatch-issue-triage.test.ts b/scripts/dispatch-issue-triage.test.ts new file mode 100644 index 00000000000..15d879b2017 --- /dev/null +++ b/scripts/dispatch-issue-triage.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from "bun:test"; + +import type { Comment, GitHubApi } from "./auto-close-duplicates"; +import { + DISPATCH_COMMENT, + DISPATCH_MARKER, + RUNNING_LABEL, + dispatchIssue, + dispatchTarget, + openPullsReferencing, + type DispatchVerdict, + type IssueForDispatch, + type TimelineEvent, +} from "./dispatch-issue-triage"; +import type { FlagConfig } from "./flag-duplicate-issue"; + +const ready = (labels: readonly string[] = ["kind:bug", "dup:clear", "lift:small", "domain:caching"], overrides: Partial = {}): IssueForDispatch => ({ + number: 41900, + state: "open", + labels: labels.map((name) => ({ name })), + ...overrides, +}); + +const crossRef = (number: number, state: string, pull = true): TimelineEvent => ({ + event: "cross-referenced", + source: { issue: { number, state, ...(pull ? { pull_request: {} } : {}) } }, +}); + +const marker: Comment = { + id: 1, + body: DISPATCH_COMMENT, + created_at: "2026-09-18T00:00:00Z", + user: { type: "Bot", login: "github-actions[bot]" }, +}; + +const config: FlagConfig = { repo: "BerriAI/litellm", issueNumber: 41900, dryRun: false }; + +const reasonOf = (verdict: DispatchVerdict): string => (verdict.kind === "skip" ? verdict.reason : ""); + +describe("dispatchTarget", () => { + test("an open bug cleared of duplicates with no needs, no repro state, no marker and no PR is dispatched", () => { + expect(dispatchTarget(ready(), [], [])).toEqual({ kind: "dispatch" }); + }); + + test("both positive signals are required, label absence never counts", () => { + expect(reasonOf(dispatchTarget(ready(["dup:clear"]), [], []))).toBe("missing kind:bug"); + expect(reasonOf(dispatchTarget(ready(["kind:bug"]), [], []))).toBe("missing dup:clear"); + expect(reasonOf(dispatchTarget(ready(["kind:feature", "dup:clear"]), [], []))).toBe("missing kind:bug"); + expect(reasonOf(dispatchTarget(ready([]), [], []))).toBe("missing kind:bug and dup:clear"); + }); + + test("closed issues and pull requests are left alone", () => { + expect(reasonOf(dispatchTarget(ready(undefined, { state: "closed" }), [], []))).toBe("is closed"); + expect(reasonOf(dispatchTarget(ready(undefined, { pull_request: {} }), [], []))).toBe("is a pull request"); + }); + + test("any needs:*, any repro:* or a duplicate marker blocks, and every blocker is named", () => { + const base = ["kind:bug", "dup:clear"]; + expect(reasonOf(dispatchTarget(ready([...base, "needs:repro"]), [], []))).toBe("carries needs:repro"); + expect(reasonOf(dispatchTarget(ready([...base, "needs:version"]), [], []))).toBe("carries needs:version"); + expect(reasonOf(dispatchTarget(ready([...base, "repro:running"]), [], []))).toBe("carries repro:running"); + expect(reasonOf(dispatchTarget(ready([...base, "repro:skip"]), [], []))).toBe("carries repro:skip"); + expect(reasonOf(dispatchTarget(ready([...base, "repro:fail"]), [], []))).toBe("carries repro:fail"); + expect(reasonOf(dispatchTarget(ready([...base, "potential-duplicate"]), [], []))).toBe("carries potential-duplicate"); + expect(reasonOf(dispatchTarget(ready([...base, "duplicate"]), [], []))).toBe("carries duplicate"); + expect(reasonOf(dispatchTarget(ready([...base, "needs:repro", "repro:skip"]), [], []))).toBe("carries needs:repro, repro:skip"); + }); + + test("a marker comment left by an earlier dispatch blocks even after the labels were cleaned up", () => { + expect(reasonOf(dispatchTarget(ready(), [marker], []))).toBe("was already dispatched once"); + const unrelated: Comment = { ...marker, body: "I hit this too" }; + expect(dispatchTarget(ready(), [unrelated], [])).toEqual({ kind: "dispatch" }); + }); + + test("only the workflow's own marker counts, a commenter pasting it cannot opt the issue out", () => { + const forged: Comment = { ...marker, user: { type: "User", login: "someone" } }; + expect(dispatchTarget(ready(), [forged], [])).toEqual({ kind: "dispatch" }); + const otherBot: Comment = { ...marker, user: { type: "Bot", login: "some-other-app[bot]" } }; + expect(dispatchTarget(ready(), [otherBot], [])).toEqual({ kind: "dispatch" }); + }); + + test("an open pull request that references the issue blocks, a merged one or a plain issue does not", () => { + expect(reasonOf(dispatchTarget(ready(), [], [crossRef(41950, "open")]))).toBe("open pull request #41950 already references it"); + expect(reasonOf(dispatchTarget(ready(), [], [crossRef(41950, "open"), crossRef(41960, "open")]))).toBe( + "open pull requests #41950, #41960 already reference it", + ); + expect(dispatchTarget(ready(), [], [crossRef(41950, "closed")])).toEqual({ kind: "dispatch" }); + expect(dispatchTarget(ready(), [], [crossRef(41950, "open", false)])).toEqual({ kind: "dispatch" }); + }); +}); + +describe("openPullsReferencing", () => { + test("keeps open pull requests once each, sorted, and ignores other timeline events", () => { + const timeline: readonly TimelineEvent[] = [ + { event: "labeled" }, + crossRef(300, "open"), + crossRef(200, "open"), + crossRef(300, "open"), + crossRef(100, "closed"), + crossRef(50, "open", false), + { event: "cross-referenced" }, + ]; + expect(openPullsReferencing(timeline)).toEqual([200, 300]); + }); +}); + +describe("dispatchIssue", () => { + function fakeApi( + issue: IssueForDispatch = ready(), + comments: readonly Comment[] = [], + timeline: readonly TimelineEvent[] = [], + ): { readonly api: GitHubApi; readonly writes: string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path === "/repos/BerriAI/litellm/issues/41900") { + return issue as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/41900/comments")) { + return comments as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/41900/timeline")) { + return timeline as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + test("a real run adds the lock label first, then the marker comment", async () => { + const { api, writes } = fakeApi(); + expect(await dispatchIssue(api, config)).toEqual({ kind: "dispatch" }); + expect(writes).toEqual([ + `POST /repos/BerriAI/litellm/issues/41900/labels {"labels":["${RUNNING_LABEL}"]}`, + `POST /repos/BerriAI/litellm/issues/41900/comments ${JSON.stringify({ body: DISPATCH_COMMENT })}`, + ]); + expect(DISPATCH_COMMENT.startsWith(DISPATCH_MARKER)).toBe(true); + }); + + test("a dry run decides the same way and writes nothing", async () => { + const { api, writes } = fakeApi(); + expect(await dispatchIssue(api, { ...config, dryRun: true })).toEqual({ kind: "dispatch" }); + expect(writes).toEqual([]); + }); + + test("a blocked issue is never written to, whatever blocked it", async () => { + const labelled = fakeApi(ready(["kind:bug", "dup:clear", "repro:running"])); + expect((await dispatchIssue(labelled.api, config)).kind).toBe("skip"); + expect(labelled.writes).toEqual([]); + + const marked = fakeApi(ready(), [marker]); + expect((await dispatchIssue(marked.api, config)).kind).toBe("skip"); + expect(marked.writes).toEqual([]); + + const referenced = fakeApi(ready(), [], [crossRef(41950, "open")]); + expect((await dispatchIssue(referenced.api, config)).kind).toBe("skip"); + expect(referenced.writes).toEqual([]); + }); + + test("the second run after a successful dispatch is a no-op because the label it just added blocks it", async () => { + const first = fakeApi(); + await dispatchIssue(first.api, config); + const second = fakeApi(ready(["kind:bug", "dup:clear", RUNNING_LABEL]), [marker]); + expect(reasonOf(await dispatchIssue(second.api, config))).toBe(`carries ${RUNNING_LABEL}`); + expect(second.writes).toEqual([]); + }); + + test("the reporter-facing comment stays within the 15 to 25 word limit", () => { + const words = DISPATCH_COMMENT.split("\n").slice(1).join(" ").trim().split(/\s+/); + expect(words.length).toBeGreaterThanOrEqual(15); + expect(words.length).toBeLessThanOrEqual(25); + }); +}); diff --git a/scripts/dispatch-issue-triage.ts b/scripts/dispatch-issue-triage.ts new file mode 100644 index 00000000000..6e992ca1f3e --- /dev/null +++ b/scripts/dispatch-issue-triage.ts @@ -0,0 +1,114 @@ +#!/usr/bin/env bun + +import { FLAG_LABEL, githubApi, listAll, type Comment, type GitHubApi } from "./auto-close-duplicates"; +import { CLEAR_LABEL, readConfig, WORKFLOW_LOGIN, type FlagConfig as DispatchConfig } from "./flag-duplicate-issue"; +import { labelName } from "./issue-labels"; + +declare const process: { readonly env: Readonly> }; + +export interface IssueForDispatch { + readonly number: number; + readonly state: string; + readonly labels: readonly { readonly name: string }[]; + readonly pull_request?: unknown; +} + +export interface TimelineSource { + readonly number: number; + readonly state: string; + readonly pull_request?: unknown; +} + +export interface TimelineEvent { + readonly event: string; + readonly source?: { readonly issue?: TimelineSource }; +} + +export type DispatchVerdict = + | { readonly kind: "dispatch" } + | { readonly kind: "skip"; readonly reason: string }; + +export const BUG_LABEL = labelName("kind", "bug"); +export const RUNNING_LABEL = labelName("repro", "running"); +export const DISPATCH_MARKER = ""; +export const DISPATCH_COMMENT = [ + DISPATCH_MARKER, + "Devin is going to try to reproduce this. A PR or a Linear ticket follows if it does. Add repro:skip to opt out", +].join("\n"); +const BLOCKING_LABELS: readonly string[] = [FLAG_LABEL, "duplicate"]; +const BLOCKING_PREFIXES: readonly string[] = ["needs:", "repro:"]; + +const skip = (reason: string): DispatchVerdict => ({ kind: "skip", reason }); + +export function openPullsReferencing(timeline: readonly TimelineEvent[]): readonly number[] { + const numbers = timeline + .flatMap((event) => (event.event === "cross-referenced" && event.source?.issue !== undefined ? [event.source.issue] : [])) + .filter((source) => source.pull_request !== undefined && source.state === "open") + .map((source) => source.number); + return [...new Set(numbers)].sort((a, b) => a - b); +} + +export function dispatchTarget( + issue: IssueForDispatch, + comments: readonly Comment[], + timeline: readonly TimelineEvent[], +): DispatchVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (issue.state !== "open") { + return skip(`is ${issue.state}`); + } + const labels = issue.labels.map((label) => label.name); + const missing = [BUG_LABEL, CLEAR_LABEL].filter((label) => !labels.includes(label)); + if (missing.length > 0) { + return skip(`missing ${missing.join(" and ")}`); + } + const blocking = labels.filter( + (label) => BLOCKING_LABELS.includes(label) || BLOCKING_PREFIXES.some((prefix) => label.startsWith(prefix)), + ); + if (blocking.length > 0) { + return skip(`carries ${blocking.join(", ")}`); + } + if (comments.some((comment) => comment.user.login === WORKFLOW_LOGIN && comment.body.includes(DISPATCH_MARKER))) { + return skip("was already dispatched once"); + } + const pulls = openPullsReferencing(timeline); + if (pulls.length === 1) { + return skip(`open pull request #${pulls[0]} already references it`); + } + if (pulls.length > 1) { + return skip(`open pull requests ${pulls.map((n) => `#${n}`).join(", ")} already reference it`); + } + return { kind: "dispatch" }; +} + +export async function dispatchIssue(api: GitHubApi, config: DispatchConfig): Promise { + const issuePath = `/repos/${config.repo}/issues/${config.issueNumber}`; + const issue = await api.request("GET", issuePath); + const comments = await listAll(api, `${issuePath}/comments`); + const timeline = await listAll(api, `${issuePath}/timeline`); + const verdict = dispatchTarget(issue, comments, timeline); + if (verdict.kind === "skip" || config.dryRun) { + return verdict; + } + await api.request("POST", `${issuePath}/labels`, { labels: [RUNNING_LABEL] }); + await api.request("POST", `${issuePath}/comments`, { body: DISPATCH_COMMENT }); + return verdict; +} + +function describe(config: DispatchConfig, verdict: DispatchVerdict): string { + if (verdict.kind === "skip") { + return `#${config.issueNumber}: skipped, ${verdict.reason}`; + } + if (config.dryRun) { + return `#${config.issueNumber}: DRY RUN, set the TRIAGE_DISPATCH_ENABLED repo variable to true to add ${RUNNING_LABEL} and comment`; + } + return `#${config.issueNumber}: added ${RUNNING_LABEL} and commented, Devin's automation takes it from here`; +} + +if (import.meta.main) { + const { token, ...config } = readConfig(process.env); + const verdict = await dispatchIssue(githubApi(token), config); + console.log(describe(config, verdict)); +} diff --git a/scripts/flag-duplicate-issue.test.ts b/scripts/flag-duplicate-issue.test.ts index 81785c668e8..434355da29d 100644 --- a/scripts/flag-duplicate-issue.test.ts +++ b/scripts/flag-duplicate-issue.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { candidateNumbers, duplicateTarget, type Comment, type GitHubApi, type Issue } from "./auto-close-duplicates"; import { + CLEAR_LABEL, MIN_CONFIDENCE, flagIssue, flagTarget, @@ -51,15 +52,15 @@ describe("parseVerdict", () => { }); describe("flagTarget", () => { - test("flags at the gate and not one hundredth below it", () => { + test("flags at the gate and clears one hundredth below it", () => { expect(flagTarget(verdict({ confidence: MIN_CONFIDENCE }), 35)).toEqual({ kind: "target", original: 10 }); - expect(flagTarget(verdict({ confidence: 0.94 }), 35).kind).toBe("skip"); + expect(flagTarget(verdict({ confidence: 0.94 }), 35).kind).toBe("clear"); }); - test("never flags nothing, itself, or a newer issue", () => { - expect(flagTarget(verdict({ duplicate_of: null }), 35).kind).toBe("skip"); - expect(flagTarget(verdict({ duplicate_of: 35 }), 35).kind).toBe("skip"); - expect(flagTarget(verdict({ duplicate_of: 36 }), 35).kind).toBe("skip"); + test("nothing, itself, or a newer issue all count as a clear result", () => { + expect(flagTarget(verdict({ duplicate_of: null }), 35).kind).toBe("clear"); + expect(flagTarget(verdict({ duplicate_of: 35 }), 35).kind).toBe("clear"); + expect(flagTarget(verdict({ duplicate_of: 36 }), 35).kind).toBe("clear"); }); }); @@ -163,12 +164,27 @@ describe("flagIssue", () => { expect(writes).toEqual([]); }); - test("a verdict below the gate never touches the API", async () => { + test("a verdict naming no duplicate adds dup:clear and nothing else", async () => { const { api, writes } = fakeApi(); - expect((await flagIssue(api, config, verdict({ confidence: 0.9 }))).kind).toBe("skip"); + expect(await flagIssue(api, config, verdict({ duplicate_of: null }))).toEqual({ kind: "clear", reason: "no duplicate named" }); + expect(writes).toEqual([`POST /repos/BerriAI/litellm/issues/35/labels {"labels":["${CLEAR_LABEL}"]}`]); + }); + + test("a verdict below the gate clears the issue too, and a dry run only reports it", async () => { + const real = fakeApi(); + expect((await flagIssue(real.api, config, verdict({ confidence: 0.9 }))).kind).toBe("clear"); + expect(real.writes).toEqual([`POST /repos/BerriAI/litellm/issues/35/labels {"labels":["${CLEAR_LABEL}"]}`]); + + const { api, writes } = fakeApi(); + expect((await flagIssue(api, { ...config, dryRun: true }, verdict({ confidence: 0.9 }))).kind).toBe("clear"); expect(writes).toEqual([]); }); + test("a garbled verdict is a skip, never a clear", () => { + expect(parseVerdict("not json").kind).toBe("skip"); + expect(CLEAR_LABEL).toBe("dup:clear"); + }); + test("an issue that already carries a notice is not flagged twice", async () => { const existing: Comment = { id: 1, @@ -179,6 +195,25 @@ describe("flagIssue", () => { const { api, writes } = fakeApi(undefined, [existing]); expect(await flagIssue(api, config, verdict())).toEqual({ kind: "skip", reason: "already carries a duplicate notice" }); expect(writes).toEqual([]); + + const rerun = fakeApi(undefined, [existing]); + expect(await flagIssue(rerun.api, config, verdict({ duplicate_of: null }))).toEqual({ + kind: "skip", + reason: "already carries a duplicate notice", + }); + expect(rerun.writes).toEqual([]); + }); + + test("a notice marker pasted by a commenter is not a notice", async () => { + const pasted: Comment = { + id: 2, + body: " looks like #10 to me", + created_at: "2026-09-10T00:00:00Z", + user: { type: "User", login: "someone" }, + }; + const { api, writes } = fakeApi(undefined, [pasted]); + expect((await flagIssue(api, config, verdict({ duplicate_of: null }))).kind).toBe("clear"); + expect(writes).toEqual([`POST /repos/BerriAI/litellm/issues/35/labels {"labels":["${CLEAR_LABEL}"]}`]); }); test("a failed comment leaves no marker, so the rerun finishes the job", async () => { diff --git a/scripts/flag-duplicate-issue.ts b/scripts/flag-duplicate-issue.ts index f10bb625ec8..08856b87fa9 100644 --- a/scripts/flag-duplicate-issue.ts +++ b/scripts/flag-duplicate-issue.ts @@ -10,6 +10,7 @@ import { type GitHubApi, type Issue, } from "./auto-close-duplicates"; +import { labelName } from "./issue-labels"; declare const process: { readonly env: Readonly> }; @@ -31,16 +32,20 @@ export type ParsedVerdict = export type FlagTarget = | { readonly kind: "target"; readonly original: number } - | { readonly kind: "skip"; readonly reason: string }; + | { readonly kind: "clear"; readonly reason: string }; export type FlagVerdict = | { readonly kind: "flagged"; readonly original: number; readonly body: string } + | { readonly kind: "clear"; readonly reason: string } | { readonly kind: "skip"; readonly reason: string }; export const MIN_CONFIDENCE = 0.95; export const NOTICE_MARKER_PREFIX = "