diff --git a/.github/scripts/close_duplicate_issues.py b/.github/scripts/close_duplicate_issues.py deleted file mode 100755 index ec522af4f88..00000000000 --- a/.github/scripts/close_duplicate_issues.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python3 -""" -Detect and close duplicate GitHub issues using title similarity. - -Modes: - --scan Compare all open issues against each other (batch) - --issue-number N Check a single issue against older open issues - -Requires the `gh` CLI to be authenticated. -""" - -import argparse -import difflib -import json -import re -import subprocess -import sys - - -def normalize_title(title: str) -> str: - """Strip common prefixes, lowercase, and collapse whitespace.""" - title = re.sub( - r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*", - "", - title, - flags=re.IGNORECASE, - ) - return " ".join(title.lower().split()) - - -def gh(*args: str) -> str: - """Run a gh CLI command and return stdout.""" - result = subprocess.run( - ["gh", *args], - capture_output=True, - text=True, - check=True, - ) - return result.stdout - - -def fetch_open_issues(repo: str | None) -> list[dict]: - """Fetch all open issues (excluding PRs) via gh api --paginate.""" - if repo: - endpoint = ( - f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - ) - else: - endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - cmd = ["api", "--paginate", endpoint] - - raw = gh(*cmd) - # gh --paginate concatenates JSON arrays, so we may get multiple arrays - issues = [] - for line in raw.strip().splitlines(): - line = line.strip() - if not line: - continue - parsed = json.loads(line) - if isinstance(parsed, list): - issues.extend(parsed) - else: - issues.append(parsed) - - # Filter out pull requests (they also appear in the issues endpoint) - return [i for i in issues if "pull_request" not in i] - - -def close_as_duplicate( - issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool -) -> None: - """Close an issue as duplicate of another, adding a comment and label.""" - repo_args = ["--repo", repo] if repo else [] - - if dry_run: - print( - f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}" - ) - return - - # Add comment - comment_body = ( - f"Closing as duplicate of #{duplicate_of}.\n\n" - "If you believe this is not a duplicate, please reopen and add context " - "explaining how this differs." - ) - gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args) - - # Add label - gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args) - - # Close with not_planned reason - gh( - "api", - f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}", - "-X", - "PATCH", - "-f", - "state=closed", - "-f", - "state_reason=not_planned", - ) - - print(f" Closed #{issue_number} as duplicate of #{duplicate_of}") - - -def find_duplicate( - issue: dict, candidates: list[dict], threshold: float -) -> dict | None: - """Return the first candidate whose normalized title is above threshold.""" - norm = normalize_title(issue["title"]) - for candidate in candidates: - if candidate["number"] == issue["number"]: - continue - cand_norm = normalize_title(candidate["title"]) - ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio() - if ratio >= threshold: - return candidate - return None - - -def scan_all( - issues: list[dict], threshold: float, repo: str | None, dry_run: bool -) -> int: - """Compare every issue against all older issues. Returns count of duplicates found.""" - # Sort oldest first - issues.sort(key=lambda i: i["number"]) - closed_count = 0 - - for idx, issue in enumerate(issues): - older = issues[:idx] - if not older: - continue - dup = find_duplicate(issue, older, threshold) - if dup: - ratio = difflib.SequenceMatcher( - None, - normalize_title(issue["title"]), - normalize_title(dup["title"]), - ).ratio() - print( - f"#{issue['number']}: \"{issue['title']}\"\n" - f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " - f"({ratio:.0%} similar)" - ) - close_as_duplicate(issue["number"], dup["number"], repo, dry_run) - closed_count += 1 - - return closed_count - - -def check_single( - issue_number: int, - issues: list[dict], - threshold: float, - repo: str | None, - dry_run: bool, -) -> bool: - """Check a single issue against all older open issues. Returns True if duplicate found.""" - target = None - for i in issues: - if i["number"] == issue_number: - target = i - break - - if target is None: - print(f"Issue #{issue_number} not found among open issues.") - return False - - older = [i for i in issues if i["number"] < issue_number] - dup = find_duplicate(target, older, threshold) - if dup: - ratio = difflib.SequenceMatcher( - None, - normalize_title(target["title"]), - normalize_title(dup["title"]), - ).ratio() - print( - f"#{target['number']}: \"{target['title']}\"\n" - f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " - f"({ratio:.0%} similar)" - ) - close_as_duplicate(issue_number, dup["number"], repo, dry_run) - return True - - print(f"#{issue_number}: no duplicate found above threshold {threshold}") - return False - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Detect and close duplicate GitHub issues" - ) - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--scan", action="store_true", help="Scan all open issues") - mode.add_argument("--issue-number", type=int, help="Check a single issue number") - parser.add_argument( - "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)" - ) - parser.add_argument( - "--close", - action="store_true", - help="Actually close duplicates (default is dry-run)", - ) - parser.add_argument( - "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted." - ) - args = parser.parse_args() - - dry_run = not args.close - - if dry_run: - print("=== DRY RUN MODE (pass --close to actually close issues) ===\n") - - print("Fetching open issues...") - issues = fetch_open_issues(args.repo) - print(f"Found {len(issues)} open issues.\n") - - if args.scan: - count = scan_all(issues, args.threshold, args.repo, dry_run) - print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}") - else: - found = check_single( - args.issue_number, issues, args.threshold, args.repo, dry_run - ) - sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/auto-close-duplicates.yml b/.github/workflows/auto-close-duplicates.yml new file mode 100644 index 00000000000..d8256917805 --- /dev/null +++ b/.github/workflows/auto-close-duplicates.yml @@ -0,0 +1,69 @@ +name: Auto-close duplicate issues + +on: + schedule: + - cron: "0 9 * * *" + workflow_dispatch: + inputs: + dry_run: + description: Log which issues would close without closing anything + type: boolean + default: true + grace_period_days: + description: Days a duplicate notice must go unanswered before the close + type: number + default: 3 + pull_request: + paths: + - .github/workflows/auto-close-duplicates.yml + - scripts/auto-close-duplicates.ts + - scripts/auto-close-duplicates.test.ts + +permissions: {} + +jobs: + test: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.4.0" + + - name: Test the sweep + run: bun test scripts/auto-close-duplicates.test.ts + + sweep: + if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + # Exact version, never latest: the next step holds an issues: write token + bun-version: "1.4.0" + + - name: Close unanswered duplicates, reopen ones the reporter answered + run: bun run scripts/auto-close-duplicates.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ inputs.dry_run == true }} + GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }} diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 78198b2c7bb..41ec43a1d9b 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -1,12 +1,19 @@ name: Check Duplicate Issues +# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later, +# and only when its title is identical to an older open issue and nobody replied. +# The HTML marker below is the handshake between the two, so keep it in the template. + on: issues: types: [opened, edited] +permissions: {} + jobs: check-duplicate: runs-on: ubuntu-latest + timeout-minutes: 5 permissions: issues: write contents: read @@ -19,35 +26,12 @@ jobs: threshold: 0.6 reaction: eyes comment: | - **⚠️ Potential duplicate detected** + + **Potential duplicate detected** - This issue appears similar to existing issue(s): + This looks similar to: {{#issues}} - - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar) + - #{{number}} - {{title}} {{/issues}} - Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference. - - - name: Checkout close script - if: github.event.action == 'opened' - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - sparse-checkout: .github/scripts - persist-credentials: false - - - name: Set up Python - if: github.event.action == 'opened' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Auto-close if high-confidence duplicate - if: github.event.action == 'opened' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - python3 .github/scripts/close_duplicate_issues.py \ - --issue-number ${{ github.event.issue.number }} \ - --repo ${{ github.repository }} \ - --threshold 0.85 \ - --close + If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open. diff --git a/scripts/auto-close-duplicates.test.ts b/scripts/auto-close-duplicates.test.ts new file mode 100644 index 00000000000..b49bf05cbc2 --- /dev/null +++ b/scripts/auto-close-duplicates.test.ts @@ -0,0 +1,348 @@ +import { describe, expect, test } from "bun:test"; + +import { + CLOSED_MARKER, + REOPEN_COMMENT, + candidateNumbers, + duplicateTarget, + normalizeTitle, + pendingNotice, + readConfig, + reopenTarget, + sweepClosedIssue, + sweepIssue, + type Comment, + type GitHubApi, + type Issue, + type Reaction, + type SweepConfig, +} from "./auto-close-duplicates"; + +const NOW = new Date("2026-09-04T09:00:00Z"); +const DAY_MS = 24 * 60 * 60 * 1000; +const daysAgo = (days: number): string => new Date(NOW.getTime() - days * DAY_MS).toISOString(); + +const issue = (number: number, title: string, overrides: Partial = {}): Issue => ({ + number, + title, + state: "open", + user: { login: "reporter" }, + ...overrides, +}); + +const notice = (candidates: readonly number[], createdAt: string, overrides: Partial = {}): Comment => ({ + id: 900, + body: `\n**Potential duplicate detected**`, + created_at: createdAt, + user: { type: "Bot", login: "github-actions[bot]" }, + ...overrides, +}); + +const humanComment = (createdAt: string, body = "It is not the same thing", login = "reporter"): Comment => ({ + id: 901, + body, + created_at: createdAt, + user: { type: "User", login }, +}); + +const config: SweepConfig = { repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW }; + +describe("normalizeTitle", () => { + test("drops the template prefix, case, and punctuation", () => { + expect(normalizeTitle("[Bug]: Gemma 4-e4b fails on Vertex!")).toBe("gemma 4 e4b fails on vertex"); + expect(normalizeTitle("[Feature]: ")).toBe(""); + }); +}); + +describe("candidateNumbers", () => { + test("reads only the marker field, keeps older issues, sorted ascending and deduplicated", () => { + const body = "\n- #1 - see #1 (100% similar)"; + expect(candidateNumbers(body, 35)).toEqual([10, 30]); + }); + + test("returns nothing without the marker", () => { + expect(candidateNumbers("- #1 - looks like #1", 35)).toEqual([]); + }); +}); + +describe("pendingNotice", () => { + test("waits out the grace period from the latest notice", () => { + const fresh = pendingNotice(issue(35, "t"), [notice([10], daysAgo(2.9))], config); + expect(fresh.kind).toBe("skip"); + const aged = pendingNotice(issue(35, "t"), [notice([10], daysAgo(3.1))], config); + expect(aged.kind).toBe("pending"); + const reposted = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(6)), notice([10], daysAgo(1), { id: 902 })], + config, + ); + expect(reposted.kind).toBe("skip"); + }); + + test("an objection posted before a re-posted notice still keeps the issue open", () => { + const verdict = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(10)), humanComment(daysAgo(7)), notice([10], daysAgo(4), { id: 902 })], + config, + ); + expect(verdict).toEqual({ kind: "skip", reason: "someone replied after the notice" }); + }); + + test("a zero-day grace period acts on the notice at once", () => { + const verdict = pendingNotice(issue(35, "t"), [notice([10], daysAgo(0.01))], { ...config, graceDays: 0 }); + expect(verdict.kind).toBe("pending"); + }); + + test("a human reply after the notice keeps the issue open, a bot reply does not", () => { + const human = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5)), humanComment(daysAgo(4))], config); + expect(human).toEqual({ kind: "skip", reason: "someone replied after the notice" }); + const bot = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(5)), { id: 903, body: "triage", created_at: daysAgo(4), user: { type: "Bot", login: "triage[bot]" } }], + config, + ); + expect(bot.kind).toBe("pending"); + }); + + test("a human quoting the marker is not a notice", () => { + const quoted = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5), { user: { type: "User", login: "reporter" } })], config); + expect(quoted).toEqual({ kind: "skip", reason: "carries no duplicate notice" }); + }); + + test("never closes an issue twice: a reopened issue is left alone", () => { + const reopened = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(9)), { id: 904, body: `Closed automatically\n\n${CLOSED_MARKER}`, created_at: daysAgo(5), user: { type: "Bot", login: "github-actions[bot]" } }], + config, + ); + expect(reopened).toEqual({ kind: "skip", reason: "was reopened after an automatic close" }); + }); + + test("skips pull requests and issues whose only candidates are newer", () => { + expect(pendingNotice(issue(35, "t", { pull_request: {} }), [notice([10], daysAgo(5))], config).kind).toBe("skip"); + expect(pendingNotice(issue(35, "t"), [notice([40], daysAgo(5))], config)).toEqual({ + kind: "skip", + reason: "no candidate is older than this issue", + }); + }); +}); + +describe("duplicateTarget", () => { + const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex"); + + test("closes only against the earliest open issue with the identical normalized title", () => { + const verdict = duplicateTarget( + reporter, + [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex"), issue(20, "[bug]: gemma 4-e4b fails on vertex"), issue(30, "[Bug]: Gemma 4-e4b fails on Vertex")], + [], + ); + expect(verdict).toEqual({ kind: "close", duplicateOf: 20 }); + }); + + test("a near miss in the title is not a duplicate", () => { + const verdict = duplicateTarget(reporter, [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex")], []); + expect(verdict).toEqual({ kind: "skip", reason: "no older open issue has the identical title" }); + }); + + test("bare template titles never match each other", () => { + const verdict = duplicateTarget(issue(35, "[Bug]: "), [issue(10, "[Bug]: ")], []); + expect(verdict.kind).toBe("skip"); + expect(verdict.kind === "skip" && verdict.reason).toContain("too short"); + }); + + test("a closed candidate or a pull request is never the target", () => { + expect(duplicateTarget(reporter, [issue(10, reporter.title, { state: "closed" })], []).kind).toBe("skip"); + expect(duplicateTarget(reporter, [issue(10, reporter.title, { pull_request: {} })], []).kind).toBe("skip"); + }); + + test("a thumbs down on the notice keeps the issue open", () => { + const verdict = duplicateTarget(reporter, [issue(10, reporter.title)], [{ content: "+1" }, { content: "-1" }]); + expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" }); + }); +}); + +describe("sweepIssue", () => { + const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex"); + const original = issue(10, "[Bug]: Gemma 4-e4b fails on Vertex"); + + function fakeApi( + comments: readonly Comment[] = [notice([10], daysAgo(5))], + reactionsByNotice: Readonly> = {}, + ): { readonly api: GitHubApi; readonly writes: readonly string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) { + return comments as T; + } + const reactionsPath = path.match(/^\/repos\/BerriAI\/litellm\/issues\/comments\/(\d+)\/reactions/); + if (reactionsPath) { + return (reactionsByNotice[Number(reactionsPath[1])] ?? []) as T; + } + if (path === "/repos/BerriAI/litellm/issues/10") { + return original as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + test("a dry run reports the close and writes nothing", async () => { + const { api, writes } = fakeApi(); + const verdict = await sweepIssue(api, { ...config, dryRun: true }, reporter); + expect(verdict).toEqual({ kind: "close", duplicateOf: 10 }); + expect(writes).toEqual([]); + }); + + test("a thumbs down on an earlier notice still keeps the issue open", async () => { + const { api, writes } = fakeApi([notice([10], daysAgo(9)), notice([10], daysAgo(5), { id: 902 })], { 900: [{ content: "-1" }] }); + const verdict = await sweepIssue(api, config, reporter); + expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" }); + expect(writes).toEqual([]); + }); + + test("a real run comments, labels, then closes with the duplicate reason", async () => { + const { api, writes } = fakeApi(); + const verdict = await sweepIssue(api, config, reporter); + expect(verdict).toEqual({ kind: "close", duplicateOf: 10 }); + expect(writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([ + "POST /repos/BerriAI/litellm/issues/35/comments", + "POST /repos/BerriAI/litellm/issues/35/labels", + "PATCH /repos/BerriAI/litellm/issues/35", + ]); + expect(writes[0]).toContain("duplicate of #10"); + expect(writes[0]).toContain("unanswered for 3 days"); + expect(writes[0]).toContain(CLOSED_MARKER); + expect(writes[1]).toContain('{"labels":["duplicate"]}'); + expect(writes[2]).toContain('{"state":"closed","state_reason":"duplicate"}'); + }); +}); + +describe("reopenTarget", () => { + const closedByBot = (overrides: Partial = {}): Issue => + issue(35, "t", { state: "closed", closed_by: { type: "Bot" }, ...overrides }); + const closeMarker = (createdAt: string): Comment => ({ + id: 905, + body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`, + created_at: createdAt, + user: { type: "Bot", login: "github-actions[bot]" }, + }); + + test("a reporter reply after the automatic close reopens", () => { + const verdict = reopenTarget(closedByBot(), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "reopen" }); + }); + + test("an issue closed by a person stays closed", () => { + const verdict = reopenTarget(closedByBot({ closed_by: { type: "User" } }), [ + closeMarker(daysAgo(2)), + humanComment(daysAgo(1)), + ]); + expect(verdict).toEqual({ kind: "skip", reason: "was closed by a person" }); + }); + + test("without the automatic-close marker nothing reopens", () => { + const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "skip", reason: "carries no automatic-close marker" }); + }); + + test("a maintainer reply alone does not reopen", () => { + const verdict = reopenTarget(closedByBot(), [ + closeMarker(daysAgo(2)), + humanComment(daysAgo(1), "Confirmed duplicate", "maintainer"), + ]); + expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" }); + }); + + test("a reporter comment from before the close does not reopen", () => { + const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(3)), closeMarker(daysAgo(2))]); + expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" }); + }); + + test("a pull request never reopens", () => { + const verdict = reopenTarget(closedByBot({ pull_request: {} }), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "skip", reason: "is a pull request" }); + }); +}); + +describe("sweepClosedIssue", () => { + function fakeApi(issueBody: Issue, comments: readonly Comment[]): { readonly api: GitHubApi; readonly writes: readonly string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) { + return comments as T; + } + if (path === "/repos/BerriAI/litellm/issues/35") { + return issueBody as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + const closedByBot = issue(35, "t", { state: "closed", closed_by: { type: "Bot" } }); + const closeMarker: Comment = { + id: 905, + body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`, + created_at: daysAgo(2), + user: { type: "Bot", login: "github-actions[bot]" }, + }; + + test("a real run unlabels, reopens, then explains", async () => { + const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]); + const verdict = await sweepClosedIssue(api, config, 35); + expect(verdict).toEqual({ kind: "reopen" }); + expect(writes).toEqual([ + "DELETE /repos/BerriAI/litellm/issues/35/labels/duplicate undefined", + 'PATCH /repos/BerriAI/litellm/issues/35 {"state":"open"}', + `POST /repos/BerriAI/litellm/issues/35/comments {"body":"${REOPEN_COMMENT}"}`, + ]); + }); + + test("a dry run reports the reopen and writes nothing", async () => { + const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]); + const verdict = await sweepClosedIssue(api, { ...config, dryRun: true }, 35); + expect(verdict).toEqual({ kind: "reopen" }); + expect(writes).toEqual([]); + }); +}); + +describe("readConfig", () => { + test("defaults to a real run with a 3-day grace period", () => { + const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm" }, NOW); + expect(parsed).toEqual({ token: "t", repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW }); + }); + + test("honors DRY_RUN and GRACE_PERIOD_DAYS overrides", () => { + const parsed = readConfig( + { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", DRY_RUN: "true", GRACE_PERIOD_DAYS: "0" }, + NOW, + ); + expect(parsed.dryRun).toBe(true); + expect(parsed.graceDays).toBe(0); + }); + + test("an empty GRACE_PERIOD_DAYS, as a schedule run renders it, means the default", () => { + const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "" }, NOW); + expect(parsed.graceDays).toBe(3); + }); + + test("refuses a missing token, a malformed repository, or a bad grace period", () => { + expect(() => readConfig({ GITHUB_REPOSITORY: "o/r" }, NOW)).toThrow("GITHUB_TOKEN"); + expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "litellm" }, NOW)).toThrow("owner/repo"); + expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "-1" }, NOW)).toThrow( + "GRACE_PERIOD_DAYS", + ); + }); +}); diff --git a/scripts/auto-close-duplicates.ts b/scripts/auto-close-duplicates.ts new file mode 100644 index 00000000000..c595104d886 --- /dev/null +++ b/scripts/auto-close-duplicates.ts @@ -0,0 +1,300 @@ +#!/usr/bin/env bun + +declare const process: { readonly env: Readonly> }; + +export interface Issue { + readonly number: number; + readonly title: string; + readonly state: string; + readonly user: { readonly login: string }; + readonly closed_by?: { readonly type: string } | null; + readonly pull_request?: unknown; +} + +export interface Comment { + readonly id: number; + readonly body: string; + readonly created_at: string; + readonly user: { readonly type: string; readonly login: string }; +} + +export interface Reaction { + readonly content: string; +} + +export interface GitHubApi { + readonly request: (method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object) => Promise; +} + +export interface SweepConfig { + readonly repo: string; + readonly graceDays: number; + readonly dryRun: boolean; + readonly now: Date; +} + +export type NoticeVerdict = + | { readonly kind: "pending"; readonly notices: readonly Comment[]; readonly candidates: readonly number[] } + | { readonly kind: "skip"; readonly reason: string }; + +export type CloseVerdict = + | { readonly kind: "close"; readonly duplicateOf: number } + | { readonly kind: "skip"; readonly reason: string }; + +export type ReopenVerdict = + | { readonly kind: "reopen" } + | { readonly kind: "skip"; readonly reason: string }; + +export const FLAG_LABEL = "potential-duplicate"; +export const CLOSED_MARKER = ""; +export const DEFAULT_GRACE_DAYS = 3; +export const REOPEN_COMMENT = + "Reopened automatically: the reporter replied after the duplicate close, so this needs a human look."; +const NOTICE_MARKER = //; +const MIN_TITLE_WORDS = 3; +const PAGE_SIZE = 100; +const DAY_MS = 24 * 60 * 60 * 1000; +const REOPEN_LOOKBACK_DAYS = 30; + +const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason }); + +export function normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/^\s*\[[^\]]*\]\s*:?/, "") + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +export function candidateNumbers(noticeBody: string, issueNumber: number): readonly number[] { + const field = noticeBody.match(NOTICE_MARKER); + if (!field) { + return []; + } + const older = field[1] + .split(",") + .filter((value) => value !== "") + .map(Number) + .filter((candidate) => candidate < issueNumber); + return [...new Set(older)].sort((a, b) => a - b); +} + +export function pendingNotice( + issue: Issue, + comments: readonly Comment[], + config: Pick, +): NoticeVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (comments.some((comment) => comment.body.includes(CLOSED_MARKER))) { + return skip("was reopened after an automatic close"); + } + const notices = comments.filter((comment) => comment.user.type === "Bot" && NOTICE_MARKER.test(comment.body)); + const first = notices[0]; + const latest = notices[notices.length - 1]; + if (first === undefined || latest === undefined) { + return skip("carries no duplicate notice"); + } + const ageDays = (config.now.getTime() - new Date(latest.created_at).getTime()) / DAY_MS; + if (ageDays < config.graceDays) { + return skip(`notice is ${ageDays.toFixed(1)} days old, grace period is ${config.graceDays}`); + } + const firstNoticeAt = new Date(first.created_at); + if (comments.some((comment) => comment.user.type !== "Bot" && new Date(comment.created_at) > firstNoticeAt)) { + return skip("someone replied after the notice"); + } + const candidates = candidateNumbers(latest.body, issue.number); + if (candidates.length === 0) { + return skip("no candidate is older than this issue"); + } + return { kind: "pending", notices, candidates }; +} + +export function duplicateTarget( + issue: Issue, + candidates: readonly Issue[], + reactions: readonly Reaction[], +): CloseVerdict { + if (reactions.some((reaction) => reaction.content === "-1")) { + return skip("someone gave the notice a thumbs down"); + } + const title = normalizeTitle(issue.title); + if (title.split(" ").length < MIN_TITLE_WORDS) { + return skip(`title "${issue.title}" is too short to match on`); + } + const original = candidates.find( + (candidate) => + candidate.state === "open" && candidate.pull_request === undefined && normalizeTitle(candidate.title) === title, + ); + if (original === undefined) { + return skip("no older open issue has the identical title"); + } + return { kind: "close", duplicateOf: original.number }; +} + +export function reopenTarget(issue: Issue, comments: readonly Comment[]): ReopenVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (issue.closed_by?.type !== "Bot") { + return skip("was closed by a person"); + } + const marker = comments.find((comment) => comment.body.includes(CLOSED_MARKER)); + if (marker === undefined) { + return skip("carries no automatic-close marker"); + } + const markerAt = new Date(marker.created_at); + if (!comments.some((comment) => comment.user.login === issue.user.login && new Date(comment.created_at) > markerAt)) { + return skip("the reporter has not replied since the close"); + } + return { kind: "reopen" }; +} + +export function closingComment(duplicateOf: number, graceDays: number): string { + return `Closed automatically as a duplicate of #${duplicateOf}. Its title is identical to that older open issue and the duplicate notice above went unanswered for ${graceDays} days. If this is wrong, comment here with how it differs from #${duplicateOf} and this issue will be reopened automatically within a day. + +${CLOSED_MARKER}`; +} + +async function listAll(api: GitHubApi, path: string, page = 1): Promise { + const separator = path.includes("?") ? "&" : "?"; + const batch = await api.request("GET", `${path}${separator}per_page=${PAGE_SIZE}&page=${page}`); + return batch.length < PAGE_SIZE ? batch : [...batch, ...(await listAll(api, path, page + 1))]; +} + +async function closeAsDuplicate( + api: GitHubApi, + config: SweepConfig, + issueNumber: number, + duplicateOf: number, +): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; + await api.request("POST", `${issuePath}/comments`, { body: closingComment(duplicateOf, config.graceDays) }); + await api.request("POST", `${issuePath}/labels`, { labels: ["duplicate"] }); + await api.request("PATCH", issuePath, { state: "closed", state_reason: "duplicate" }); +} + +async function reopenForReporter(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; + await api.request("DELETE", `${issuePath}/labels/duplicate`); + await api.request("PATCH", issuePath, { state: "open" }); + await api.request("POST", `${issuePath}/comments`, { body: REOPEN_COMMENT }); +} + +export async function sweepClosedIssue(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise { + const issue = await api.request("GET", `/repos/${config.repo}/issues/${issueNumber}`); + const comments = await listAll(api, `/repos/${config.repo}/issues/${issueNumber}/comments`); + const verdict = reopenTarget(issue, comments); + if (verdict.kind === "reopen" && !config.dryRun) { + await reopenForReporter(api, config, issueNumber); + } + return verdict; +} + +export async function sweepIssue(api: GitHubApi, config: SweepConfig, issue: Issue): Promise { + const comments = await listAll(api, `/repos/${config.repo}/issues/${issue.number}/comments`); + const pending = pendingNotice(issue, comments, config); + if (pending.kind === "skip") { + return pending; + } + const reactions = ( + await Promise.all( + pending.notices.map((notice) => listAll(api, `/repos/${config.repo}/issues/comments/${notice.id}/reactions`)), + ) + ).flat(); + const candidates = await Promise.all( + pending.candidates.map((candidate) => api.request("GET", `/repos/${config.repo}/issues/${candidate}`)), + ); + const verdict = duplicateTarget(issue, candidates, reactions); + if (verdict.kind === "close" && !config.dryRun) { + await closeAsDuplicate(api, config, issue.number, verdict.duplicateOf); + } + return verdict; +} + +function describe(issue: Issue, verdict: CloseVerdict, dryRun: boolean): string { + if (verdict.kind === "skip") { + return `#${issue.number}: skipped, ${verdict.reason}`; + } + return `#${issue.number}: ${dryRun ? "would close" : "closed"} as a duplicate of #${verdict.duplicateOf}`; +} + +export async function sweep(api: GitHubApi, config: SweepConfig): Promise { + const issues = await listAll(api, `/repos/${config.repo}/issues?state=open&labels=${FLAG_LABEL}`); + console.log(`${issues.length} open issues carry the ${FLAG_LABEL} label in ${config.repo}${config.dryRun ? " (dry run)" : ""}`); + return issues.reduce>(async (previous, issue) => { + const verdicts = await previous; + const verdict = await sweepIssue(api, config, issue); + console.log(describe(issue, verdict, config.dryRun)); + return [...verdicts, verdict]; + }, Promise.resolve([])); +} + +function describeReopen(issueNumber: number, verdict: ReopenVerdict, dryRun: boolean): string { + if (verdict.kind === "skip") { + return `#${issueNumber}: skipped, ${verdict.reason}`; + } + return `#${issueNumber}: ${dryRun ? "would reopen" : "reopened"} for the reporter's reply`; +} + +export async function reopenSweep(api: GitHubApi, config: SweepConfig): Promise { + const since = new Date(config.now.getTime() - REOPEN_LOOKBACK_DAYS * DAY_MS).toISOString(); + const closedPath = `/repos/${config.repo}/issues?state=closed&labels=duplicate,${FLAG_LABEL}&since=${encodeURIComponent(since)}`; + const issues = await listAll(api, closedPath); + console.log(`${issues.length} recently closed issues carry the duplicate and ${FLAG_LABEL} labels in ${config.repo}${config.dryRun ? " (dry run)" : ""}`); + return issues.reduce>(async (previous, issue) => { + const verdicts = await previous; + const verdict = await sweepClosedIssue(api, config, issue.number); + console.log(describeReopen(issue.number, verdict, config.dryRun)); + return [...verdicts, verdict]; + }, Promise.resolve([])); +} + +export function readConfig(env: Readonly>, now: Date): SweepConfig & { readonly token: string } { + const token = env.GITHUB_TOKEN; + const repo = env.GITHUB_REPOSITORY; + if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo)) { + throw new Error("GITHUB_TOKEN and GITHUB_REPOSITORY (owner/repo) are required"); + } + const rawGraceDays = env.GRACE_PERIOD_DAYS?.trim(); + const graceDays = rawGraceDays === undefined || rawGraceDays === "" ? DEFAULT_GRACE_DAYS : Number(rawGraceDays); + if (!Number.isFinite(graceDays) || graceDays < 0) { + throw new Error(`GRACE_PERIOD_DAYS must be a non-negative number, got "${env.GRACE_PERIOD_DAYS}"`); + } + return { token, repo, graceDays, dryRun: env.DRY_RUN === "true", now }; +} + +export function githubApi(token: string): GitHubApi { + return { + request: async (method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object): Promise => { + const response = await fetch(`https://api.github.com${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "litellm-auto-close-duplicates", + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`${method} ${path} failed: ${response.status} ${response.statusText}`); + } + return (await response.json()) as T; + }, + }; +} + +if (import.meta.main) { + const { token, ...config } = readConfig(process.env, new Date()); + const api = githubApi(token); + const closeVerdicts = await sweep(api, config); + const reopenVerdicts = await reopenSweep(api, config); + const closed = closeVerdicts.filter((verdict) => verdict.kind === "close").length; + const reopened = reopenVerdicts.filter((verdict) => verdict.kind === "reopen").length; + console.log( + `${config.dryRun ? "Would close" : "Closed"} ${closed} of ${closeVerdicts.length} flagged issues, ${config.dryRun ? "would reopen" : "reopened"} ${reopened}`, + ); +}