From 8f694ddc7cec47184c49db6b90d10a251fa07cbe Mon Sep 17 00:00:00 2001 From: wowo Date: Thu, 9 Apr 2026 15:04:33 +0800 Subject: [PATCH] [codex] add issue triage automation mvp (#268) * add issue triage automation mvp * Document issue automation design in Chinese * Fix legacy compat slug tests --- .github/scripts/github.ts | 230 +++++ .github/scripts/issue-backlog-rescore.ts | 128 +++ .github/scripts/issue-handoff-brief.ts | 257 +++++ .github/scripts/issue-llm-config.ts | 139 +++ .github/scripts/issue-llm-evaluator.ts | 466 +++++++++ .github/scripts/issue-llm-provider.ts | 206 ++++ .github/scripts/issue-llm-types.ts | 62 ++ .github/scripts/issue-triage-config.ts | 236 +++++ .github/scripts/issue-triage-lib.ts | 923 ++++++++++++++++++ .github/scripts/issue-triage-merge.ts | 166 ++++ .github/scripts/issue-triage-types.ts | 93 ++ .github/scripts/issue-triage.ts | 129 +++ .github/workflows/issue-backlog-rescore.yml | 51 + .github/workflows/issue-triage.yml | 62 ++ docs/2026-04-08-issue-automation-design.md | 323 ++++++ .../compat/ClawHubCompatControllerTest.java | 23 +- 16 files changed, 3491 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/github.ts create mode 100644 .github/scripts/issue-backlog-rescore.ts create mode 100644 .github/scripts/issue-handoff-brief.ts create mode 100644 .github/scripts/issue-llm-config.ts create mode 100644 .github/scripts/issue-llm-evaluator.ts create mode 100644 .github/scripts/issue-llm-provider.ts create mode 100644 .github/scripts/issue-llm-types.ts create mode 100644 .github/scripts/issue-triage-config.ts create mode 100644 .github/scripts/issue-triage-lib.ts create mode 100644 .github/scripts/issue-triage-merge.ts create mode 100644 .github/scripts/issue-triage-types.ts create mode 100644 .github/scripts/issue-triage.ts create mode 100644 .github/workflows/issue-backlog-rescore.yml create mode 100644 .github/workflows/issue-triage.yml create mode 100644 docs/2026-04-08-issue-automation-design.md diff --git a/.github/scripts/github.ts b/.github/scripts/github.ts new file mode 100644 index 00000000..7b9778b5 --- /dev/null +++ b/.github/scripts/github.ts @@ -0,0 +1,230 @@ +interface GitHubUser { + login: string; +} + +interface GitHubLabelRef { + name?: string; +} + +export interface GitHubIssue { + number: number; + title: string; + body: string | null; + state: string; + labels: GitHubLabelRef[]; + comments: number; + created_at: string; + updated_at: string; + user: GitHubUser; + html_url: string; + pull_request?: Record; +} + +export interface GitHubIssueComment { + id: number; + body: string; + user: GitHubUser; + created_at: string; + updated_at: string; + html_url: string; +} + +export interface GitHubLabelDefinition { + name: string; + color: string; + description: string; +} + +function buildApiUrl(path: string) { + return `https://api.github.com${path}`; +} + +export class GitHubClient { + constructor( + private readonly token: string, + private readonly owner: string, + private readonly repo: string, + ) {} + + async getIssue(issueNumber: number): Promise { + return this.request( + "GET", + `/repos/${this.owner}/${this.repo}/issues/${issueNumber}`, + ); + } + + async listIssueComments(issueNumber: number): Promise { + return this.paginate( + `/repos/${this.owner}/${this.repo}/issues/${issueNumber}/comments?per_page=100`, + ); + } + + async listOpenIssuesByLabel( + label: string, + limit = 0, + ): Promise { + const collected: GitHubIssue[] = []; + const unlimited = limit === 0; + let page = 1; + + while (unlimited || collected.length < limit) { + const pageItems = await this.request( + "GET", + `/repos/${this.owner}/${this.repo}/issues?state=open&labels=${ + encodeURIComponent(label) + }&per_page=100&page=${page}`, + ); + + const nonPrIssues = pageItems.filter((item) => !item.pull_request); + collected.push(...nonPrIssues); + + if (pageItems.length < 100) { + break; + } + + page += 1; + } + + return unlimited ? collected : collected.slice(0, limit); + } + + async replaceIssueLabels(issueNumber: number, labels: string[]) { + await this.request( + "PUT", + `/repos/${this.owner}/${this.repo}/issues/${issueNumber}/labels`, + { labels }, + ); + } + + async upsertLabel(definition: GitHubLabelDefinition) { + const encodedName = encodeURIComponent(definition.name); + + try { + await this.request( + "PATCH", + `/repos/${this.owner}/${this.repo}/labels/${encodedName}`, + { + new_name: definition.name, + color: definition.color, + description: definition.description, + }, + ); + } catch (error) { + if (!(error instanceof GitHubApiError) || error.status !== 404) { + throw error; + } + + await this.request("POST", `/repos/${this.owner}/${this.repo}/labels`, { + name: definition.name, + color: definition.color, + description: definition.description, + }); + } + } + + async createIssueComment(issueNumber: number, body: string) { + return this.request( + "POST", + `/repos/${this.owner}/${this.repo}/issues/${issueNumber}/comments`, + { body }, + ); + } + + async updateIssueComment(commentId: number, body: string) { + return this.request( + "PATCH", + `/repos/${this.owner}/${this.repo}/issues/comments/${commentId}`, + { body }, + ); + } + + private async paginate(path: string): Promise { + const collected: T[] = []; + let nextPath: string | null = path; + + while (nextPath) { + const response = await fetch(buildApiUrl(nextPath), { + headers: this.headers(), + }); + + if (!response.ok) { + throw await GitHubApiError.fromResponse(response); + } + + const pageItems = (await response.json()) as T[]; + collected.push(...pageItems); + nextPath = parseNextLink(response.headers.get("link")); + } + + return collected; + } + + private async request( + method: string, + path: string, + body?: unknown, + ): Promise { + const response = await fetch(buildApiUrl(path), { + method, + headers: this.headers(), + body: body ? JSON.stringify(body) : undefined, + }); + + if (!response.ok) { + throw await GitHubApiError.fromResponse(response); + } + + if (response.status === 204) { + return undefined as T; + } + + return (await response.json()) as T; + } + + private headers() { + return { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${this.token}`, + "Content-Type": "application/json", + "User-Agent": "skillhub-issue-triage", + "X-GitHub-Api-Version": "2022-11-28", + }; + } +} + +export class GitHubApiError extends Error { + constructor( + readonly status: number, + readonly responseBody: string, + ) { + super(`GitHub API request failed with status ${status}: ${responseBody}`); + } + + static async fromResponse(response: Response) { + return new GitHubApiError(response.status, await response.text()); + } +} + +function parseNextLink(linkHeader: string | null) { + if (!linkHeader) { + return null; + } + + const nextEntry = linkHeader + .split(",") + .map((item) => item.trim()) + .find((item) => item.endsWith('rel="next"')); + + if (!nextEntry) { + return null; + } + + const urlMatch = nextEntry.match(/<([^>]+)>/); + + if (!urlMatch) { + return null; + } + + const url = new URL(urlMatch[1]); + return `${url.pathname}${url.search}`; +} diff --git a/.github/scripts/issue-backlog-rescore.ts b/.github/scripts/issue-backlog-rescore.ts new file mode 100644 index 00000000..dd0cf9c0 --- /dev/null +++ b/.github/scripts/issue-backlog-rescore.ts @@ -0,0 +1,128 @@ +import { GitHubClient } from "./github.ts"; +import { readIssueLlmConfig, shouldUseLlm } from "./issue-llm-config.ts"; +import { evaluateIssueWithLlm } from "./issue-llm-evaluator.ts"; +import { TRIAGE_MANUAL_OVERRIDE_LABEL } from "./issue-triage-config.ts"; +import { + analyzeIssue, + buildManagedLabels, + ensureManagedLabels, + findTriageComment, + parseTriageMachineState, + previewTriageMutation, + syncManagedLabels, + upsertTriageComment, +} from "./issue-triage-lib.ts"; +import { mergeRuleAndLlm } from "./issue-triage-merge.ts"; + +function readFlag(name: string) { + const index = Deno.args.indexOf(`--${name}`); + return index >= 0 ? Deno.args[index + 1] : undefined; +} + +function hasFlag(name: string) { + return Deno.args.includes(`--${name}`); +} + +const owner = readFlag("owner"); +const repo = readFlag("repo"); +const limitValue = readFlag("limit") ?? "0"; +const dryRun = hasFlag("dry-run"); +const token = Deno.env.get("GH_TOKEN") ?? Deno.env.get("GITHUB_TOKEN"); + +if (!owner || !repo || !token) { + throw new Error( + "Usage: deno run issue-backlog-rescore.ts --owner --repo [--limit 0 for all] with GH_TOKEN set.", + ); +} + +const limit = Number.parseInt(limitValue, 10); + +if (Number.isNaN(limit) || limit < 0) { + throw new Error(`Invalid limit: ${limitValue}`); +} + +const client = new GitHubClient(token, owner, repo); +if (!dryRun) { + await ensureManagedLabels(client); +} +const llmConfig = readIssueLlmConfig(); + +const issues = await client.listOpenIssuesByLabel("triage/deferred", limit); +const dryRunResults: Array> = []; + +for (const issue of issues) { + if ( + issue.labels.some((label) => label.name === TRIAGE_MANUAL_OVERRIDE_LABEL) + ) { + console.log( + `Skipping #${issue.number} because ${TRIAGE_MANUAL_OVERRIDE_LABEL} is set.`, + ); + continue; + } + + const comments = await client.listIssueComments(issue.number); + const ruleResult = analyzeIssue(issue, comments); + const existingComment = findTriageComment(comments); + const previousState = existingComment + ? parseTriageMachineState(existingComment.body) + : null; + let result = ruleResult; + + if (llmConfig) { + const llmDecision = shouldUseLlm(issue, ruleResult); + + if (llmDecision.use) { + const { inputHash, assessment } = await evaluateIssueWithLlm( + llmConfig, + issue, + comments, + ruleResult, + previousState, + ); + + result = mergeRuleAndLlm({ + ...ruleResult, + inputHash, + llm: assessment, + mode: assessment.mode === "assist" ? "llm-assist" : "llm-shadow", + }); + } + } + + if (dryRun) { + const preview = previewTriageMutation(result, comments); + dryRunResults.push({ + issue: issue.number, + mode: result.mode, + route: result.route, + priority: result.priority, + effort: result.effort, + confidence: result.confidence, + riskLevel: result.riskLevel, + labels: preview.labels, + commentAction: preview.existingComment ? "update" : "create", + commentBody: preview.commentBody, + }); + continue; + } + + await syncManagedLabels(client, issue, result); + await upsertTriageComment(client, issue.number, result, comments); + + console.log( + JSON.stringify( + { + issue: issue.number, + route: result.route, + priority: result.priority, + labels: buildManagedLabels(issue, result), + }, + null, + 2, + ), + ); +} + +if (dryRun) { + console.log(JSON.stringify({ dryRun: true, issues: dryRunResults }, null, 2)); +} diff --git a/.github/scripts/issue-handoff-brief.ts b/.github/scripts/issue-handoff-brief.ts new file mode 100644 index 00000000..e19dacaa --- /dev/null +++ b/.github/scripts/issue-handoff-brief.ts @@ -0,0 +1,257 @@ +import { MaintainerHandoffBrief, TriageResult } from "./issue-triage-types.ts"; + +const AREA_RULES: Array<{ keywords: string[]; area: string }> = [ + { + keywords: ["clawhub publish", "publish skill", "publish", "namespace"], + area: + "CLI 发布命令参数解析与 namespace 感知发布流程 / CLI publish command option parsing and namespace-aware publish flow", + }, + { + keywords: ["clawhub install", "install skill", "install"], + area: + "技能安装流程与 registry/lockfile 集成 / Skill installation flow and registry/lockfile integration", + }, + { + keywords: ["clawhub update", "update skill", "update"], + area: + "已安装技能更新流程与版本解析 / Installed skill update flow and version resolution", + }, + { + keywords: ["clawhub sync", "sync skill", "sync"], + area: + "本地技能同步流程与发布 diff 检测 / Local skill sync flow and publish diff detection", + }, + { + keywords: ["inspect", "search", "explore"], + area: + "Registry 发现与 CLI 查询流程 / Registry discovery and CLI query workflow", + }, + { + keywords: ["auth", "login", "ldap", "sso", "token"], + area: + "认证、会话与身份集成 / Authentication, session, and identity integration", + }, + { + keywords: ["openapi", "sdk", "api contract", "contract"], + area: + "公开 API 契约、生成 SDK 与兼容性表面 / Public API contract, generated SDKs, and compatibility surface", + }, + { + keywords: ["docs", "documentation", "manual", "help", "--help"], + area: + "文档、操作指引与 CLI help 输出 / Documentation, operator guidance, and CLI help output", + }, + { + keywords: ["scanner", "security", "audit"], + area: + "安全扫描流程与审计/报告行为 / Security scanner pipeline and audit/reporting behavior", + }, +]; + +export function buildMaintainerHandoffBrief( + result: TriageResult, +): MaintainerHandoffBrief | undefined { + if (result.route !== "core") { + return undefined; + } + + const summary = buildSummary(result); + const whyCore = unique([ + result.requiresCoreMaintainer + ? "阻塞 OpenClaw/ClawHub 核心工作流，因此即便改动范围看起来可控，也需要 maintainer judgment / Blocks an OpenClaw/ClawHub core workflow, so maintainer judgment is required even if the code change looks bounded." + : "", + result.riskLevel === "high" + ? "触及高风险区域，未经 maintainer 审查不应直接信任自动修复 / Touches a higher-risk area where automated fixes should not be trusted without maintainer review." + : "", + result.effort >= 4 + ? "大概率跨多个模块或公共兼容面 / Likely spans multiple modules or a public compatibility surface." + : "", + result.confidence <= 3 + ? "问题本身重要，但仍需要 maintainer 先收敛范围再实施 / The issue is important, but a maintainer still needs to tighten scope before implementation." + : "", + ...result.highRiskReasons, + ]).slice(0, 4); + + const reproduction = buildReproduction(result); + const suspectedAreas = inferSuspectedAreas(result); + const risks = buildRisks(result, suspectedAreas); + const validation = buildValidation(result, suspectedAreas); + + return { + summary, + whyCore, + reproduction, + suspectedAreas, + risks, + validation, + }; +} + +function buildSummary(result: TriageResult) { + const llmSummary = result.llm?.summaryZh ?? result.llm?.summary ?? + result.llm?.summaryEn; + + if (llmSummary && llmSummary.trim().length > 0) { + return llmSummary.trim(); + } + + const preferred = [ + result.sections["summary"], + result.sections["problem"], + result.sections["expected behavior"], + ].find((value) => value && value.trim().length > 0); + + if (preferred) { + return compact(preferred); + } + + return result.issue.title.replace(/^\[[^\]]+\]\s*/, "").trim(); +} + +function buildReproduction(result: TriageResult) { + const commandFocusedSteps = extractCommandAndErrorLines( + result.sections["steps to reproduce"], + ); + + if (commandFocusedSteps.length > 0) { + return commandFocusedSteps.slice(0, 4); + } + + const steps = splitIntoBullets(result.sections["steps to reproduce"]); + + if (steps.length > 0) { + return steps.slice(0, 5); + } + + const problem = splitIntoBullets(result.sections["problem"]); + + if (problem.length > 0) { + return problem.slice(0, 4); + } + + return [ + "按 issue 中描述的操作路径复现，并确认当前失败模式 / Recreate the operator flow described in the issue and confirm the current failure mode.", + ]; +} + +function inferSuspectedAreas(result: TriageResult) { + const text = [ + result.issue.title, + result.sections["summary"] ?? "", + result.sections["problem"] ?? "", + result.sections["steps to reproduce"] ?? "", + result.sections["impact"] ?? "", + result.sections["api contract impact"] ?? "", + result.sections["contract or sdk impact"] ?? "", + ] + .join("\n") + .toLowerCase(); + + const areas = AREA_RULES.filter((rule) => + rule.keywords.some((keyword) => text.includes(keyword)) + ).map((rule) => rule.area); + + if (areas.length > 0) { + return unique(areas).slice(0, 5); + } + + return [ + "最接近该失败路径的 owner-facing 工作流模块 / The closest owner-facing workflow module for the issue's reported failure path", + "当前对外承诺该行为的文档或 help 文本 / Any docs or help text that currently promise the affected behavior", + ]; +} + +function buildRisks(result: TriageResult, suspectedAreas: string[]) { + const risks = unique([ + ...result.highRiskReasons, + result.llm?.riskFlags.includes("cli-protocol") + ? "CLI 行为、文档和操作预期可能发生漂移，需要同步更新命令 help 与兼容性说明 / CLI behavior, docs, and operator expectations may drift unless command help and compatibility notes are updated together." + : "", + suspectedAreas.some((area) => area.toLowerCase().includes("namespace")) + ? "namespace 范围行为如果没有保留 fallback routing，可能回归默认 publish/install 流程 / Namespace-scoped behavior can regress default publish/install flows if fallback routing is not preserved." + : "", + result.requiresCoreMaintainer + ? "该问题影响已定义主流程，回归会很快被终端用户感知 / This issue affects a documented primary workflow, so regressions would be visible to end users quickly." + : "", + ]); + + return risks.length > 0 ? risks.slice(0, 4) : [ + "合并前检查相邻用户路径是否出现回归 / Check for regressions in adjacent user-facing workflow paths before merging.", + ]; +} + +function buildValidation(result: TriageResult, suspectedAreas: string[]) { + const validation = unique([ + result.sections["steps to reproduce"] + ? "按 issue 中的复现步骤逐条回放，确认报告的问题已消失 / Replay the exact reproduction steps from the issue and confirm the reported failure disappears." + : "修复后端到端验证主报告流程 / Validate the primary reported workflow end-to-end after the fix.", + result.sections["expected behavior"] + ? `确认最终行为符合 issue 期望 / Confirm the final behavior matches the issue's expected outcome: ${ + compact(result.sections["expected behavior"]) + }` + : "", + suspectedAreas.some((area) => area.toLowerCase().includes("documentation")) + ? "更新或核对文档与 CLI help 输出，确保其与实现行为一致 / Update or verify documentation and CLI help output so they match the implemented behavior." + : "", + suspectedAreas.some((area) => area.toLowerCase().includes("api contract")) + ? "发布前检查下游 API/SDK/CLI 的兼容性预期 / Check for downstream API/SDK/CLI compatibility expectations before shipping." + : "", + suspectedAreas.some((area) => area.toLowerCase().includes("namespace")) + ? "同时验证 namespace 范围行为与默认非 namespace 流程 / Verify both namespace-scoped behavior and the default non-namespace flow." + : "", + result.requiresCoreMaintainer + ? "围绕受影响的 OpenClaw/ClawHub 用户路径执行最小必要回归测试 / Run the smallest relevant regression test around the affected OpenClaw/ClawHub user journey." + : "", + ]); + + return validation.slice(0, 5); +} + +function splitIntoBullets(value: string | undefined) { + if (!value) { + return []; + } + + return value + .split("\n") + .map((line) => line.trim()) + .filter((line) => + line.length > 0 && + line !== "```" && + !line.startsWith("PS ") && + !line.startsWith("Usage:") && + !line.startsWith("Options:") && + !line.startsWith("Arguments:") + ) + .map((line) => line.replace(/^[*-]\s*/, "")) + .slice(0, 6); +} + +function extractCommandAndErrorLines(value: string | undefined) { + if (!value) { + return []; + } + + return value + .split("\n") + .map((line) => line.trim()) + .filter((line) => + line.length > 0 && + ( + line.toLowerCase().includes("clawhub ") || + line.toLowerCase().startsWith("error:") || + line.toLowerCase().includes("unknown option") || + line.toLowerCase().includes("usage:") + ) + ) + .map((line) => line.replace(/^[>*-]\s*/, "")) + .slice(0, 4); +} + +function compact(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +function unique(values: string[]) { + return [...new Set(values.filter((value) => value.trim().length > 0))]; +} diff --git a/.github/scripts/issue-llm-config.ts b/.github/scripts/issue-llm-config.ts new file mode 100644 index 00000000..f204d89b --- /dev/null +++ b/.github/scripts/issue-llm-config.ts @@ -0,0 +1,139 @@ +import { GitHubIssue } from "./github.ts"; +import { IssueLlmConfig } from "./issue-llm-types.ts"; +import { TriageResult } from "./issue-triage-types.ts"; + +const DEFAULT_TIMEOUT_MS = 30000; +const DEFAULT_MAX_ATTEMPTS = 2; +const DEFAULT_RETRY_BACKOFF_MS = 1500; +const DEFAULT_TEMPERATURE = 0.1; +const DEFAULT_MAX_COMMENTS = 4; +const DEFAULT_MAX_COMMENT_CHARS = 900; +const DEFAULT_MAX_BODY_CHARS = 6000; + +export function readIssueLlmConfig(): IssueLlmConfig | null { + const mode = normalizeMode(Deno.env.get("ISSUE_TRIAGE_LLM_MODE")); + + if (mode === "off") { + return null; + } + + const baseUrl = normalizeUrl(Deno.env.get("ISSUE_TRIAGE_LLM_BASE_URL")); + const apiKey = Deno.env.get("ISSUE_TRIAGE_LLM_API_KEY")?.trim() ?? ""; + const model = Deno.env.get("ISSUE_TRIAGE_LLM_MODEL")?.trim() ?? ""; + + if (!baseUrl || !apiKey || !model) { + console.warn( + "LLM triage is configured in a non-off mode but base URL, model, or API key is missing. Falling back to rules-only.", + ); + return null; + } + + return { + mode, + provider: "openai-compatible", + baseUrl, + apiKey, + model, + timeoutMs: parseInteger( + Deno.env.get("ISSUE_TRIAGE_LLM_TIMEOUT_MS"), + DEFAULT_TIMEOUT_MS, + ), + maxAttempts: Math.max( + 1, + parseInteger( + Deno.env.get("ISSUE_TRIAGE_LLM_MAX_ATTEMPTS"), + DEFAULT_MAX_ATTEMPTS, + ), + ), + retryBackoffMs: Math.max( + 0, + parseInteger( + Deno.env.get("ISSUE_TRIAGE_LLM_RETRY_BACKOFF_MS"), + DEFAULT_RETRY_BACKOFF_MS, + ), + ), + temperature: parseFloatSetting( + Deno.env.get("ISSUE_TRIAGE_LLM_TEMPERATURE"), + DEFAULT_TEMPERATURE, + ), + maxComments: parseInteger( + Deno.env.get("ISSUE_TRIAGE_LLM_MAX_COMMENTS"), + DEFAULT_MAX_COMMENTS, + ), + maxCommentChars: parseInteger( + Deno.env.get("ISSUE_TRIAGE_LLM_MAX_COMMENT_CHARS"), + DEFAULT_MAX_COMMENT_CHARS, + ), + maxBodyChars: parseInteger( + Deno.env.get("ISSUE_TRIAGE_LLM_MAX_BODY_CHARS"), + DEFAULT_MAX_BODY_CHARS, + ), + }; +} + +export function shouldUseLlm(issue: GitHubIssue, result: TriageResult) { + const reasons: string[] = []; + + if (result.route === "needs-info") { + reasons.push("route-needs-info"); + } + + if (result.route === "core") { + reasons.push("route-core"); + } + + if (result.priority >= 3 && result.priority <= 4.2) { + reasons.push("priority-near-threshold"); + } + + if (result.confidence <= 3) { + reasons.push("confidence-low"); + } + + if (issue.comments >= 4) { + reasons.push("discussion-heavy"); + } + + if ((issue.body ?? "").length >= 1200) { + reasons.push("body-long"); + } + + if (result.issueKind === "feature" || result.issueKind === "reward") { + reasons.push("non-bug-judgment"); + } + + return { + use: reasons.length > 0, + reasons, + }; +} + +function normalizeMode(raw: string | undefined | null) { + const value = raw?.trim().toLowerCase(); + + if (value === "shadow" || value === "assist") { + return value; + } + + return "off"; +} + +function normalizeUrl(value: string | undefined | null) { + const trimmed = value?.trim(); + + if (!trimmed) { + return ""; + } + + return trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed; +} + +function parseInteger(raw: string | undefined, fallback: number) { + const parsed = Number.parseInt(raw ?? "", 10); + return Number.isNaN(parsed) ? fallback : parsed; +} + +function parseFloatSetting(raw: string | undefined, fallback: number) { + const parsed = Number.parseFloat(raw ?? ""); + return Number.isNaN(parsed) ? fallback : parsed; +} diff --git a/.github/scripts/issue-llm-evaluator.ts b/.github/scripts/issue-llm-evaluator.ts new file mode 100644 index 00000000..c9ac93ef --- /dev/null +++ b/.github/scripts/issue-llm-evaluator.ts @@ -0,0 +1,466 @@ +import { GitHubIssue, GitHubIssueComment } from "./github.ts"; +import { + IssueLlmConfig, + IssueLlmPayload, + IssueLlmResponse, +} from "./issue-llm-types.ts"; +import { requestOpenAiCompatibleJson } from "./issue-llm-provider.ts"; +import { + IssueRoute, + LlmAssessment, + TriageMachineState, + TriageResult, +} from "./issue-triage-types.ts"; + +const ALLOWED_RISK_FLAGS = new Set([ + "auth", + "security", + "token", + "permission", + "migration", + "schema", + "api-contract", + "sdk", + "cli-protocol", + "data-loss", +]); +const PROMPT_VERSION = 3; + +export async function evaluateIssueWithLlm( + config: IssueLlmConfig, + issue: GitHubIssue, + comments: GitHubIssueComment[], + ruleResult: TriageResult, + previousState: TriageMachineState | null, +) { + const payload = buildPayload(config, issue, comments, ruleResult); + const inputHash = await buildIssueInputHash(payload); + const cached = previousState?.llm; + + if ( + cached && + cached.inputHash === inputHash && + cached.provider === config.provider && + cached.model === config.model && + cached.mode === config.mode && + !cached.failed + ) { + return { + inputHash, + assessment: { + ...cached, + reused: true, + } as LlmAssessment, + }; + } + + try { + const rawJson = await requestOpenAiCompatibleJson( + config, + buildSystemPrompt(), + JSON.stringify(payload, null, 2), + ); + const parsed = validateLlmResponse(JSON.parse(rawJson), ruleResult); + + return { + inputHash, + assessment: { + provider: config.provider, + model: config.model, + mode: config.mode, + inputHash, + summary: parsed.summary_zh || parsed.summary || parsed.summary_en || "", + summaryEn: parsed.summary_en || parsed.summary || parsed.summary_zh || + "", + summaryZh: parsed.summary_zh || parsed.summary || parsed.summary_en || + "", + impact: parsed.impact, + urgency: parsed.urgency, + effort: parsed.effort, + confidence: parsed.confidence, + riskFlags: parsed.risk_flags, + missingInfo: parsed.missing_info, + suggestedQuestions: parsed.suggested_questions, + recommendedRoute: parsed.recommended_route, + rationale: parsed.rationale, + reused: false, + failed: false, + } satisfies LlmAssessment, + }; + } catch (error) { + const failureReason = error instanceof Error + ? error.message + : String(error); + + return { + inputHash, + assessment: { + provider: config.provider, + model: config.model, + mode: config.mode, + inputHash, + summary: "", + summaryEn: "", + summaryZh: "", + impact: ruleResult.impact, + urgency: ruleResult.urgency, + effort: ruleResult.effort, + confidence: ruleResult.confidence, + riskFlags: [], + missingInfo: [], + suggestedQuestions: [], + recommendedRoute: ruleResult.route, + rationale: [], + reused: false, + failed: true, + failureReason, + } satisfies LlmAssessment, + }; + } +} + +function buildPayload( + config: IssueLlmConfig, + issue: GitHubIssue, + comments: GitHubIssueComment[], + ruleResult: TriageResult, +): IssueLlmPayload { + const latestComments = comments + .filter((comment) => + !comment.body.includes("`; +} + +function calculateConfidence( + issueKind: IssueKind, + rawBody: string, + sections: Record, + missingFields: string[], +) { + const required = requiredFields(issueKind); + const requiredFilled = + required.filter((field) => hasMeaningfulSection(sections[field])).length; + const supportFields = Object.entries(sections).filter( + ([key, value]) => !required.includes(key) && hasMeaningfulSection(value), + ).length; + + let score = 1; + score += requiredFilled; + score += supportFields >= 1 ? 0.5 : 0; + score += supportFields >= 3 ? 0.5 : 0; + score += rawBody.length >= 400 ? 0.5 : 0; + score -= missingFields.length > 0 ? 1 : 0; + + return clamp(Math.round(score), 1, 5); +} + +function calculateAgePolicy(createdAt: string, now: Date) { + const created = new Date(createdAt); + const openDays = Math.floor( + (now.getTime() - created.getTime()) / (24 * 60 * 60 * 1000), + ); + const safeOpenDays = Math.max(0, openDays); + + if (safeOpenDays >= 14) { + return { + openDays: safeOpenDays, + ageBoost: 1.5, + priorityFloor: 4.4, + reason: + `已打开 ${safeOpenDays} 天，超过 14 天闭环 SLA，优先级强制提升到 P0 / Open for ${safeOpenDays} days; the 14-day closure SLA is breached, so priority is forced to P0.`, + }; + } + + if (safeOpenDays >= 10) { + return { + openDays: safeOpenDays, + ageBoost: 1, + priorityFloor: 3.6, + reason: + `已打开 ${safeOpenDays} 天，为避免超过 14 天仍未闭环，强制进入 active lane / Open for ${safeOpenDays} days; forced into an active lane before the 14-day closure SLA is missed.`, + }; + } + + if (safeOpenDays >= 7) { + return { + openDays: safeOpenDays, + ageBoost: 0.6, + priorityFloor: 2.6, + reason: + `已打开 ${safeOpenDays} 天，开始进入 2 周闭环预热窗口 / Open for ${safeOpenDays} days; entering the 2-week closure warm-up window.`, + }; + } + + return { + openDays: safeOpenDays, + ageBoost: 0, + priorityFloor: 0, + reason: "", + }; +} + +function calculateEngagementBoost( + commentCount: number, + rewardAmountText?: string, +) { + let boost = Math.min(0.8, commentCount * 0.1); + const rewardAmount = Number.parseFloat( + (rewardAmountText ?? "").replaceAll(/[^0-9.]/g, ""), + ); + + if (!Number.isNaN(rewardAmount)) { + if (rewardAmount >= 500) { + boost += 0.6; + } else if (rewardAmount >= 100) { + boost += 0.3; + } else if (rewardAmount > 0) { + boost += 0.1; + } + } + + return Math.min(1, boost); +} + +function requiredFields(issueKind: IssueKind) { + return REQUIRED_SECTIONS[issueKind] ?? []; +} + +function buildSearchText(issue: GitHubIssue, sections: Record) { + return [issue.title, issue.body ?? "", ...Object.values(sections)].join("\n") + .toLowerCase(); +} + +function buildRiskText(issue: GitHubIssue, sections: Record) { + const preferredSections = [ + "summary", + "problem", + "proposed solution", + "expected behavior", + "steps to reproduce", + "impact", + "api contract impact", + "contract or sdk impact", + ]; + + return [ + issue.title, + ...preferredSections.map((section) => sections[section] ?? ""), + ] + .join("\n") + .toLowerCase(); +} + +function buildWorkflowText( + issue: GitHubIssue, + sections: Record, +) { + const preferredSections = [ + "summary", + "problem", + "steps to reproduce", + "expected behavior", + "impact", + ]; + + return [ + issue.title, + ...preferredSections.map((section) => sections[section] ?? ""), + ] + .join("\n") + .toLowerCase(); +} + +function normalizeHeading(value: string) { + return value.trim().toLowerCase(); +} + +function cleanupSectionContent(value: string) { + return value + .replaceAll(/^_No response_\s*$/gim, "") + .replaceAll(/^no response\s*$/gim, "") + .trim(); +} + +function hasMeaningfulSection(value: string | undefined) { + return Boolean(value && cleanupSectionContent(value).length >= 3); +} + +function uniqueNonEmpty(values: string[]) { + return [...new Set(values.filter((value) => value.trim().length > 0))]; +} + +function clamp(value: number, min: number, max: number) { + return Math.max(min, Math.min(max, value)); +} + +function roundToOneDecimal(value: number) { + return Math.round(value * 10) / 10; +} + +export function findTriageComment(comments: GitHubIssueComment[]) { + return comments.find((comment) => + comment.body.includes(TRIAGE_COMMENT_MARKER) + ); +} + +export function buildManagedLabels(issue: GitHubIssue, result: TriageResult) { + const existingLabels = issue.labels + .map((label) => label.name) + .filter((label): label is string => Boolean(label)); + + const unmanagedLabels = existingLabels.filter( + (label) => + !MANAGED_LABEL_PREFIXES.some((prefix) => label.startsWith(prefix)), + ); + + return [ + ...unmanagedLabels, + routeLabel(result.route), + priorityLabel(result.priority), + effortLabel(result.effort), + ...riskLabels(result.riskLevel), + ]; +} + +export function previewTriageMutation( + result: TriageResult, + comments: GitHubIssueComment[], +) { + return { + labels: uniqueNonEmpty(buildManagedLabels(result.issue, result)), + commentBody: renderTriageComment(result), + existingComment: findTriageComment(comments) ?? null, + }; +} + +export function parseTriageMachineState( + commentBody: string, +): TriageMachineState | null { + const start = commentBody.indexOf(TRIAGE_COMMENT_MARKER); + + if (start < 0) { + return null; + } + + const jsonStart = start + TRIAGE_COMMENT_MARKER.length; + const end = commentBody.indexOf("-->", jsonStart); + + if (end < 0) { + return null; + } + + const rawJson = commentBody.slice(jsonStart, end).trim(); + + try { + const parsed = JSON.parse(rawJson) as TriageMachineState; + + if ( + typeof parsed !== "object" || + parsed === null || + typeof parsed.issue !== "number" || + typeof parsed.route !== "string" + ) { + return null; + } + + return parsed; + } catch { + return null; + } +} diff --git a/.github/scripts/issue-triage-merge.ts b/.github/scripts/issue-triage-merge.ts new file mode 100644 index 00000000..63c162dc --- /dev/null +++ b/.github/scripts/issue-triage-merge.ts @@ -0,0 +1,166 @@ +import { TriageResult, TriageSnapshot } from "./issue-triage-types.ts"; +import { buildMaintainerHandoffBrief } from "./issue-handoff-brief.ts"; + +export function mergeRuleAndLlm(ruleResult: TriageResult): TriageResult { + const llm = ruleResult.llm; + + if (!llm || llm.failed || llm.mode !== "assist") { + return { + ...ruleResult, + handoffBrief: ruleResult.route === "core" + ? buildMaintainerHandoffBrief(ruleResult) + : undefined, + mode: llm && !llm.failed && llm.mode === "shadow" + ? "llm-shadow" + : "rules-only", + inputHash: llm?.inputHash ?? ruleResult.inputHash, + }; + } + + const impact = nudgeScore(ruleResult.impact, llm.impact); + const urgency = nudgeScore(ruleResult.urgency, llm.urgency); + const effort = nudgeScore(ruleResult.effort, llm.effort); + const confidence = nudgeScore(ruleResult.confidence, llm.confidence); + const missingFields = unique([ + ...ruleResult.missingFields, + ...llm.missingInfo, + ]); + const highRiskReasons = unique([ + ...ruleResult.highRiskReasons, + ...llm.riskFlags.map((flag) => + `LLM 标记了高风险区域：${flag} / LLM flagged high-risk area: ${flag}.` + ), + ]); + const requiresCoreMaintainer = ruleResult.requiresCoreMaintainer; + const riskLevel = highRiskReasons.length > 0 ? "high" : "low"; + const priority = clamp( + roundToOneDecimal( + impact * 0.45 + + urgency * 0.35 + + ruleResult.ageBoost + + ruleResult.engagementBoost, + ), + 1, + 5, + ); + const route = determineRoute( + priority, + effort, + confidence, + riskLevel, + missingFields, + requiresCoreMaintainer, + ); + const nextAction = describeNextAction(route, missingFields); + const reasons = unique([ + ...ruleResult.reasons, + ...llm.rationale, + llm.summary + ? `LLM 摘要：${llm.summaryZh || llm.summary} / LLM summary: ${ + llm.summaryEn || llm.summary + }` + : "", + ]).slice(0, 6); + + const mergedSnapshot: TriageSnapshot = { + route, + riskLevel, + requiresCoreMaintainer, + openDays: ruleResult.openDays, + impact, + urgency, + effort, + confidence, + priority, + ageBoost: ruleResult.ageBoost, + priorityFloor: ruleResult.priorityFloor, + engagementBoost: ruleResult.engagementBoost, + missingFields, + reasons, + highRiskReasons, + nextAction, + }; + + return { + ...ruleResult, + ...mergedSnapshot, + mode: "llm-assist", + inputHash: llm.inputHash, + handoffBrief: route === "core" + ? buildMaintainerHandoffBrief({ + ...ruleResult, + ...mergedSnapshot, + mode: "llm-assist", + inputHash: llm.inputHash, + }) + : undefined, + }; +} + +export function determineRoute( + priority: number, + effort: number, + confidence: number, + riskLevel: "low" | "high", + missingFields: string[], + requiresCoreMaintainer = false, +) { + if (requiresCoreMaintainer) { + return "core"; + } + + if (missingFields.length > 0 || confidence <= 2) { + return "needs-info"; + } + + if (priority < 3.6) { + return "deferred"; + } + + if (riskLevel === "high" || effort >= 4 || confidence <= 3) { + return "core"; + } + + return "agent-ready"; +} + +export function describeNextAction( + route: TriageResult["route"], + missingFields: string[], +) { + if (route === "needs-info") { + return `等待补充更多信息；作者更新 issue 或评论 \`/retriage\` 后重新分流 / Wait for more detail, then rerun triage after the author edits the issue or comments \`/retriage\`. Missing: ${ + missingFields.join(", ") + }.`; + } + + if (route === "deferred") { + return "将 issue 保留在 deferred 队列，并由 6 小时一次的 rescore 持续抬升；最晚在第 10 天强制进入 active lane。若第 14 天仍未闭环，应按 SLA 视为 P0 升级目标，并在下一次 triage 中重点处理 / Keep the issue in the deferred queue and let the 6-hour rescore keep lifting it; it is forced into an active lane by day 10. If it is still open on day 14, treat it as a P0 escalation target under the SLA and prioritize it in the next triage pass."; + } + + if (route === "core") { + return "交给 core maintainer，并结合本地编程Agent协助完成复现、收敛范围与验证闭环 / Hand the issue to a core maintainer and use a local programming agent for reproduction, scoping, and validation."; + } + + return "在 self-hosted issue-agent runner 启用后，将其标记为低风险 agent 可执行候选 / Mark as a candidate for low-risk agent execution once the self-hosted issue-agent runner is enabled."; +} + +function nudgeScore(ruleScore: number, llmScore: number) { + if (llmScore === ruleScore) { + return ruleScore; + } + + return clamp(ruleScore + Math.sign(llmScore - ruleScore), 1, 5); +} + +function unique(values: string[]) { + return [...new Set(values.filter((value) => value.trim().length > 0))]; +} + +function clamp(value: number, min: number, max: number) { + return Math.max(min, Math.min(max, value)); +} + +function roundToOneDecimal(value: number) { + return Math.round(value * 10) / 10; +} diff --git a/.github/scripts/issue-triage-types.ts b/.github/scripts/issue-triage-types.ts new file mode 100644 index 00000000..e2830a15 --- /dev/null +++ b/.github/scripts/issue-triage-types.ts @@ -0,0 +1,93 @@ +import { GitHubIssue } from "./github.ts"; + +export type IssueKind = "bug" | "feature" | "reward" | "other"; +export type IssueRoute = "needs-info" | "deferred" | "core" | "agent-ready"; +export type RiskLevel = "low" | "high"; +export type LlmMode = "off" | "shadow" | "assist"; +export type AnalysisMode = "rules-only" | "llm-shadow" | "llm-assist"; + +export interface ParsedIssueBody { + sections: Record; + missingFields: string[]; +} + +export interface TriageSnapshot { + route: IssueRoute; + riskLevel: RiskLevel; + requiresCoreMaintainer: boolean; + openDays: number; + impact: number; + urgency: number; + effort: number; + confidence: number; + priority: number; + ageBoost: number; + priorityFloor: number; + engagementBoost: number; + missingFields: string[]; + reasons: string[]; + highRiskReasons: string[]; + nextAction: string; +} + +export interface MaintainerHandoffBrief { + summary: string; + whyCore: string[]; + reproduction: string[]; + suspectedAreas: string[]; + risks: string[]; + validation: string[]; +} + +export interface LlmAssessment { + provider: string; + model: string; + mode: LlmMode; + inputHash: string; + summary: string; + summaryEn?: string; + summaryZh?: string; + impact: number; + urgency: number; + effort: number; + confidence: number; + riskFlags: string[]; + missingInfo: string[]; + suggestedQuestions: string[]; + recommendedRoute: IssueRoute; + rationale: string[]; + reused: boolean; + failed: boolean; + failureReason?: string; +} + +export interface TriageResult extends TriageSnapshot { + issue: GitHubIssue; + issueKind: IssueKind; + sections: Record; + mode: AnalysisMode; + inputHash: string; + rule: TriageSnapshot; + llm?: LlmAssessment; + handoffBrief?: MaintainerHandoffBrief; +} + +export interface TriageMachineState { + version: number; + issue: number; + inputHash?: string; + mode?: AnalysisMode; + route: IssueRoute; + priority: number; + requiresCoreMaintainer?: boolean; + impact: number; + urgency: number; + effort: number; + confidence: number; + riskLevel: RiskLevel; + ageBoost: number; + engagementBoost: number; + missingFields: string[]; + updatedAt: string; + llm?: LlmAssessment; +} diff --git a/.github/scripts/issue-triage.ts b/.github/scripts/issue-triage.ts new file mode 100644 index 00000000..c9269ac4 --- /dev/null +++ b/.github/scripts/issue-triage.ts @@ -0,0 +1,129 @@ +import { GitHubClient } from "./github.ts"; +import { readIssueLlmConfig, shouldUseLlm } from "./issue-llm-config.ts"; +import { evaluateIssueWithLlm } from "./issue-llm-evaluator.ts"; +import { TRIAGE_MANUAL_OVERRIDE_LABEL } from "./issue-triage-config.ts"; +import { + analyzeIssue, + buildManagedLabels, + ensureManagedLabels, + findTriageComment, + parseTriageMachineState, + previewTriageMutation, + syncManagedLabels, + upsertTriageComment, +} from "./issue-triage-lib.ts"; +import { mergeRuleAndLlm } from "./issue-triage-merge.ts"; + +function readFlag(name: string) { + const index = Deno.args.indexOf(`--${name}`); + return index >= 0 ? Deno.args[index + 1] : undefined; +} + +function hasFlag(name: string) { + return Deno.args.includes(`--${name}`); +} + +const owner = readFlag("owner"); +const repo = readFlag("repo"); +const issueNumberValue = readFlag("issue-number"); +const dryRun = hasFlag("dry-run"); +const token = Deno.env.get("GH_TOKEN") ?? Deno.env.get("GITHUB_TOKEN"); + +if (!owner || !repo || !issueNumberValue || !token) { + throw new Error( + "Usage: deno run issue-triage.ts --owner --repo --issue-number with GH_TOKEN set.", + ); +} + +const issueNumber = Number.parseInt(issueNumberValue, 10); + +if (Number.isNaN(issueNumber)) { + throw new Error(`Invalid issue number: ${issueNumberValue}`); +} + +const client = new GitHubClient(token, owner, repo); +const issue = await client.getIssue(issueNumber); + +if (issue.pull_request) { + console.log(`Skipping #${issue.number} because it is a pull request conversation.`); + Deno.exit(0); +} + +if (issue.labels.some((label) => label.name === TRIAGE_MANUAL_OVERRIDE_LABEL)) { + console.log(`Skipping #${issue.number} because ${TRIAGE_MANUAL_OVERRIDE_LABEL} is set.`); + Deno.exit(0); +} + +const comments = await client.listIssueComments(issueNumber); +const ruleResult = analyzeIssue(issue, comments); +const existingComment = findTriageComment(comments); +const previousState = existingComment + ? parseTriageMachineState(existingComment.body) + : null; +const llmConfig = readIssueLlmConfig(); +let result = ruleResult; + +if (llmConfig) { + const llmDecision = shouldUseLlm(issue, ruleResult); + + if (llmDecision.use) { + const { inputHash, assessment } = await evaluateIssueWithLlm( + llmConfig, + issue, + comments, + ruleResult, + previousState, + ); + + result = mergeRuleAndLlm({ + ...ruleResult, + inputHash, + llm: assessment, + mode: assessment.mode === "assist" ? "llm-assist" : "llm-shadow", + }); + } +} + +if (dryRun) { + const preview = previewTriageMutation(result, comments); + console.log( + JSON.stringify( + { + dryRun: true, + issue: issue.number, + mode: result.mode, + route: result.route, + priority: result.priority, + effort: result.effort, + confidence: result.confidence, + riskLevel: result.riskLevel, + labels: preview.labels, + commentAction: preview.existingComment ? "update" : "create", + commentBody: preview.commentBody, + }, + null, + 2, + ), + ); + Deno.exit(0); +} + +await ensureManagedLabels(client); +await syncManagedLabels(client, issue, result); +await upsertTriageComment(client, issueNumber, result, comments); + +console.log( + JSON.stringify( + { + issue: issue.number, + route: result.route, + priority: result.priority, + effort: result.effort, + confidence: result.confidence, + riskLevel: result.riskLevel, + labels: buildManagedLabels(issue, result), + }, + null, + 2, + ), +); diff --git a/.github/workflows/issue-backlog-rescore.yml b/.github/workflows/issue-backlog-rescore.yml new file mode 100644 index 00000000..99f11939 --- /dev/null +++ b/.github/workflows/issue-backlog-rescore.yml @@ -0,0 +1,51 @@ +name: Issue Backlog Rescore + +on: + schedule: + - cron: "0 */6 * * *" + workflow_dispatch: + inputs: + limit: + description: Maximum number of deferred issues to rescore + required: false + default: "0" + +concurrency: + group: issue-backlog-rescore + cancel-in-progress: false + +permissions: + contents: read + issues: write + +jobs: + rescore: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: denoland/setup-deno@667a34cdef165d8d2b2e98dde39547c9daac7282 # v2.0.4 + with: + deno-version: v2.x + + - name: Rescore deferred issues + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TRIAGE_LLM_MODE: ${{ vars.ISSUE_TRIAGE_LLM_MODE }} + ISSUE_TRIAGE_LLM_BASE_URL: ${{ vars.ISSUE_TRIAGE_LLM_BASE_URL }} + ISSUE_TRIAGE_LLM_MODEL: ${{ vars.ISSUE_TRIAGE_LLM_MODEL }} + ISSUE_TRIAGE_LLM_TIMEOUT_MS: ${{ vars.ISSUE_TRIAGE_LLM_TIMEOUT_MS }} + ISSUE_TRIAGE_LLM_MAX_ATTEMPTS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_ATTEMPTS }} + ISSUE_TRIAGE_LLM_RETRY_BACKOFF_MS: ${{ vars.ISSUE_TRIAGE_LLM_RETRY_BACKOFF_MS }} + ISSUE_TRIAGE_LLM_TEMPERATURE: ${{ vars.ISSUE_TRIAGE_LLM_TEMPERATURE }} + ISSUE_TRIAGE_LLM_MAX_COMMENTS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_COMMENTS }} + ISSUE_TRIAGE_LLM_MAX_COMMENT_CHARS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_COMMENT_CHARS }} + ISSUE_TRIAGE_LLM_MAX_BODY_CHARS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_BODY_CHARS }} + ISSUE_TRIAGE_LLM_API_KEY: ${{ secrets.ISSUE_TRIAGE_LLM_API_KEY }} + run: | + deno run --allow-env --allow-net \ + .github/scripts/issue-backlog-rescore.ts \ + --owner "${{ github.repository_owner }}" \ + --repo "${{ github.event.repository.name }}" \ + --limit "${{ inputs.limit || '0' }}" diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 00000000..c60d5d9e --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,62 @@ +name: Issue Triage + +on: + issues: + types: + - opened + - edited + - reopened + issue_comment: + types: + - created + workflow_dispatch: + inputs: + issue_number: + description: Issue number to re-triage manually + required: true + +concurrency: + group: issue-triage-${{ github.event.issue.number || inputs.issue_number }} + cancel-in-progress: true + +permissions: + contents: read + issues: write + +jobs: + triage: + if: | + github.event_name != 'issue_comment' || + ( + github.event.issue.pull_request == null && + contains(github.event.comment.body, '/retriage') + ) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: denoland/setup-deno@667a34cdef165d8d2b2e98dde39547c9daac7282 # v2.0.4 + with: + deno-version: v2.x + + - name: Run triage + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TRIAGE_LLM_MODE: ${{ vars.ISSUE_TRIAGE_LLM_MODE }} + ISSUE_TRIAGE_LLM_BASE_URL: ${{ vars.ISSUE_TRIAGE_LLM_BASE_URL }} + ISSUE_TRIAGE_LLM_MODEL: ${{ vars.ISSUE_TRIAGE_LLM_MODEL }} + ISSUE_TRIAGE_LLM_TIMEOUT_MS: ${{ vars.ISSUE_TRIAGE_LLM_TIMEOUT_MS }} + ISSUE_TRIAGE_LLM_MAX_ATTEMPTS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_ATTEMPTS }} + ISSUE_TRIAGE_LLM_RETRY_BACKOFF_MS: ${{ vars.ISSUE_TRIAGE_LLM_RETRY_BACKOFF_MS }} + ISSUE_TRIAGE_LLM_TEMPERATURE: ${{ vars.ISSUE_TRIAGE_LLM_TEMPERATURE }} + ISSUE_TRIAGE_LLM_MAX_COMMENTS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_COMMENTS }} + ISSUE_TRIAGE_LLM_MAX_COMMENT_CHARS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_COMMENT_CHARS }} + ISSUE_TRIAGE_LLM_MAX_BODY_CHARS: ${{ vars.ISSUE_TRIAGE_LLM_MAX_BODY_CHARS }} + ISSUE_TRIAGE_LLM_API_KEY: ${{ secrets.ISSUE_TRIAGE_LLM_API_KEY }} + run: | + deno run --allow-env --allow-net \ + .github/scripts/issue-triage.ts \ + --owner "${{ github.repository_owner }}" \ + --repo "${{ github.event.repository.name }}" \ + --issue-number "${{ github.event.issue.number || inputs.issue_number }}" diff --git a/docs/2026-04-08-issue-automation-design.md b/docs/2026-04-08-issue-automation-design.md new file mode 100644 index 00000000..420e78ab --- /dev/null +++ b/docs/2026-04-08-issue-automation-design.md @@ -0,0 +1,323 @@ +# Issue 自动分诊 MVP 设计 + +## 目标 + +通过自动将 GitHub issue 分诊到三个队列中，降低维护者负担： + +- `triage/deferred`：低优先级 issue，会随着时间推移逐步上浮 +- `triage/core`：高优先级或高风险 issue，需要 core maintainer 接手 +- `triage/agent-ready`：高优先级、低风险 issue，适合作为后续 agent 执行候选 + +本 MVP 版本还不会自动修复 issue。它聚焦在评分、路由、打标签，以及让 +backlog 持续流动。 + +当前版本支持两种执行模式： + +- 仅规则分诊 +- 规则 + 兼容 OpenAI 的 LLM 辅助 + +## 为什么这样拆分 + +最初的方案把优先级和执行难度混在同一个决策里。实践上，如果把它们拆开， +系统会更容易调参： + +- `Priority`：这个 issue 现在是否值得投入时间？ +- `Route`：一旦值得处理，应该由谁来接手？ + +这样一来，高价值但高难度的 issue 仍然可以保持高优先级，同时继续路由到 +`triage/core`。 + +## 输入 + +自动化会读取 issue 的实时标题、正文、标签、评论和时间戳。 + +结构化的 issue 表单字段来自： + +- [bug_report.yml](../.github/ISSUE_TEMPLATE/bug_report.yml) +- [feature_request.yml](../.github/ISSUE_TEMPLATE/feature_request.yml) +- [reward-task.yml](../.github/ISSUE_TEMPLATE/reward-task.yml) + +## 评分模型 + +每个 issue 会沿四个维度评分： + +- `impact`（1-5）：对用户和工作流的影响 +- `urgency`（1-5）：发布时间压力、功能损坏情况或重复讨论程度 +- `effort`（1-5）：预估改动规模和协作成本 +- `confidence`（1-5）：issue 描述的完整性和可执行程度 + +优先级计算公式如下： + +```text +priority = impact * 0.45 + urgency * 0.35 + age_boost + engagement_boost +``` + +其中： + +- `age_boost`：基于 SLA 的升级机制 + - 第 7-9 天：预热阶段，最低提升到 `priority/p2` + - 第 10-13 天：强制移出 `triage/deferred`，最低提升到 `priority/p1` + - 第 14 天及以后：在下一次 triage/rescore 时，将该 issue 视为已违反 SLA， + 并至少提升到 `priority/p0` +- `engagement_boost`：由评论压力和奖励金额共同决定，上限为 +1.0 + +在 MVP 中，`effort` 不会直接降低优先级，它只影响路由。 + +## LLM 辅助分诊 + +配置后，工作流可以调用兼容 OpenAI 的 chat completions API。 + +LLM 不会替代规则引擎。它只用于辅助： + +- 生成 issue 摘要 +- 对软性分数做微调 +- 生成 `needs-info` 的追问问题 +- 为维护者提供更好的判断依据 +- 为 `triage/core` 生成 maintainer 交接摘要 + +硬性门槛仍然由规则控制： + +- 缺失必填信息 +- auth、schema、migration、SDK 或公共契约变更等高风险区域 +- 最终是否可以提升到 `triage/agent-ready` + +issue 正文和评论都视为不可信输入。工作流会： + +- 在发送给模型前截断过长的正文和评论 +- 明确告诉模型，issue 文本是数据而不是指令 +- 使用严格的 JSON 协议校验模型输出 +- 如果 provider 调用失败或 JSON 校验失败，则回退到仅规则模式 + +### 模式 + +- `off`：仅规则 +- `shadow`：调用 LLM 并展示其建议，但最终仍沿用仅规则的路由和标签 +- `assist`：允许 LLM 对软性分数做最多 `+/-1` 的微调，然后重新应用硬性门槛 + +### 何时使用 LLM + +工作流只会在 issue 看起来存在歧义或价值较高时调用 LLM，例如： + +- `triage/needs-info` +- `triage/core` +- 靠近路由阈值的 issue +- 低置信度案例 +- 正文很长或讨论很多的 issue +- 需要更多判断的 feature 或 reward issue + +## 路由规则 + +1. `triage/needs-info` + 当缺少必填字段或 `confidence <= 2` 时触发。 + +2. `triage/deferred` + 当 `priority < 3.6`、issue 不受信息缺失阻塞、且 issue 年龄仍低于 SLA + 升级底线时触发。 + +3. `triage/core` + 当 `priority >= 3.6` 且满足以下任一条件时触发： + - issue 阻塞了 OpenClaw/ClawHub 核心工作流，例如 install、publish、 + update、sync 或基于 namespace 的发布 + - `effort >= 4` + - `confidence <= 3` + - 存在高风险关键词或会影响契约的字段 + +4. `triage/agent-ready` + 当 `priority >= 3.6`、`effort <= 3`、`confidence >= 4`，且不存在高风险 + 信号时触发。 + +在 `assist` 模式下，LLM 建议可以对 `impact`、`urgency`、`effort` 和 +`confidence` 各自最多调整 1 分。规则引擎随后会重新计算优先级和路由。 + +涉及 OpenClaw/ClawHub 核心工作流的 issue 是进入 `triage/core` 的硬性门槛； +LLM 辅助不会放宽这一规则。 + +## 受管标签 + +自动化负责管理以下标签前缀： + +- `triage/` +- `priority/` +- `effort/` +- `risk/` + +当前使用的具体标签有： + +- `triage/needs-info` +- `triage/deferred` +- `triage/core` +- `triage/agent-ready` +- `priority/p0` +- `priority/p1` +- `priority/p2` +- `priority/p3` +- `effort/s` +- `effort/m` +- `effort/l` +- `risk/high` + +其余所有标签都保持不变。 + +另外，自动化还识别一个不由其管理的人工操作标签： + +- `triage-manual`：冻结该 issue 的自动分诊更新 + +## 工作流 + +### 1. Issue 分诊 + +文件：[issue-triage.yml](../.github/workflows/issue-triage.yml) + +触发条件： + +- `issues.opened` +- `issues.edited` +- `issues.reopened` +- 当评论包含 `/retriage` 时触发 `issue_comment.created` +- `workflow_dispatch` + +执行动作： + +- 拉取 issue 和评论 +- 计算分数和路由 +- 更新或创建受管标签 +- 更新或创建一条分诊评论，其中同时包含人类可读的判断理由和隐藏的机器状态 +- 可选调用兼容 OpenAI 的 provider，并合并结果 + +### 2. Deferred Backlog 重新评分 + +文件： +[issue-backlog-rescore.yml](../.github/workflows/issue-backlog-rescore.yml) + +触发条件： + +- 每 6 小时一次 +- `workflow_dispatch` + +执行动作： + +- 列出所有带有 `triage/deferred` 标签的 open issue +- 结合年龄和参与度加成重新计算优先级 +- 决定将每个 issue 升级还是保留 +- 原地更新分诊评论 +- 当 issue 内容未变化时复用缓存的 LLM 结果 + +试运行说明： + +- 当前定时 rescore 只扫描 `triage/deferred` 队列中的 issue +- 这可以保证低优先级 backlog 不会在 `deferred` 中闲置超过第 10 天 +- 一旦某个 issue 已经从 `deferred` 中升级出去，之后第 14 天的进一步升级 + 依赖新的 triage 事件或手动 `/retriage` +- 在试运行阶段，14 天规则应被视为运营层面的 SLA 目标，而不是仓库范围内的 + 硬性计时器 + +## 脚本 + +新的 GitHub 自动化脚本位于 +[`.github/scripts`](/Users/wowo/workspace/skillhub/.github/scripts)： + +- [github.ts](/Users/wowo/workspace/skillhub/.github/scripts/github.ts)：精简版 + GitHub REST 客户端 +- [issue-triage-config.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-triage-config.ts)： + 标签、阈值和关键词规则 +- [issue-llm-config.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-llm-config.ts)： + LLM 模式、环境变量和调用启发式 +- [issue-llm-provider.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-llm-provider.ts)： + 兼容 OpenAI 的 chat completions 客户端 +- [issue-llm-evaluator.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-llm-evaluator.ts)： + prompt 构造、JSON 校验和缓存 key 生成 +- [issue-triage-lib.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-triage-lib.ts)： + 解析、评分、路由和评论渲染 +- [issue-triage-merge.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-triage-merge.ts)： + 有界合并和硬性门槛重应用 +- [issue-triage.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-triage.ts)： + 单 issue 入口 +- [issue-backlog-rescore.ts](/Users/wowo/workspace/skillhub/.github/scripts/issue-backlog-rescore.ts)： + deferred 队列重新评分入口 + +## 配置 + +设置以下 GitHub 仓库变量和 secret，即可启用 LLM 辅助分诊： + +仓库变量： + +- `ISSUE_TRIAGE_LLM_MODE` +- `ISSUE_TRIAGE_LLM_BASE_URL` +- `ISSUE_TRIAGE_LLM_MODEL` +- `ISSUE_TRIAGE_LLM_TIMEOUT_MS` 可选 +- `ISSUE_TRIAGE_LLM_TEMPERATURE` 可选 +- `ISSUE_TRIAGE_LLM_MAX_COMMENTS` 可选 +- `ISSUE_TRIAGE_LLM_MAX_COMMENT_CHARS` 可选 +- `ISSUE_TRIAGE_LLM_MAX_BODY_CHARS` 可选 + +仓库 secret： + +- `ISSUE_TRIAGE_LLM_API_KEY` + +建议的第一轮上线方式： + +- `ISSUE_TRIAGE_LLM_MODE=shadow` +- 先观察几天分诊评论 +- 等 LLM 建议看起来稳定后，再切换到 `assist` + +兼容 OpenAI 的变量示例： + +```text +ISSUE_TRIAGE_LLM_MODE=shadow +ISSUE_TRIAGE_LLM_BASE_URL=https://your-provider.example.com/v1 +ISSUE_TRIAGE_LLM_MODEL=gpt-4.1-mini +``` + +## 推出计划 + +### Phase 1：当前阶段 + +- 启用 triage 和 backlog rescore +- 观察几周的 issue 流量后微调阈值 +- 允许维护者通过 `triage-manual` 冻结特定 issue 的自动化处理 +- 如果使用 LLM，从 `shadow` 模式开始 + +### Phase 2：Maintainer 交接 + +为 `triage/core` issue 增加 issue-brief 生成器，输出内容包括： + +- 复现提示 +- 可能涉及的模块 +- 风险备注 +- 验证清单 + +这些输出可以直接用于本地编程 agent 会话，以及现有的并行 worktree 流程。 + +当前 MVP 已经会在 `triage/core` issue 的分诊评论中直接嵌入一个 +`Maintainer Brief` 区块。该摘要包括： + +- 简洁的 issue 摘要 +- issue 为什么被升级到 core +- 复现路径或操作路径备注 +- 疑似相关模块或工作流负责人 +- 风险提示 +- 验证清单 + +### Phase 3：自托管 Issue Agent + +增加一个自托管 runner，监听 `triage/agent-ready`，并执行： + +- 创建隔离的分支和 worktree +- 运行解决 issue 的 agent +- 执行最小相关测试集 +- 打开一个 draft PR + +在这个阶段，以下场景仍应保留硬性阻断： + +- auth 和权限变更 +- 安全敏感变更 +- schema 或 migration 相关工作 +- 公共 API、SDK 或 CLI 契约变更 + +## 待调优问题 + +- 参与度加成是否只看评论数就够了，还是也应该拉取 reactions +- reward issue 是否应比当前 MVP 获得更强的价值加成 +- `agent-ready` 是否应要求 `effort <= 2`，而不是 `<= 3` +- 某些区域（如 `scanner`）是否应默认视为高风险 +- 某些团队是否应长期保持 `shadow` 模式，只把 `assist` 用在更窄的仓库子集上 diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/compat/ClawHubCompatControllerTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/compat/ClawHubCompatControllerTest.java index 421fa1d8..1a03b672 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/compat/ClawHubCompatControllerTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/compat/ClawHubCompatControllerTest.java @@ -1,12 +1,17 @@ package com.iflytek.skillhub.compat; import com.iflytek.skillhub.auth.rbac.PlatformPrincipal; -import com.iflytek.skillhub.domain.namespace.NamespaceMemberRepository; import com.iflytek.skillhub.auth.device.DeviceAuthService; +import com.iflytek.skillhub.domain.namespace.Namespace; +import com.iflytek.skillhub.domain.namespace.NamespaceMemberRepository; import com.iflytek.skillhub.domain.skill.service.SkillQueryService; +import com.iflytek.skillhub.domain.skill.Skill; +import com.iflytek.skillhub.domain.skill.SkillVisibility; import com.iflytek.skillhub.dto.SkillLifecycleVersionResponse; import com.iflytek.skillhub.dto.SkillSummaryResponse; import com.iflytek.skillhub.service.SkillSearchAppService; +import java.math.BigDecimal; +import java.time.Instant; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; @@ -18,9 +23,8 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.web.servlet.MockMvc; import java.util.List; +import java.util.Optional; import java.util.Set; -import java.math.BigDecimal; -import java.time.Instant; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -49,6 +53,9 @@ class ClawHubCompatControllerTest { @MockBean private SkillQueryService skillQueryService; + @MockBean + private CompatSkillLookupService compatSkillLookupService; + @Test void search_returns_mapped_results() throws Exception { when(skillSearchAppService.search("test", null, "relevance", 0, 20, null, null)) @@ -124,6 +131,8 @@ class ClawHubCompatControllerTest { @Test void resolve_query_with_legacy_slug_keeps_legacy_lookup_behavior() throws Exception { + when(compatSkillLookupService.findByLegacySlug("my-skill")) + .thenReturn(legacyCompatContext("global", "my-skill")); when(skillQueryService.resolveVersion("global", "my-skill", null, "latest", null, null, java.util.Map.of())) .thenReturn(new SkillQueryService.ResolvedVersionDTO( 1L, "global", "my-skill", "latest", 2L, "sha", true, "/api/v1/skills/global/my-skill/download")); @@ -149,6 +158,8 @@ class ClawHubCompatControllerTest { @Test void download_query_with_legacy_slug_keeps_legacy_lookup_behavior() throws Exception { + when(compatSkillLookupService.findByLegacySlug("my-skill")) + .thenReturn(legacyCompatContext("global", "my-skill")); mockMvc.perform(get("/api/v1/download") .param("slug", "my-skill") .param("version", "latest")) @@ -192,4 +203,10 @@ class ClawHubCompatControllerTest { .andExpect(jsonPath("$.user.displayName").value("tester")) .andExpect(jsonPath("$.user.image").value("https://example.com/avatar.png")); } + + private CompatSkillLookupService.CompatSkillContext legacyCompatContext(String namespaceSlug, String skillSlug) { + Namespace namespace = new Namespace(namespaceSlug, namespaceSlug, "tester"); + Skill skill = new Skill(1L, skillSlug, "tester", SkillVisibility.PUBLIC); + return new CompatSkillLookupService.CompatSkillContext(namespace, skill, Optional.empty()); + } }