From 30e0c7c7261eb246992c915d5be4d22b0eeaa95c Mon Sep 17 00:00:00 2001 From: rcarmel999 <79278568+rcarmel999@users.noreply.github.com> Date: Sat, 9 May 2026 03:07:24 -0500 Subject: [PATCH 01/11] fix(csharp): include generic typed properties in context and impact (#1399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix C# context and impact for generic typed properties * Address C# typed-property review feedback --------- Co-authored-by: Richard Carmel Co-authored-by: Gergő Magyar --- .../field-extractors/configs/csharp.ts | 11 +- .../ingestion/languages/csharp/captures.ts | 95 ++++++++++++++ .../core/ingestion/languages/csharp/query.ts | 6 + gitnexus/src/core/lbug/csv-generator.ts | 9 +- gitnexus/src/core/lbug/lbug-adapter.ts | 13 ++ gitnexus/src/core/lbug/schema.ts | 13 +- gitnexus/src/mcp/local/local-backend.ts | 119 +++++++++++++++--- .../csharp-generic-type-refs/Program.cs | 18 +++ .../context-typed-property.test.ts | 77 ++++++++++++ .../test/integration/resolvers/csharp.test.ts | 18 +++ .../test/integration/resolvers/helpers.ts | 3 + gitnexus/test/unit/schema.test.ts | 6 + .../csharp/csharp-captures.test.ts | 43 +++++++ 13 files changed, 408 insertions(+), 23 deletions(-) create mode 100644 gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs create mode 100644 gitnexus/test/integration/context-typed-property.test.ts diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts index 4d83a6cd7..75032a9d7 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts @@ -9,6 +9,11 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; const CSHARP_VIS = new Set(['public', 'private', 'protected', 'internal']); +const extractCsharpDeclaredType = (typeNode: SyntaxNode): string | undefined => { + if (typeNode.type === 'generic_name') return typeNode.text.trim(); + return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim(); +}; + /** * C# field extraction config. * @@ -53,17 +58,17 @@ export const csharpConfig: FieldExtractionConfig = { const child = node.namedChild(i); if (child?.type === 'variable_declaration') { const typeNode = child.childForFieldName('type'); - if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim(); + if (typeNode) return extractCsharpDeclaredType(typeNode); // fallback: first child that is a type const first = child.firstNamedChild; if (first && first.type !== 'variable_declarator') { - return extractSimpleTypeName(first) ?? first.text?.trim(); + return extractCsharpDeclaredType(first); } } } // property_declaration: type is first named child const typeNode = node.childForFieldName('type'); - if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim(); + if (typeNode) return extractCsharpDeclaredType(typeNode); return undefined; }, diff --git a/gitnexus/src/core/ingestion/languages/csharp/captures.ts b/gitnexus/src/core/ingestion/languages/csharp/captures.ts index 2f913a14e..e29552e78 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/captures.ts @@ -42,6 +42,39 @@ const FUNCTION_NODE_TYPES = [ 'local_function_statement', ] as const; +const BUILTIN_TYPE_NAMES = new Set([ + 'bool', + 'byte', + 'char', + 'decimal', + 'double', + 'float', + 'int', + 'long', + 'object', + 'sbyte', + 'short', + 'string', + 'uint', + 'ulong', + 'ushort', + 'void', +]); + +function shouldEmitReadMember(memberNode: SyntaxNode): boolean { + const parent = memberNode.parent; + if (parent === null) return true; + + switch (parent.type) { + case 'invocation_expression': + return parent.childForFieldName('function')?.id !== memberNode.id; + case 'assignment_expression': + return parent.childForFieldName('left')?.id !== memberNode.id; + default: + return true; + } +} + export function emitCsharpScopeCaptures( sourceText: string, _filePath: string, @@ -94,6 +127,14 @@ export function emitCsharpScopeCaptures( continue; } + if (grouped['@reference.read.member'] !== undefined) { + const anchor = grouped['@reference.read.member']; + const memberNode = findNodeAtRange(tree.rootNode, anchor.range, 'member_access_expression'); + if (memberNode === null || !shouldEmitReadMember(memberNode)) { + continue; + } + } + // Synthesize `this` / `base` receiver type-bindings on every // instance method-like. Tree-sitter can't cleanly express "the // implicit receiver of a non-static member of a class/struct/ @@ -209,9 +250,63 @@ export function emitCsharpScopeCaptures( } } + out.push(...synthesizeGenericTypeArgumentReferences(tree.rootNode)); + return out; } +function synthesizeGenericTypeArgumentReferences(root: SyntaxNode): CaptureMatch[] { + const out: CaptureMatch[] = []; + // Treat all generic type arguments as static type references, including + // declaration signatures and call-site generic instantiations. + visit(root, (node) => { + if (node.type !== 'generic_name') return; + const args = findNamedChild(node, 'type_argument_list'); + if (args === null) return; + + for (const arg of args.namedChildren) { + if (arg === null) continue; + const nameNode = terminalTypeNameNode(arg); + if (nameNode === null) continue; + if (BUILTIN_TYPE_NAMES.has(nameNode.text)) continue; + out.push({ + '@reference.type': nodeToCapture('@reference.type', nameNode), + '@reference.name': nodeToCapture('@reference.name', nameNode), + }); + } + }); + return out; +} + +function terminalTypeNameNode(node: SyntaxNode): SyntaxNode | null { + switch (node.type) { + case 'identifier': + return node; + case 'nullable_type': + return node.firstNamedChild === null ? null : terminalTypeNameNode(node.firstNamedChild); + case 'qualified_name': + return node.lastNamedChild; + case 'generic_name': + return node.childForFieldName('name') ?? node.firstNamedChild; + default: + return null; + } +} + +function findNamedChild(node: SyntaxNode, type: string): SyntaxNode | null { + for (const child of node.namedChildren) { + if (child !== null && child.type === type) return child; + } + return null; +} + +function visit(node: SyntaxNode, cb: (node: SyntaxNode) => void): void { + cb(node); + for (const child of node.namedChildren) { + if (child !== null) visit(child, cb); + } +} + /** C# 12 primary constructor: `class X(a, b) { }` / `record X(a, b)`. * The parameters are a bare `parameter_list` named child of the type * declaration (no `constructor_declaration` node). Emit a synthetic diff --git a/gitnexus/src/core/ingestion/languages/csharp/query.ts b/gitnexus/src/core/ingestion/languages/csharp/query.ts index 11da2d8cd..f299aaafa 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/query.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/query.ts @@ -499,6 +499,12 @@ const CSHARP_SCOPE_QUERY = ` left: (member_access_expression expression: "base" @reference.receiver name: (identifier) @reference.name)) @reference.write.member + +;; References — field/property reads: \`obj.Name\` +;; Emit-side filtering drops call targets and assignment left-hand sides. +(member_access_expression + expression: (_) @reference.receiver + name: (identifier) @reference.name) @reference.read.member `; let _parser: Parser | null = null; diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 616a00cac..20df51ebd 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -301,11 +301,15 @@ export const streamAllCSVsToDisk = async ( 'Template', 'Module', ] as const; + const propertyHeader = 'id,name,filePath,startLine,endLine,content,description,declaredType'; const multiLangWriters = new Map(); for (const t of MULTI_LANG_TYPES) { multiLangWriters.set( t, - new BufferedCSVWriter(path.join(csvDir, `${t.toLowerCase()}.csv`), multiLangHeader), + new BufferedCSVWriter( + path.join(csvDir, `${t.toLowerCase()}.csv`), + t === 'Property' ? propertyHeader : multiLangHeader, + ), ); } @@ -478,6 +482,9 @@ export const streamAllCSVsToDisk = async ( escapeCSVNumber(node.properties.endLine, -1), escapeCSVField(content), escapeCSVField(node.properties.description || ''), + ...(node.label === 'Property' + ? [escapeCSVField(node.properties.declaredType || '')] + : []), ].join(','), ); } diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index fb4cf76de..cb88986ca 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -608,6 +608,9 @@ const getCopyQuery = (table: NodeTableName, filePath: string): string => { if (table === 'Method') { return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description, parameterCount, returnType) FROM "${filePath}" ${COPY_CSV_OPTS}`; } + if (table === 'Property') { + return `COPY ${t}(id, name, filePath, startLine, endLine, content, description, declaredType) FROM "${filePath}" ${COPY_CSV_OPTS}`; + } // TypeScript/JS code element tables have isExported; multi-language tables do not if (TABLES_WITH_EXPORTED.has(table)) { return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description) FROM "${filePath}" ${COPY_CSV_OPTS}`; @@ -659,6 +662,11 @@ export const insertNodeToLbug = async ( ? `, description: ${escapeValue(properties.description)}` : ''; query = `CREATE (n:${t} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, isExported: ${!!properties.isExported}, content: ${escapeValue(properties.content || '')}${descPart}})`; + } else if (label === 'Property') { + const descPart = properties.description + ? `, description: ${escapeValue(properties.description)}` + : ''; + query = `CREATE (n:${t} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, content: ${escapeValue(properties.content || '')}${descPart}, declaredType: ${escapeValue(properties.declaredType || '')}})`; } else { // Multi-language tables (Struct, Impl, Trait, Macro, etc.) — no isExported const descPart = properties.description @@ -737,6 +745,11 @@ export const batchInsertNodesToLbug = async ( ? `, n.description = ${escapeValue(properties.description)}` : ''; query = `MERGE (n:${t} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.isExported = ${!!properties.isExported}, n.content = ${escapeValue(properties.content || '')}${descPart}`; + } else if (label === 'Property') { + const descPart = properties.description + ? `, n.description = ${escapeValue(properties.description)}` + : ''; + query = `MERGE (n:${t} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.content = ${escapeValue(properties.content || '')}${descPart}, n.declaredType = ${escapeValue(properties.declaredType || '')}`; } else { const descPart = properties.description ? `, n.description = ${escapeValue(properties.description)}` diff --git a/gitnexus/src/core/lbug/schema.ts b/gitnexus/src/core/lbug/schema.ts index 5579036f1..cd6838dd8 100644 --- a/gitnexus/src/core/lbug/schema.ts +++ b/gitnexus/src/core/lbug/schema.ts @@ -167,7 +167,18 @@ export const TYPE_ALIAS_SCHEMA = CODE_ELEMENT_BASE('TypeAlias'); export const CONST_SCHEMA = CODE_ELEMENT_BASE('Const'); export const STATIC_SCHEMA = CODE_ELEMENT_BASE('Static'); export const VARIABLE_SCHEMA = CODE_ELEMENT_BASE('Variable'); -export const PROPERTY_SCHEMA = CODE_ELEMENT_BASE('Property'); +export const PROPERTY_SCHEMA = ` +CREATE NODE TABLE \`Property\` ( + id STRING, + name STRING, + filePath STRING, + startLine INT64, + endLine INT64, + content STRING, + description STRING, + declaredType STRING, + PRIMARY KEY (id) +)`; export const RECORD_SCHEMA = CODE_ELEMENT_BASE('Record'); export const DELEGATE_SCHEMA = CODE_ELEMENT_BASE('Delegate'); export const ANNOTATION_SCHEMA = CODE_ELEMENT_BASE('Annotation'); diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 6175af875..167c1db81 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -1756,12 +1756,13 @@ export class LocalBackend { repo.id, ` MATCH (caller)-[r:CodeRelation]->(n {id: $symId}) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind LIMIT 30 `, { symId }, ); + let typedPropertyRows: any[] = []; // Fix #480: Class/Interface nodes have no direct CALLS/IMPORTS edges — // those point to Constructor and File nodes respectively. Fetch those @@ -1795,23 +1796,24 @@ export class LocalBackend { if (isClassLike) { try { - // Run both incoming-ref queries in parallel — they are independent. - const [ctorIncoming, fileIncoming] = await Promise.all([ - executeParameterized( - repo.id, - ` + // Run incoming-ref queries in parallel — they are independent. + const [ctorIncoming, fileIncoming, typedPropertyIncoming, typedProperties] = + await Promise.all([ + executeParameterized( + repo.id, + ` MATCH (n)-[hm:CodeRelation]->(ctor:Constructor) WHERE n.id = $symId AND hm.type = 'HAS_METHOD' MATCH (caller)-[r:CodeRelation]->(ctor) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'ACCESSES'] RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind LIMIT 30 `, - { symId }, - ), - executeParameterized( - repo.id, - ` + { symId }, + ), + executeParameterized( + repo.id, + ` MATCH (f:File)-[rel:CodeRelation]->(n) WHERE n.id = $symId AND rel.type = 'DEFINES' MATCH (caller)-[r:CodeRelation]->(f) @@ -1819,9 +1821,45 @@ export class LocalBackend { RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind LIMIT 30 `, - { symId }, - ), - ]); + { symId }, + ), + executeParameterized( + repo.id, + ` + MATCH (p:\`Property\`) + WHERE p.declaredType = $name + OR p.declaredType STARTS WITH $genericPrefix + OR p.declaredType CONTAINS $genericArg + MATCH (caller)-[r:CodeRelation]->(p) + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'ACCESSES'] + RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind + LIMIT 30 + `, + { + name: sym.name, + genericPrefix: `${sym.name}<`, + genericArg: `<${sym.name}>`, + }, + ), + executeParameterized( + repo.id, + ` + MATCH (p:\`Property\`) + WHERE p.declaredType = $name + OR p.declaredType STARTS WITH $genericPrefix + OR p.declaredType CONTAINS $genericArg + RETURN p.id AS uid, p.name AS name, p.filePath AS filePath, labels(p)[0] AS kind, + p.declaredType AS declaredType + LIMIT 30 + `, + { + name: sym.name, + genericPrefix: `${sym.name}<`, + genericArg: `<${sym.name}>`, + }, + ), + ]); + typedPropertyRows = typedProperties; // Deduplicate by (relType, uid) — a caller can have multiple relation // types to the same target (e.g. both IMPORTS and CALLS), and each @@ -1829,7 +1867,7 @@ export class LocalBackend { const seenKeys = new Set( incomingRows.map((r: any) => `${r.relType || r[0]}:${r.uid || r[1]}`), ); - for (const r of [...ctorIncoming, ...fileIncoming]) { + for (const r of [...ctorIncoming, ...fileIncoming, ...typedPropertyIncoming]) { const key = `${r.relType || r[0]}:${r.uid || r[1]}`; if (!seenKeys.has(key)) { seenKeys.add(key); @@ -1846,7 +1884,7 @@ export class LocalBackend { repo.id, ` MATCH (n {id: $symId})-[r:CodeRelation]->(target) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] RETURN r.type AS relType, target.id AS uid, target.name AS name, target.filePath AS filePath, labels(target)[0] AS kind LIMIT 30 `, @@ -1935,6 +1973,17 @@ export class LocalBackend { }, incoming: categorize(incomingRows), outgoing: categorize(outgoingRows), + ...(typedPropertyRows.length > 0 + ? { + typed_properties: typedPropertyRows.map((r: any) => ({ + uid: r.uid || r[0], + name: r.name || r[1], + filePath: r.filePath || r[2], + kind: r.kind || r[3], + declaredType: r.declaredType || r[4], + })), + } + : {}), processes: processRows.map((r: any) => ({ id: r.pid || r[0], name: r.label || r[1], @@ -2500,6 +2549,7 @@ export class LocalBackend { const mappedRelTypes = params.relationTypes?.flatMap((t: string) => t === 'OVERRIDES' ? ['OVERRIDES', 'METHOD_OVERRIDES'] : [t], ); + const hasExplicitRelationTypes = mappedRelTypes !== undefined && mappedRelTypes.length > 0; const rawRelTypes = mappedRelTypes && mappedRelTypes.length > 0 ? mappedRelTypes.filter((t: string) => VALID_RELATION_TYPES.has(t)) @@ -2508,6 +2558,7 @@ export class LocalBackend { 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', + 'USES', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', @@ -2520,6 +2571,7 @@ export class LocalBackend { 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', + 'USES', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', @@ -2579,9 +2631,16 @@ export class LocalBackend { }; const symType = outcome.resolvedLabel || outcome.symbol.type || ''; + const effectiveRelationTypes = + (symType === 'Class' || symType === 'Interface') && + !hasExplicitRelationTypes && + !relationTypes.includes('ACCESSES') + ? [...relationTypes, 'ACCESSES'] + : relationTypes; + return this._runImpactBFS(repo, sym, symType, direction, { maxDepth, - relationTypes, + relationTypes: effectiveRelationTypes, includeTests, minConfidence, }); @@ -2660,6 +2719,30 @@ export class LocalBackend { frontier.push(rid); } } + + const typedPropertyRows = await executeParameterized( + repo.id, + ` + MATCH (p:\`Property\`) + WHERE p.declaredType = $name + OR p.declaredType STARTS WITH $genericPrefix + OR p.declaredType CONTAINS $genericArg + RETURN p.id AS id, p.name AS name, labels(p)[0] AS type, p.filePath AS filePath + `, + { + name: sym.name, + genericPrefix: `${sym.name}<`, + genericArg: `<${sym.name}>`, + }, + ); + + for (const r of typedPropertyRows) { + const rid = r.id || r[0]; + if (rid && !visited.has(rid)) { + visited.add(rid); + frontier.push(rid); + } + } } catch (e) { logQueryError('impact:class-node-expansion', e); } diff --git a/gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs b/gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs new file mode 100644 index 000000000..28674ecda --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs @@ -0,0 +1,18 @@ +namespace App; + +public class USER_INFO +{ + public string? USER_ID { get; set; } +} + +public interface IEntityTypeConfiguration +{ +} + +public class UserInfoConfiguration : IEntityTypeConfiguration +{ + public Task> Load(List users) + { + return Task.FromResult(users); + } +} diff --git a/gitnexus/test/integration/context-typed-property.test.ts b/gitnexus/test/integration/context-typed-property.test.ts new file mode 100644 index 000000000..aef442147 --- /dev/null +++ b/gitnexus/test/integration/context-typed-property.test.ts @@ -0,0 +1,77 @@ +/** + * Integration test: context() expands Class symbols through typed properties. + * + * Reproduces EF-style usage where code reads a DbContext property + * (`db.USER_INFO`) whose source type is `DbSet`. The direct + * graph edge is Method -> Property, not Method -> Class, so context() must + * use the same typed-property bridge that impact() uses. + */ +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; + +vi.mock('../../src/storage/repo-manager.js', () => ({ + listRegisteredRepos: vi.fn().mockResolvedValue([]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), +})); + +const SEED = [ + `CREATE (c:Class {id:'Class:Models/USER_INFO.cs:USER_INFO', name:'USER_INFO', filePath:'Models/USER_INFO.cs', startLine:1, endLine:5, content:'public class USER_INFO {}', description:''})`, + `CREATE (p:\`Property\` {id:'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO', name:'USER_INFO', filePath:'Data/UserDbContext.cs', startLine:10, endLine:10, content:'public DbSet USER_INFO { get; set; }', description:'', declaredType:'DbSet'})`, + `CREATE (m:Method {id:'Method:Services/UserService.cs:UserService.GetUserInfo#1', name:'GetUserInfo', filePath:'Services/UserService.cs', startLine:20, endLine:30, isExported:false, content:'db.USER_INFO.FirstOrDefault();', description:'', parameterCount:1, returnType:'USER_INFO'})`, + `MATCH (m:Method {id:'Method:Services/UserService.cs:UserService.GetUserInfo#1'}), (p:\`Property\` {id:'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO'}) CREATE (m)-[:CodeRelation {type:'ACCESSES', confidence:1.0, reason:'read', step:1}]->(p)`, +]; + +withTestLbugDB( + 'context-typed-property', + (handle) => { + let backend: LocalBackend; + + beforeAll(async () => { + backend = (handle as any)._backend; + }); + + describe('context() typed-property expansion', () => { + it('surfaces property callers and explains the typed property bridge', async () => { + const result = await backend.callTool('context', { + uid: 'Class:Models/USER_INFO.cs:USER_INFO', + }); + + expect(result.status).toBe('found'); + expect(result.symbol.kind).toBe('Class'); + + const accesses = result.incoming.accesses || []; + expect(accesses.map((r: any) => r.name)).toContain('GetUserInfo'); + + expect(result.typed_properties).toEqual([ + expect.objectContaining({ + uid: 'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO', + name: 'USER_INFO', + declaredType: 'DbSet', + }), + ]); + }); + }); + }, + { + seed: SEED, + poolAdapter: true, + afterSetup: async (handle) => { + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc123', + stats: { files: 3, nodes: 3, communities: 0, processes: 0 }, + }, + ]); + const backend = new LocalBackend(); + await backend.init(); + (handle as any)._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/resolvers/csharp.test.ts b/gitnexus/test/integration/resolvers/csharp.test.ts index 613ff7ec2..073b2da16 100644 --- a/gitnexus/test/integration/resolvers/csharp.test.ts +++ b/gitnexus/test/integration/resolvers/csharp.test.ts @@ -1440,6 +1440,24 @@ describe('Write access tracking (C#)', () => { }); }); +// --------------------------------------------------------------------------- +// Generic type references: IEntityTypeConfiguration, List +// --------------------------------------------------------------------------- + +describe('C# generic type-reference tracking', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'csharp-generic-type-refs'), () => {}); + }, 60000); + + it('emits USES edges for generic type arguments', () => { + const uses = getRelationships(result, 'USES').filter((e) => e.target === 'USER_INFO'); + expect(edgeSet(uses)).toContain('UserInfoConfiguration → USER_INFO'); + expect(edgeSet(uses)).toContain('Load → USER_INFO'); + }); +}); + // --------------------------------------------------------------------------- // Call-result variable binding (Phase 9): var user = GetUser(); user.Save() // --------------------------------------------------------------------------- diff --git a/gitnexus/test/integration/resolvers/helpers.ts b/gitnexus/test/integration/resolvers/helpers.ts index 5368c1715..e92526b19 100644 --- a/gitnexus/test/integration/resolvers/helpers.ts +++ b/gitnexus/test/integration/resolvers/helpers.ts @@ -11,6 +11,9 @@ import type { GraphRelationship } from 'gitnexus-shared'; const LEGACY_RESOLVER_PARITY_EXPECTED_FAILURES: Readonly>> = { csharp: new Set([ 'emits the using-import edge App/Program.cs -> Models/User.cs through the scope-resolution path', + // Generic type-argument USES edges are emitted by the registry-primary + // resolver only; the legacy DAG path does not synthesize these references. + 'emits USES edges for generic type arguments', ]), go: new Set([ // The legacy DAG path does not resolve method calls when the method is diff --git a/gitnexus/test/unit/schema.test.ts b/gitnexus/test/unit/schema.test.ts index cab80984a..17db9bdf9 100644 --- a/gitnexus/test/unit/schema.test.ts +++ b/gitnexus/test/unit/schema.test.ts @@ -13,6 +13,7 @@ import { CLASS_SCHEMA, INTERFACE_SCHEMA, METHOD_SCHEMA, + PROPERTY_SCHEMA, CODE_ELEMENT_SCHEMA, COMMUNITY_SCHEMA, PROCESS_SCHEMA, @@ -117,6 +118,11 @@ describe('LadybugDB Schema', () => { expect(FUNCTION_SCHEMA).toContain('isExported BOOLEAN'); }); + it('Property schema preserves declaredType', () => { + expect(SCHEMA_QUERIES).toContain(PROPERTY_SCHEMA); + expect(PROPERTY_SCHEMA).toContain('declaredType STRING'); + }); + it('Community schema has heuristicLabel and cohesion', () => { expect(COMMUNITY_SCHEMA).toContain('heuristicLabel STRING'); expect(COMMUNITY_SCHEMA).toContain('cohesion DOUBLE'); diff --git a/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts b/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts index c8111d203..9fa9bf181 100644 --- a/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts +++ b/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts @@ -422,4 +422,47 @@ describe('emitCsharpScopeCaptures — references', () => { expect(m!['@reference.receiver'].text).toBe('obj'); expect(m!['@reference.name'].text).toBe('Name'); }); + + it('captures member reads `obj.Name`', () => { + const m = findMatch('class A { void M(User obj) { var name = obj.Name; } }', (t) => + t.includes('@reference.read.member'), + ); + expect(m).toBeDefined(); + expect(m!['@reference.receiver'].text).toBe('obj'); + expect(m!['@reference.name'].text).toBe('Name'); + }); + + it('does not capture member calls as member reads', () => { + const matches = emitCsharpScopeCaptures( + 'class A { void M(User obj) { obj.Save(); } }', + 'test.cs', + ); + expect(matches.some((m) => '@reference.call.member' in m)).toBe(true); + expect(matches.some((m) => '@reference.read.member' in m)).toBe(false); + }); + + it('captures generic type arguments as type references', () => { + const matches = emitCsharpScopeCaptures( + 'class A : IEntityTypeConfiguration { public Task> Load(List users) => null!; }', + 'test.cs', + ); + const names = matches + .filter((m) => '@reference.type' in m) + .map((m) => m['@reference.name'].text); + + expect(names).toContain('USER_INFO'); + expect(names).not.toContain('string'); + }); + + it('captures call-site generic type arguments as type references', () => { + const matches = emitCsharpScopeCaptures( + 'class A { void M(IRepo repo) { repo.Get(); } }', + 'test.cs', + ); + const names = matches + .filter((m) => '@reference.type' in m) + .map((m) => m['@reference.name'].text); + + expect(names).toContain('USER_INFO'); + }); }); From 32b5c0e3fc8c1ec7eca1f30a76850cacc45728da Mon Sep 17 00:00:00 2001 From: WENJIE HUANG <82434538+SZU-WenjieHuang@users.noreply.github.com> Date: Sat, 9 May 2026 16:31:59 +0800 Subject: [PATCH 02/11] feat: add IncludeExtractor for C++ cross-repo include tracking (group) (#1156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add IncludeExtractor for C++ cross-repo include tracking (group) * fix: address CodeQL warnings on include-extractor - Remove unused HEADER_GLOB constant in include-extractor.ts - Use fs.mkdtempSync for secure temp dir creation in tests (CodeQL: 'Insecure temporary file') * fix(group): close missing ); in manifest-extractor include branch The 'include' branch in ManifestExtractor.resolveSymbol was missing the closing ); for the executor() call, causing a syntax error that broke ESLint, Prettier, and the full test CI on all platforms. Reported by Claude PR review on #1156. * chore: drop test/global-setup.ts + test/vitest.d.ts Upstream removed these in commit 3f0c74fe (ladybugdb 0.16.0 upgrade). Commit 3f5d21c5 accidentally restored them during a rebase dance. * style(group): reformat VALID_CONTRACT_TYPES array to satisfy prettier Adding 'include' pushed the array over prettier's 100-char limit, so prettier prefers multi-line. Apply the reformat to unbreak ci-quality/format job. * fix(include-extractor): address PR #1156 Claude review findings #3-#7 Claude Deep Review raised 7 findings on the IncludeExtractor. #1/#2 (BLOCKERs) were fixed earlier. This commit closes the remaining five. #3 HIGH case-sensitive FS -> provider contract-id collision Document the deliberate case-folding trade-off on normalizeIncludePath (matches C/C++ convention on Windows/macOS; collapses Foo.h & foo.h on Linux). Add a unit test pinning the behavior. #4 HIGH suffixResolve short-suffix match silently drops cross-repo include When a local file ends with the same basename as an external include (e.g. local internal/api.h vs. #include "ext/api.h"), suffixResolve returned a bogus local hit and suppressed the cross-repo consumer. Replace the suffixResolve lookup inside include-extractor with a strict isLocalInclude() that only accepts full-path hits via SuffixIndex.get / getInsensitive. Callers of suffixResolve elsewhere are unaffected. Add 3 unit tests covering the regression. #5 MEDIUM regex fallback matched #include inside /* ... */ Strip block comments before running the fallback regex scan. Add a unit test. #6 MEDIUM meta.source was hard-coded to 'tree_sitter' Track the actual extraction path with an extractionSource local and write it into meta.source so downstream audits can distinguish tree-sitter parses from regex fallbacks. Add 2 unit tests. #7 MEDIUM missing end-to-end coverage Add test/integration/group/include-extractor-sync.test.ts with 3 cases exercising extractor -> syncGroup -> CrossLink (mocked contracts, mixed-case/backslash normalization, real temp repos). Tests: 21 unit + 3 integration, all green. * fix(lbug): robust Windows lock acquisition for CI integration tests LadybugDB's `new Database()` raises `Could not set lock on file` from local_file_system.cpp synchronously inside the constructor — before any query is issued, so `withLbugDb`'s query-time retry never sees it. On Windows CI this surfaces as flaky integration tests due to AV-scanner holds, libuv handle-release lag, and stale `.wal` sidecars from aborted prior runs. This change closes the gap at *open time*: - `openLbugConnection` now wraps `new lbug.Database()` in a bounded busy-retry (5x100ms back-off) inside `lbug-config.ts`. Errors that exhaust the budget are tagged via `LBUG_OPEN_RETRY_EXHAUSTED` so `withLbugDb`'s outer 3x retry skips re-retrying a freshly-exhausted path (eliminates the 3x5=15-attempt / ~6s tail latency). - For recognized test fixtures only (immediate-parent dir matches a known prefix AND resolves under `os.tmpdir()`), one final stale- sidecar sweep removes `.wal`/`.lock` and retries once. Production paths never enter this branch. - `safeClose` on Windows runs a bounded `fs.open` probe to absorb native handle-release lag; logs a warning if the probe exhausts so operators can spot AV interference. - `isDbBusyError` is now defined in `lbug-config.ts` as the single source of truth, re-exported from `lbug-adapter.ts` for compatibility. - New tests cover open-time retry (happy/retry/exhaust/non-busy/tag), stale-sidecar sweep (test-fixture-only, production-rejection, preserves-original-error), `isTestFixturePath` direct unit suite (accept/reject/traversal/nested/trailing-sep), and `waitForWindowsHandleRelease` (openable/ENOENT/no-leak). - The two new test files are added to vitest's existing serialized `lbug-db` project (already `fileParallelism: false`). Closes the chronic Windows CI flake on lbug-touching integration tests while preserving the existing single-writable-Database-per-process LadybugDB contract. No public API surface changed. Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(lbug): drop isDbBusyError re-export, import from lbug-config directly The re-export from lbug-adapter.ts was a transitional convenience — with the matcher now living in lbug-config.ts, having two import paths for the same symbol invites future drift. Updated the two real consumers (lbug-lock-retry.test.ts, lbug-open-retry.test.ts) to import from lbug-config directly, removed the re-export equality test (now vacuous), and refreshed the explanatory comment so it no longer references a re-export pattern that doesn't exist. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(lbug): silence benign LadybugDB v0.16.1 schema-init lock warnings on Windows doInitLbug logs "⚠️ Schema creation warning: ... Could not set lock on file" on every CREATE NODE TABLE call after the first init on a given dbPath, on Windows. The lock is internal to LadybugDB v0.16.1 and is resolved before the table is created — same tolerance pattern as the existing "already exists" filter. Genuine cross-process lock contention still surfaces on the next operation through withLbugDb's retry, so filtering at the schema-init catch only suppresses noise, not signal. Also extend the safeClose Windows handle-release probe to cover the .wal sidecar (the previous Database's WAL handle was the slowest to release, surfacing as the schema-query lock contention) and switch the probe back to 'r+' so it actually detects exclusive locks. Test loop in lbug-close-handle-release.test.ts simplified to 10 plain iterations now that the underlying noise is filtered upstream. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(lbug): isDbBusyError review fixes - Drop redundant `could not set lock` term — already subsumed by `lock`. - Document the intentionally-broad matcher: graph-DB lock-shaped errors ("deadlock", "unlock failed", "lock contention", "could not open lock file") are all treated as transient. If a non-transient surfaces, tighten the matcher rather than raise the retry budget. - Add positive test cases covering those lock-shaped strings so the intent is visible and a future tightening would deliberately break these. - Fix the open-retry back-off comment: max sleep is 100+200+300+400 = 1000ms (no sleep after the final attempt), not 1.5s. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(group): address PR #1156 follow-up review findings Addresses two blockers and two mediums from the deep review. BLOCKER 1: Windows CI ENOTEMPTY in sync.test.ts After this PR added writeBridge() to syncGroup, the existing test "writes registry to groupDir when skipWrite is false" fails on windows-latest. LadybugDB's checkpoint thread briefly outlives closeBridgeDb, holding a Win32 lock on bridge.lbug; the test's fs.rmSync then fails with ENOTEMPTY. Switched the test cleanup to cleanupTempDir from test/helpers/test-db.ts which already tolerates EBUSY/EPERM/EACCES/ENOTEMPTY with bounded retries — same pattern used elsewhere for LadybugDB-touching tests. BLOCKER 2: Graph provider absolute-path bug extractProvidersGraph queried File.filePath from the LadybugDB graph but never stripped the repo root, so provider contract IDs ended up as include::/abs/path/foo.h while consumers emitted include::foo.h. These never matched through runExactMatch — silently producing 0 cross-links for any indexed C++ repo (the primary use case). Now passes repoPath into extractProvidersGraph and applies path.relative(); rows that resolve outside repoPath (stale absolute paths from another machine, system headers somehow indexed) are dropped instead of polluting the registry. MEDIUM: `../` relative includes produce spurious noise `#include "../foo.h"` is almost always intra-repo, but the suffix index can never match a `..`-prefixed path so it became a consumer contract no provider could satisfy. Now skipped before matching; covers both forward-slash and backslash forms. MEDIUM: writeBridge error in sync.ts propagates uncaught contracts.json is the canonical source of truth and was just written successfully when writeBridge runs. A bridge-only failure (disk full, schema error, permission denied) shouldn't mask the registry. Wrapped writeBridge in try/catch with a logger.warn surfacing the path and recovery instructions. Tests added: - extractProvidersGraph repo-relative ID generation (stub Cypher executor returns absolute paths) - extractProvidersGraph drops rows whose path resolves outside repo - `../foo.h` forward-slash skip - `..\foo.h` backslash-form skip Skipped findings: - canExtract() removal (#5, low): canExtract is part of the ContractExtractor interface; every other extractor implements the same `return true` shape. Removing it from IncludeExtractor would break the interface contract — keeping for consistency. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(group): close PR #1156 Codex adversarial findings Two HIGH findings from the Codex adversarial review on feat/group-include-extractor: 1. Default-on extraction silently changes existing groups (BLOCKER) DEFAULT_DETECT.includes was true, so any pre-existing group.yaml that omits the new field would gain a wave of include::* contracts on the next sync after upgrade. Flipped to false (opt-in). The integration test already declares includes: true explicitly so it survives unchanged; the unit extractor tests bypass parseGroupConfig entirely; the sync test uses extractorOverride. Only config-parser needed regression tests covering omitted/explicit/false variants. 2. IncludeExtractor scans outside the indexed file universe (BLOCKER) The extractor was running glob('**/*', { ignore: STANDARD_IGNORES }) twice with a hand-rolled 9-pattern list, no .gitignore/.gitnexusignore honoring, and no max-file-size cap. That meant File: contracts could appear for files ingestion would never index, producing cross-links group impact cannot fan out to (silent false-negatives). Refactored to a single discoverIndexableFiles() helper that mirrors walkRepositoryPaths exactly: createIgnoreFilter + getMaxFileSizeBytes, one discovery pass shared by provider and consumer paths. Dropped STANDARD_IGNORES and SOURCE_GLOB entirely. third_party and 3rdparty (the C/C++ vendored-deps conventions) were in the local ignore list but not in the canonical DEFAULT_IGNORE_LIST used by ingestion. Folded both into the canonical set rather than keep a parallel list — the whole point of the Codex finding is that two file-discovery implementations drift. Single source of truth. Tests: 5 new regression tests for the discovery alignment (.gitignore, .gitnexusignore, max-file-size on both provider and consumer paths) plus 4 for the opt-in default. All 30 include-extractor tests + the 494-test group suite + ignore-service tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(review): apply autofix feedback ce-code-review surfaced 6 safe_auto findings on commit a9936a9b: - T1 (testing, P2): the sync.ts:174 gate was untested with includes:false. Added a sync-level test mirroring the existing thrift-off pattern at sync.test.ts:545, asserting zero include contracts when the gate is disabled in a real syncGroup call. - T3 (testing, P3): third_party and 3rdparty entries in DEFAULT_IGNORE_LIST had no regression test. Added both to ignore-service.test.ts's dependency-directories it.each block. - M1 (maintainability, P3): discoverIndexableFiles JSDoc lacked a fork-warning relative to walkRepositoryPaths. Added a MAINTENANCE note explaining why the duplication is tolerated and the contract the two implementations must keep. - M2 (maintainability, P3): thrift-extractor still hand-rolls its ignore array with no signal that DEFAULT_IGNORE_LIST additions silently do not apply there. Added TODO(#1156-followup) comments above both call sites. - M3 (maintainability, P3): SOURCE_EXTENSIONS duplicated the four HEADER_EXTENSIONS entries with no expressed subset relationship. Spread HEADER_EXTENSIONS into SOURCE_EXTENSIONS so future header- extension additions propagate. - C1+T4 (correctness+testing, P3, cross-reviewer corroborated): discoverIndexableFiles swallowed all fs.stat errors silently, including EACCES/EMFILE/EIO. Narrowed the catch to ENOENT (the documented benign glob/stat race) and added a logger.warn for any other code so operators can spot permission/resource issues. All 629 tests pass; typecheck + prettier clean. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(group): use retryRename in writeContractRegistry to absorb Windows EPERM `storage.ts:62` used raw `fsp.rename` for the contracts.json atomic swap. On Windows, AV scanners and concurrent renames briefly hold the destination handle between rename calls, surfacing as EPERM/EBUSY. The `insecure-tempfile.test.ts > concurrent writes do not collide` test was flaking with `EPERM: operation not permitted, rename` on windows-latest CI. `bridge-db.ts` already has a battle-tested `retryRename(src, dst, 3)` helper used at six call sites for exactly this pattern. Reusing it here keeps the Windows-rename policy single-source-of-truth across the group package. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(group): drop macro-style #include from consumer contracts Tree-sitter's `(_) @import.source` wildcard matches the identifier node of `#include PLATFORM_HEADER`, so the cleaned value `PLATFORM_HEADER` slipped past the system-header / `..` filters and was emitted as a permanently orphaned consumer contract (no file is named after a macro identifier, so no provider can ever match). Add a shape guard that skips cleaned values lacking both a path separator and an extension dot, plus regression tests for single and multi-macro files. Also document `IncludeExtractor.canExtract()` as unused by sync.ts (gated via `config.detect.includes` instead) and kept solely for ContractExtractor interface uniformity. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: HuangWenjie Co-authored-by: Gergő Magyar Co-authored-by: Claude Opus 4.7 (1M context) --- gitnexus/src/config/ignore-service.ts | 2 + gitnexus/src/core/group/config-parser.ts | 20 +- .../group/extractors/include-extractor.ts | 610 ++++++++++++++++++ .../group/extractors/manifest-extractor.ts | 10 + .../core/group/extractors/thrift-extractor.ts | 8 + gitnexus/src/core/group/matching.ts | 2 + gitnexus/src/core/group/storage.ts | 9 +- gitnexus/src/core/group/sync.ts | 36 ++ gitnexus/src/core/group/types.ts | 3 +- .../group/include-extractor-sync.test.ts | 195 ++++++ .../test/unit/group/config-parser.test.ts | 56 ++ .../test/unit/group/include-extractor.test.ts | 563 ++++++++++++++++ gitnexus/test/unit/group/sync.test.ts | 49 +- gitnexus/test/unit/ignore-service.test.ts | 2 + 14 files changed, 1561 insertions(+), 4 deletions(-) create mode 100644 gitnexus/src/core/group/extractors/include-extractor.ts create mode 100644 gitnexus/test/integration/group/include-extractor-sync.test.ts create mode 100644 gitnexus/test/unit/group/include-extractor.test.ts diff --git a/gitnexus/src/config/ignore-service.ts b/gitnexus/src/config/ignore-service.ts index ce1fda913..2c3eebe3b 100644 --- a/gitnexus/src/config/ignore-service.ts +++ b/gitnexus/src/config/ignore-service.ts @@ -25,6 +25,8 @@ const DEFAULT_IGNORE_LIST = new Set([ 'bower_components', 'jspm_packages', 'vendor', // PHP/Go + 'third_party', // C/C++ (Google-style vendored dependencies) + '3rdparty', // C/C++ (alternate spelling, also Qt convention) // 'packages' removed - commonly used for monorepo source code (lerna, pnpm, yarn workspaces) 'venv', '.venv', diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index 73a9021b9..29c868171 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -4,9 +4,26 @@ import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -const VALID_CONTRACT_TYPES: ContractType[] = ['http', 'grpc', 'thrift', 'topic', 'lib', 'custom']; +const VALID_CONTRACT_TYPES: ContractType[] = [ + 'http', + 'grpc', + 'thrift', + 'topic', + 'lib', + 'custom', + 'include', +]; const VALID_ROLES: ContractRole[] = ['provider', 'consumer']; +// Defaults matter for backward compatibility: any group.yaml that omits a +// `detect.` key inherits its value from this constant. Adding a new +// extractor that defaults to `true` silently changes the behavior of every +// existing group on the next sync. New extractors must default to `false` +// (opt-in) so operators consciously enable them via group.yaml. +// +// `includes`: opt-in. The C/C++ IncludeExtractor (PR #1156) ships disabled by +// default; enable with `detect.includes: true` for groups containing C/C++ +// repos that need cross-repo header tracking. const DEFAULT_DETECT = { http: true, grpc: true, @@ -14,6 +31,7 @@ const DEFAULT_DETECT = { topics: true, shared_libs: true, embedding_fallback: true, + includes: false, workspace_deps: false, }; diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts new file mode 100644 index 000000000..7bbfd61ed --- /dev/null +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -0,0 +1,610 @@ +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import { glob } from 'glob'; +import Parser from 'tree-sitter'; +import C from 'tree-sitter-c'; +import Cpp from 'tree-sitter-cpp'; +import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; +import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafe } from './fs-utils.js'; +import { buildSuffixIndex, type SuffixIndex } from '../../ingestion/import-resolvers/utils.js'; +import { createIgnoreFilter } from '../../../config/ignore-service.js'; +import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { logger } from '../../logger.js'; + +/** + * Cross-repo C/C++ `#include` dependency extractor. + * + * **Provider side:** registers every `.h/.hpp/.hxx/.hh` file in the repo + * as a provider contract with `include::`. + * + * **Consumer side:** parses all C/C++ source/header files for `#include "…"` + * directives, attempts suffix-based resolution against the repo's own file + * list (reusing the same algorithm as the single-repo ingestion pipeline), + * and emits unresolved include paths as consumer contracts. + * + * Matching: a consumer's `include::map/base/dice_map_view.h` in repo A + * matches a provider's `include::map/base/dice_map_view.h` in repo B via + * exact contract-id equality in `runExactMatch`. + */ + +// ---------- constants ---------- + +const HEADER_EXTENSIONS = new Set(['.h', '.hpp', '.hxx', '.hh']); + +// Source = headers (provider-eligible) ∪ implementation files (.c/.cpp/.cc/.cxx). +// Spread keeps the subset relationship explicit so a future contributor adding +// a new header extension to HEADER_EXTENSIONS does not have to remember to +// also add it here. +const SOURCE_EXTENSIONS = new Set([...HEADER_EXTENSIONS, '.c', '.cpp', '.cc', '.cxx']); + +const INCLUDE_QUERY_SRC = '(preproc_include path: (_) @import.source) @import'; + +/** + * Well-known C/C++ standard library headers that can appear in `#include "…"` + * form (some projects use quotes for system headers). + */ +const SYSTEM_HEADERS = new Set([ + // C standard + 'assert.h', + 'complex.h', + 'ctype.h', + 'errno.h', + 'fenv.h', + 'float.h', + 'inttypes.h', + 'iso646.h', + 'limits.h', + 'locale.h', + 'math.h', + 'setjmp.h', + 'signal.h', + 'stdalign.h', + 'stdarg.h', + 'stdatomic.h', + 'stdbool.h', + 'stddef.h', + 'stdint.h', + 'stdio.h', + 'stdlib.h', + 'stdnoreturn.h', + 'string.h', + 'tgmath.h', + 'threads.h', + 'time.h', + 'uchar.h', + 'wchar.h', + 'wctype.h', + // C++ standard (extensionless) + 'algorithm', + 'any', + 'array', + 'atomic', + 'barrier', + 'bit', + 'bitset', + 'cassert', + 'cctype', + 'cerrno', + 'cfenv', + 'cfloat', + 'charconv', + 'chrono', + 'cinttypes', + 'climits', + 'clocale', + 'cmath', + 'codecvt', + 'compare', + 'complex', + 'concepts', + 'condition_variable', + 'coroutine', + 'csetjmp', + 'csignal', + 'cstdarg', + 'cstddef', + 'cstdint', + 'cstdio', + 'cstdlib', + 'cstring', + 'ctime', + 'cuchar', + 'cwchar', + 'cwctype', + 'deque', + 'exception', + 'execution', + 'expected', + 'filesystem', + 'format', + 'forward_list', + 'fstream', + 'functional', + 'future', + 'generator', + 'initializer_list', + 'iomanip', + 'ios', + 'iosfwd', + 'iostream', + 'istream', + 'iterator', + 'latch', + 'limits', + 'list', + 'locale', + 'map', + 'mdspan', + 'memory', + 'memory_resource', + 'mutex', + 'new', + 'numbers', + 'numeric', + 'optional', + 'ostream', + 'print', + 'queue', + 'random', + 'ranges', + 'ratio', + 'regex', + 'scoped_allocator', + 'semaphore', + 'set', + 'shared_mutex', + 'source_location', + 'span', + 'spanstream', + 'sstream', + 'stack', + 'stacktrace', + 'stdexcept', + 'stdfloat', + 'stop_token', + 'streambuf', + 'string', + 'string_view', + 'strstream', + 'syncstream', + 'system_error', + 'thread', + 'tuple', + 'type_traits', + 'typeindex', + 'typeinfo', + 'unordered_map', + 'unordered_set', + 'utility', + 'valarray', + 'variant', + 'vector', + 'version', +]); + +/** Path prefixes that indicate system/kernel headers. */ +const SYSTEM_PATH_PREFIXES = [ + 'sys/', + 'net/', + 'netinet/', + 'arpa/', + 'linux/', + 'asm/', + 'bits/', + 'gnu/', + 'mach/', + 'machine/', + 'xlocale/', +]; + +/** Regex fallback for files that exceed tree-sitter's 32 KB parse limit. */ +const INCLUDE_REGEX = /^[ \t]*#\s*include\s*"([^"]+)"/gm; + +// ---------- helpers ---------- + +/** + * Normalize an include path to a canonical lowercase forward-slash form. + * + * IMPORTANT — case-folding caveat (PR #1156 review finding #3): + * Header paths are lowercased so consumer `#include "Foo/Bar.h"` and + * provider file `Foo/Bar.h` normalize to the same contract-id. This is + * the right trade-off on case-insensitive filesystems (macOS, Windows) + * but on case-sensitive Linux filesystems two distinct headers `Foo.h` + * and `foo.h` in the same repo will collide onto the same provider + * contract-id; only one survives `dedupe()`. The gain (reliable + * cross-platform matching) outweighs the cost (extremely rare header + * casing collisions inside a single repo). + */ +function normalizeIncludePath(raw: string): string { + return raw.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase(); +} + +/** + * Strip C/C++ block comments from a source blob. Used only by the + * regex-fallback path to avoid emitting consumer contracts for + * commented-out #include directives. Line comments (`// …`) cannot hide + * #include directives because the regex anchors on start-of-line. + * See PR #1156 review finding #5. + */ +function stripBlockComments(src: string): string { + return src.replace(/\/\*[\s\S]*?\*\//g, ''); +} + +function isAngleBracketInclude(rawNodeText: string): boolean { + const trimmed = rawNodeText.trim(); + return trimmed.startsWith('<') && trimmed.endsWith('>'); +} + +function isSystemHeader(cleanedPath: string): boolean { + // Check well-known standard headers + if (SYSTEM_HEADERS.has(cleanedPath)) return true; + // Check system path prefixes + const lower = cleanedPath.toLowerCase(); + return SYSTEM_PATH_PREFIXES.some((prefix) => lower.startsWith(prefix)); +} + +function isHeaderFile(filePath: string): boolean { + return HEADER_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + +function getLanguageForFile(filePath: string): unknown | null { + const ext = path.extname(filePath).toLowerCase(); + switch (ext) { + case '.c': + case '.h': + return C; + case '.cpp': + case '.cc': + case '.cxx': + case '.hpp': + case '.hxx': + case '.hh': + return Cpp; + default: + return null; + } +} + +/** + * Check whether an include path resolves to a file inside the local repo. + * + * Uses *exact full-path* matching on the suffix index — we never accept a + * truncated suffix match. For `#include "foo/bar.h"` this checks: + * (a) a file whose path ends with the full `foo/bar.h` + * (b) if the include omitted the extension, a file whose path ends with + * the include + one of the C/C++ header extensions + * + * Returns `true` when a local file matches — caller should suppress the + * cross-repo consumer contract. + * + * See PR #1156 review finding #4 (suffixResolve ambiguity). + */ +function isLocalInclude(cleaned: string, suffixIndex: SuffixIndex): boolean { + const candidates = [cleaned]; + if (!/\.[a-zA-Z0-9]+$/.test(cleaned)) { + for (const ext of ['.h', '.hpp', '.hxx', '.hh']) candidates.push(cleaned + ext); + } + for (const c of candidates) { + if (suffixIndex.get(c) || suffixIndex.getInsensitive(c)) return true; + } + return false; +} + +// ---------- main class ---------- + +export class IncludeExtractor implements ContractExtractor { + type = 'include' as const; + + /** + * Always returns `true`. NOT called by `sync.ts`, which gates extraction via + * `config.detect.includes` instead (see `sync.ts:174`). Kept solely to satisfy + * the `ContractExtractor` interface so the type stays uniform across extractors. + */ + async canExtract(_repo: RepoHandle): Promise { + return true; + } + + async extract( + dbExecutor: CypherExecutor | null, + repoPath: string, + _repo: RepoHandle, + ): Promise { + // 1. Build the local file list using the same discovery as ingestion + // (createIgnoreFilter + getMaxFileSizeBytes). This guarantees the + // universe of provider/consumer paths matches the universe of File + // nodes in the LadybugDB graph — so no cross-link points at a UID + // that group impact cannot fan out to. + // (PR #1156 Codex follow-up: discovery aligned with ingestion.) + const allFiles = await this.discoverIndexableFiles(repoPath); + const normalizedFiles = allFiles.map((f) => f.replace(/\\/g, '/')); + const suffixIndex = buildSuffixIndex(normalizedFiles, allFiles); + + // 2. Provider: register all header files + const providers = await this.extractProviders(dbExecutor, repoPath, allFiles); + + // 3. Consumer: filter the shared discovery list for source extensions + // and parse #include directives in those files. + const sourceFiles = allFiles.filter((f) => + SOURCE_EXTENSIONS.has(path.extname(f).toLowerCase()), + ); + const consumers = await this.extractConsumers(repoPath, sourceFiles, suffixIndex); + + return this.dedupe([...providers, ...consumers]); + } + + /** + * Discover repo-relative file paths using exactly the same rules the + * ingestion pipeline uses (`walkRepositoryPaths` in + * `gitnexus/src/core/ingestion/filesystem-walker.ts`): + * - `createIgnoreFilter` honors `.gitignore`, `.gitnexusignore`, the + * hardcoded ignore list, and `.gitnexusignore` last-match-wins + * negation. + * - `getMaxFileSizeBytes()` drops files larger than the cap so we + * never emit `File:` UIDs for files ingestion would skip. + * + * Uses sequential stat — there is no `READ_CONCURRENCY` batching here + * because group sync runs at startup-time, not the ingestion hot path, + * and parallelism gains are not worth the import-graph weight. + * + * MAINTENANCE: if `walkRepositoryPaths` changes its glob options, ignore + * filter shape, or size-cap logic, mirror those changes here. The two + * implementations exist because the consumers need different return + * shapes (string[] vs ScannedFile[]) and different concurrency, but + * they MUST agree on which files are reachable — that is what makes + * `File:` UIDs in cross-links correspond to graph File nodes. + */ + private async discoverIndexableFiles(repoPath: string): Promise { + const ignoreFilter = await createIgnoreFilter(repoPath); + const maxFileSizeBytes = getMaxFileSizeBytes(); + + const candidates = await glob('**/*', { + cwd: repoPath, + nodir: true, + dot: false, + ignore: ignoreFilter, + }); + + const survivors: string[] = []; + for (const rel of candidates) { + try { + const stat = await fs.stat(path.join(repoPath, rel)); + if (stat.size > maxFileSizeBytes) continue; + survivors.push(rel); + } catch (err) { + // ENOENT is the documented benign race (glob enumerated a file + // that was deleted before we stat'd it — same race + // walkRepositoryPaths absorbs via Promise.allSettled). Anything + // else (EACCES, EMFILE, EIO) deserves a warning so an operator + // can spot a permission/resource problem instead of silently + // shipping fewer contracts than expected. + const code = (err as NodeJS.ErrnoException | undefined)?.code; + if (code !== 'ENOENT') { + logger.warn( + { err: (err as Error).message, file: rel, repoPath }, + '⚠️ IncludeExtractor: stat failed during discovery; skipping file', + ); + } + } + } + return survivors; + } + + // ---------- provider extraction ---------- + + private async extractProviders( + dbExecutor: CypherExecutor | null, + repoPath: string, + allFiles: string[], + ): Promise { + // Strategy A: graph-assisted + if (dbExecutor) { + const graphProviders = await this.extractProvidersGraph(dbExecutor, repoPath); + if (graphProviders.length > 0) return graphProviders; + } + // Strategy B: filesystem fallback + return this.extractProvidersFallback(repoPath, allFiles); + } + + private async extractProvidersGraph( + db: CypherExecutor, + repoPath: string, + ): Promise { + try { + const rows = await db( + `MATCH (f:File) + WHERE f.filePath =~ '.*\\\\.(h|hpp|hxx|hh)$' + RETURN f.filePath AS filePath, f.id AS fileId`, + ); + // gitnexus analyze stores absolute paths in the File.filePath column. + // Provider contract IDs MUST be repo-relative — otherwise the consumer + // emits `include::map/base/view.h` and the provider emits + // `include::/abs/path/to/repo/map/base/view.h`, which never match + // through runExactMatch and the cross-link silently disappears. + // (PR #1156 follow-up review: graph provider absolute-path bug.) + const normalizedRepoPath = path.resolve(repoPath); + const out: ExtractedContract[] = []; + for (const r of rows) { + if (typeof r.filePath !== 'string' || !r.filePath) continue; + const absolute = r.filePath as string; + const rel = path.relative(normalizedRepoPath, absolute); + // Skip rows that resolve outside the repo (e.g., system headers + // somehow indexed, or stale absolute paths from a different machine). + // path.relative returns a `..`-prefixed path or an absolute path + // when the target is outside the base — both are wrong for our IDs. + if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) continue; + const normalizedRel = rel.replace(/\\/g, '/'); + out.push({ + contractId: `include::${normalizeIncludePath(normalizedRel)}`, + type: 'include' as const, + role: 'provider' as const, + symbolUid: String(r.fileId ?? ''), + symbolRef: { filePath: normalizedRel, name: path.basename(normalizedRel) }, + symbolName: path.basename(normalizedRel), + confidence: 1.0, + meta: { source: 'graph' }, + }); + } + return out; + } catch { + return []; + } + } + + private extractProvidersFallback(_repoPath: string, allFiles: string[]): ExtractedContract[] { + return allFiles + .filter((f) => isHeaderFile(f)) + .map((f) => { + const filePath = f.replace(/\\/g, '/'); + return { + contractId: `include::${normalizeIncludePath(filePath)}`, + type: 'include' as const, + role: 'provider' as const, + symbolUid: `File:${filePath}`, + symbolRef: { filePath, name: path.basename(filePath) }, + symbolName: path.basename(filePath), + confidence: 0.95, + meta: { source: 'filesystem' }, + }; + }); + } + + // ---------- consumer extraction ---------- + + private async extractConsumers( + repoPath: string, + sourceFiles: string[], + suffixIndex: SuffixIndex, + ): Promise { + const parser = new Parser(); + const out: ExtractedContract[] = []; + // Compile the include query once per grammar to avoid re-compilation per file + const queryCache = new Map(); + + for (const rel of sourceFiles) { + const lang = getLanguageForFile(rel); + if (!lang) continue; + + const content = readSafe(repoPath, rel); + if (!content) continue; + + let query = queryCache.get(lang); + if (!query) { + try { + query = new Parser.Query(lang, INCLUDE_QUERY_SRC); + queryCache.set(lang, query); + } catch { + continue; + } + } + + // Collect raw include paths: tree-sitter first, regex fallback for large files. + // `extractionSource` is stamped on each emitted consumer contract so + // regex-fallback contracts stay auditable post-hoc (PR #1156 review finding #6). + let rawIncludes: string[]; + let extractionSource: 'tree_sitter' | 'regex_fallback'; + try { + parser.setLanguage(lang); + const tree = parser.parse(content); + let matches: Parser.QueryMatch[]; + try { + matches = query.matches(tree.rootNode); + } catch { + matches = []; + } + rawIncludes = []; + extractionSource = 'tree_sitter'; + for (const match of matches) { + const sourceNode = match.captures.find((c) => c.name === 'import.source'); + if (!sourceNode) continue; + const rawText = sourceNode.node.text; + if (isAngleBracketInclude(rawText)) continue; + const cleaned = rawText.replace(/['"<>]/g, ''); + if (cleaned && cleaned.length <= 2048) rawIncludes.push(cleaned); + } + } catch { + // tree-sitter failed (e.g. file > 32 KB) — fall back to regex. + // Strip block comments first so we don't emit a consumer contract + // for a commented-out #include (PR #1156 review finding #5). + rawIncludes = []; + extractionSource = 'regex_fallback'; + const scanTarget = stripBlockComments(content); + INCLUDE_REGEX.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = INCLUDE_REGEX.exec(scanTarget)) !== null) { + if (m[1] && m[1].length <= 2048) rawIncludes.push(m[1]); + } + } + + for (const cleaned of rawIncludes) { + // Filter: skip known system headers and system path prefixes + if (isSystemHeader(cleaned)) continue; + + // Skip relative-up includes: `#include "../include/foo.h"` is + // almost always an intra-repo reference. The suffix index is built + // from repo-relative paths, so isLocalInclude can never match + // `../foo.h`, and emitting it as a consumer contract just pollutes + // the registry with an entry no provider can ever satisfy. + // (PR #1156 follow-up review: `../` relative includes produce + // spurious consumer contracts.) + if (cleaned.startsWith('../') || cleaned.startsWith('..\\')) continue; + + // Skip macro-style includes: `#include PLATFORM_HEADER` parses as an + // identifier under tree-sitter's `(_) @import.source` wildcard. The + // identifier text passes the strip/clean step unchanged, so without + // this guard we would emit `include::platform_header` as a consumer + // contract — and no provider in any repo will ever expose a contract + // for a macro identifier (no file is named `PLATFORM_HEADER`). The + // contract would sit permanently orphaned in the registry. Real + // header references always contain a path separator (`/`, `\`) or an + // extension dot (`foo.h`), so an absent both is a reliable signal we + // are looking at a macro identifier. (PR #1156 follow-up review: + // macro includes emit orphaned consumer contracts.) + if (!/[./\\]/.test(cleaned)) continue; + + // Local resolution (PR #1156 review finding #4): only accept an + // exact-suffix match on the *full* include path. The generic + // suffixResolve() iterates all truncated suffixes, which would + // silently suppress a cross-repo `#include "map/base/view.h"` + // when the local repo has any `internal/view.h` — a realistic + // false-negative in large C++ codebases. Here we only resolve + // locally if a file path ends with the complete include string + // (optionally re-appending one of the C/C++ header extensions + // when the include already omits it). + if (isLocalInclude(cleaned, suffixIndex)) continue; + + // Unresolved: emit as consumer contract + const normalizedRel = rel.replace(/\\/g, '/'); + out.push({ + contractId: `include::${normalizeIncludePath(cleaned)}`, + type: 'include' as const, + role: 'consumer' as const, + symbolUid: `File:${normalizedRel}`, + symbolRef: { filePath: normalizedRel, name: cleaned }, + symbolName: cleaned, + confidence: 0.85, + meta: { + source: extractionSource, + includePath: cleaned, + }, + }); + } + } + + return out; + } + + // ---------- deduplication ---------- + + private dedupe(items: ExtractedContract[]): ExtractedContract[] { + const seen = new Set(); + const out: ExtractedContract[] = []; + for (const c of items) { + const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; + if (seen.has(k)) continue; + seen.add(k); + out.push(c); + } + return out; + } +} diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 2af3db595..f4d0f77cf 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -274,6 +274,14 @@ export class ManifestExtractor { LIMIT 1`, { contract: link.contract }, ); + } else if (link.type === 'include') { + rows = await executor( + `MATCH (f:File) WHERE f.filePath = $contract + RETURN f.id AS uid, f.name AS name, f.filePath AS filePath + ORDER BY f.filePath ASC + LIMIT 1`, + { contract: link.contract }, + ); } else if (link.type === 'custom') { // Workspace extractors produce qualified contracts like "mathlex::Expression". // Graph nodes store the unqualified symbol name ("Expression"), so strip @@ -358,6 +366,8 @@ export class ManifestExtractor { return `lib::${contract}`; case 'custom': return `custom::${contract}`; + case 'include': + return `include::${contract}`; default: { const _exhaustive: never = type; throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`); diff --git a/gitnexus/src/core/group/extractors/thrift-extractor.ts b/gitnexus/src/core/group/extractors/thrift-extractor.ts index cfd8fef02..709968790 100644 --- a/gitnexus/src/core/group/extractors/thrift-extractor.ts +++ b/gitnexus/src/core/group/extractors/thrift-extractor.ts @@ -217,6 +217,10 @@ export async function buildThriftContext(repoPath: string): Promise(); @@ -290,6 +294,10 @@ export class ThriftExtractor implements ContractExtractor { cwd: repoPath, absolute: false, nodir: true, + // TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter + // (the canonical ingestion ignore filter, like include-extractor.ts now uses). + // New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g. + // third_party, 3rdparty added in commit a9936a9b) silently do not apply here. ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'], }); diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index 3431f8ddf..0b27655c6 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -107,6 +107,8 @@ export function normalizeContractId(id: string): string { return `topic::${rest.trim().toLowerCase()}`; case 'lib': return `lib::${rest.toLowerCase()}`; + case 'include': + return `include::${rest.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase()}`; default: return id; } diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index cc3dbfdc9..bc08fd7f9 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -4,6 +4,7 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { randomBytes } from 'node:crypto'; import type { ContractRegistry } from './types.js'; +import { retryRename } from './bridge-db.js'; /** * Build an unpredictable suffix for atomic-write tmp files. Replaces the @@ -59,7 +60,13 @@ export async function writeContractRegistry( } finally { await handle.close(); } - await fsp.rename(tmpPath, targetPath); + // retryRename absorbs the documented Windows EPERM/EBUSY/EACCES race that + // fires when AV scanners or another concurrent rename briefly hold the + // destination handle between rename calls. Same helper bridge-db.ts uses + // (lines 304, 583, 587, 595, 605, 677) for the bridge.lbug atomic swap — + // single source of truth for the Windows-rename pattern across the group + // package. + await retryRename(tmpPath, targetPath); } export async function readContractRegistry(groupDir: string): Promise { diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 7ed065131..cd64fdf8c 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -8,12 +8,14 @@ import { HttpRouteExtractor } from './extractors/http-route-extractor.js'; import { GrpcExtractor } from './extractors/grpc-extractor.js'; import { ThriftExtractor } from './extractors/thrift-extractor.js'; import { TopicExtractor } from './extractors/topic-extractor.js'; +import { IncludeExtractor } from './extractors/include-extractor.js'; import { ManifestExtractor } from './extractors/manifest-extractor.js'; import { discoverWorkspaceLinks } from './extractors/workspace-extractor.js'; import { buildProviderIndex, runExactMatch, runWildcardMatch } from './matching.js'; import { detectServiceBoundaries, assignService } from './service-boundary-detector.js'; import type { CypherExecutor } from './contract-extractor.js'; import { writeContractRegistry } from './storage.js'; +import { writeBridge } from './bridge-db.js'; import type { ContractRegistry } from './types.js'; import { logger } from '../logger.js'; @@ -100,6 +102,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis const grpcEx = new GrpcExtractor(); const thriftEx = new ThriftExtractor(); const topicEx = new TopicExtractor(); + const includeEx = new IncludeExtractor(); dbExecutors = new Map(); const openPoolIds: string[] = []; @@ -168,6 +171,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis } } + if (config.detect.includes) { + const extracted = await includeEx.extract(executor, handle.repoPath, handle); + for (const c of extracted) { + autoContracts.push({ + ...c, + repo: groupPath, + service: assignService(c.symbolRef.filePath, boundaries), + }); + } + } + const metaPath = path.join(handle.storagePath, 'meta.json'); try { const raw = await fs.readFile(metaPath, 'utf-8'); @@ -270,6 +284,28 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis if (opts?.groupDir && !opts.skipWrite) { await writeContractRegistry(opts.groupDir, registry); + // writeBridge failure (disk full, schema error, permission denied) must + // not mask the registry — contracts.json was just written successfully + // and is the canonical source of truth. A stale or absent bridge + // degrades impact queries to empty results, which is recoverable on + // the next sync. Surface the failure as a warning so operators can + // act, but do not propagate it. + // (PR #1156 follow-up review: writeBridge error in sync.ts propagates + // uncaught.) + try { + await writeBridge(opts.groupDir, { + contracts: allContracts, + crossLinks, + repoSnapshots, + missingRepos, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn( + { err: msg, groupDir: opts.groupDir }, + '⚠️ writeBridge failed; contracts.json is intact but bridge.lbug is stale. Re-run `gitnexus group sync` to retry.', + ); + } } return { diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index 7d0a14251..8e43ff78f 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,4 +1,4 @@ -export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom'; +export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include'; export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding'; export type ContractRole = 'provider' | 'consumer'; @@ -28,6 +28,7 @@ export interface DetectConfig { topics: boolean; shared_libs: boolean; embedding_fallback: boolean; + includes: boolean; workspace_deps: boolean; } diff --git a/gitnexus/test/integration/group/include-extractor-sync.test.ts b/gitnexus/test/integration/group/include-extractor-sync.test.ts new file mode 100644 index 000000000..908664bdc --- /dev/null +++ b/gitnexus/test/integration/group/include-extractor-sync.test.ts @@ -0,0 +1,195 @@ +/** + * Integration test: IncludeExtractor output → group matching → bridge DB. + * + * Covers PR #1156 review finding #7: verifies that the full runtime path + * (IncludeExtractor → StoredContract → runExactMatch → CrossLinks → writeBridge) + * stays wired up. A regression in either normalizeContractId or the include + * branch of ManifestExtractor.resolveSymbol would produce 0 cross-links and + * fail this test. + */ +import { describe, it, expect } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { parseGroupConfig } from '../../../src/core/group/config-parser.js'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import type { StoredContract } from '../../../src/core/group/types.js'; +import { IncludeExtractor } from '../../../src/core/group/extractors/include-extractor.js'; +import { normalizeContractId } from '../../../src/core/group/matching.js'; + +const GROUP_YAML = [ + 'version: 1', + 'name: include-test-group', + 'description: "IncludeExtractor integration test"', + '', + 'repos:', + ' app/provider: include-provider', + ' app/consumer: include-consumer', + '', + 'links: []', + 'packages: {}', + '', + 'detect:', + ' http: false', + ' grpc: false', + ' topics: false', + ' shared_libs: false', + ' includes: true', + ' embedding_fallback: false', + '', + 'matching:', + ' bm25_threshold: 0.7', + ' embedding_threshold: 0.65', + ' max_candidates_per_step: 3', +].join('\n'); + +describe('IncludeExtractor → syncGroup integration (finding #7)', () => { + it('produces a CrossLink when provider and consumer emit the same include contract-id', async () => { + const config = parseGroupConfig(GROUP_YAML); + + // Mock the IncludeExtractor output directly — a header provider in one + // repo and a quoted #include consumer in the other, both normalized to + // the same include::map/base/view.h contract-id. + const mockContracts: StoredContract[] = [ + { + contractId: 'include::map/base/view.h', + type: 'include', + role: 'provider', + symbolUid: 'File:map/base/view.h', + symbolRef: { filePath: 'map/base/view.h', name: 'view.h' }, + symbolName: 'view.h', + confidence: 0.95, + meta: { source: 'filesystem' }, + repo: 'app/provider', + }, + { + contractId: 'include::map/base/view.h', + type: 'include', + role: 'consumer', + symbolUid: 'File:src/controller.cpp', + symbolRef: { filePath: 'src/controller.cpp', name: 'map/base/view.h' }, + symbolName: 'map/base/view.h', + confidence: 0.85, + meta: { source: 'tree_sitter', includePath: 'map/base/view.h' }, + repo: 'app/consumer', + }, + ]; + + const result = await syncGroup(config, { + extractorOverride: async () => mockContracts, + skipWrite: true, + }); + + const includeLinks = result.crossLinks.filter((l) => l.type === 'include'); + expect(includeLinks.length).toBeGreaterThanOrEqual(1); + + const link = includeLinks[0]; + expect(link.contractId).toBe('include::map/base/view.h'); + expect(link.matchType).toBe('exact'); + expect(link.from.repo).toBe('app/consumer'); + expect(link.to.repo).toBe('app/provider'); + }); + + it('normalizes mixed-case / backslash include paths to the same contract-id end-to-end', async () => { + const config = parseGroupConfig(GROUP_YAML); + + // Provider writes the canonical form; consumer's include has mixed case + // and a backslash. After normalizeContractId they must still match. + const providerId = 'include::map/base/view.h'; + const rawConsumerId = 'include::Map\\Base\\View.h'; + + // Sanity — normalizeContractId must collapse them. + expect(normalizeContractId(rawConsumerId)).toBe(providerId); + + const mockContracts: StoredContract[] = [ + { + contractId: providerId, + type: 'include', + role: 'provider', + symbolUid: 'File:map/base/view.h', + symbolRef: { filePath: 'map/base/view.h', name: 'view.h' }, + symbolName: 'view.h', + confidence: 0.95, + meta: { source: 'filesystem' }, + repo: 'app/provider', + }, + { + contractId: rawConsumerId, + type: 'include', + role: 'consumer', + symbolUid: 'File:src/controller.cpp', + symbolRef: { filePath: 'src/controller.cpp', name: 'Map/Base/View.h' }, + symbolName: 'Map/Base/View.h', + confidence: 0.85, + meta: { source: 'tree_sitter', includePath: 'Map\\Base\\View.h' }, + repo: 'app/consumer', + }, + ]; + + const result = await syncGroup(config, { + extractorOverride: async () => mockContracts, + skipWrite: true, + }); + + const includeLinks = result.crossLinks.filter((l) => l.type === 'include'); + expect(includeLinks.length).toBeGreaterThanOrEqual(1); + }); + + it('round-trip: extractor output from two real temp repos produces matching contract-ids', async () => { + // Drives the extractor directly (no `syncGroup`) against two on-disk + // fixture repos, then hands the StoredContract-shaped output to + // syncGroup via extractorOverride. This exercises the real extraction + // code + the matching pipeline together. + const providerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-int-provider-')); + const consumerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-int-consumer-')); + try { + fs.mkdirSync(path.join(providerDir, 'shared/api'), { recursive: true }); + fs.writeFileSync( + path.join(providerDir, 'shared/api/client.h'), + '#pragma once\nstruct Client {};', + ); + fs.mkdirSync(path.join(consumerDir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(consumerDir, 'src/main.cpp'), + '#include "shared/api/client.h"\nint main(){return 0;}', + ); + + const extractor = new IncludeExtractor(); + const providerOutput = await extractor.extract(null, providerDir, { + id: 'provider', + path: 'app/provider', + repoPath: providerDir, + storagePath: path.join(providerDir, '.gitnexus'), + }); + const consumerOutput = await extractor.extract(null, consumerDir, { + id: 'consumer', + path: 'app/consumer', + repoPath: consumerDir, + storagePath: path.join(consumerDir, '.gitnexus'), + }); + + const stored: StoredContract[] = [ + ...providerOutput + .filter((c) => c.role === 'provider') + .map((c) => ({ ...c, repo: 'app/provider' })), + ...consumerOutput + .filter((c) => c.role === 'consumer') + .map((c) => ({ ...c, repo: 'app/consumer' })), + ]; + + const config = parseGroupConfig(GROUP_YAML); + const result = await syncGroup(config, { + extractorOverride: async () => stored, + skipWrite: true, + }); + + const includeLinks = result.crossLinks.filter((l) => l.type === 'include'); + expect(includeLinks.length).toBeGreaterThanOrEqual(1); + expect(includeLinks[0].contractId).toBe('include::shared/api/client.h'); + expect(includeLinks[0].matchType).toBe('exact'); + } finally { + fs.rmSync(providerDir, { recursive: true, force: true }); + fs.rmSync(consumerDir, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index e1d3b540f..22bb2ad26 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -75,6 +75,62 @@ repos: expect(config.detect.thrift).toBe(true); }); + // PR #1156 Codex follow-up: include extraction is opt-in. Existing + // group.yaml files that do not declare `detect.includes` must not gain + // a wave of new include::* contracts on the next sync after upgrade. + describe('detect.includes opt-in default', () => { + it('defaults includes detection to false when detect block omits it', () => { + const minimal = ` +version: 1 +name: test +repos: + app: my-app +`; + const config = parseGroupConfig(minimal); + expect(config.detect.includes).toBe(false); + }); + + it('defaults includes detection to false when detect block is present but omits the key', () => { + const yaml = ` +version: 1 +name: test +repos: + app: my-app +detect: + http: true + grpc: false +`; + const config = parseGroupConfig(yaml); + expect(config.detect.includes).toBe(false); + }); + + it('honors explicit detect.includes: true (opt-in works)', () => { + const yaml = ` +version: 1 +name: test +repos: + app: my-app +detect: + includes: true +`; + const config = parseGroupConfig(yaml); + expect(config.detect.includes).toBe(true); + }); + + it('honors explicit detect.includes: false', () => { + const yaml = ` +version: 1 +name: test +repos: + app: my-app +detect: + includes: false +`; + const config = parseGroupConfig(yaml); + expect(config.detect.includes).toBe(false); + }); + }); + it('parses thrift manifest links', () => { const yaml = ` version: 1 diff --git a/gitnexus/test/unit/group/include-extractor.test.ts b/gitnexus/test/unit/group/include-extractor.test.ts new file mode 100644 index 000000000..3956cd5f2 --- /dev/null +++ b/gitnexus/test/unit/group/include-extractor.test.ts @@ -0,0 +1,563 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { IncludeExtractor } from '../../../src/core/group/extractors/include-extractor.js'; +import type { RepoHandle } from '../../../src/core/group/types.js'; +import { normalizeContractId } from '../../../src/core/group/matching.js'; + +describe('IncludeExtractor', () => { + let tmpDir: string; + let extractor: IncludeExtractor; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-')); + extractor = new IncludeExtractor(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function writeFile(relPath: string, content: string): void { + const full = path.join(tmpDir, relPath); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } + + const makeRepo = (repoPath: string): RepoHandle => ({ + id: 'test-repo', + path: 'test/app', + repoPath, + storagePath: path.join(repoPath, '.gitnexus'), + }); + + // ---- Provider detection ---- + + describe('provider extraction', () => { + it('registers .h files as providers', async () => { + writeFile('map/base/view.h', '#pragma once\nclass View {};'); + writeFile('map/base/types.h', '#pragma once\nstruct Point {};'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers).toHaveLength(2); + const ids = providers.map((p) => p.contractId).sort(); + expect(ids).toEqual(['include::map/base/types.h', 'include::map/base/view.h']); + expect(providers[0].type).toBe('include'); + expect(providers[0].confidence).toBeGreaterThanOrEqual(0.95); + }); + + it('registers .hpp files as providers', async () => { + writeFile('utils/helper.hpp', '#pragma once\ntemplate T id(T x) { return x; }'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers).toHaveLength(1); + expect(providers[0].contractId).toBe('include::utils/helper.hpp'); + }); + + it('does not register .cpp files as providers', async () => { + writeFile('src/main.cpp', 'int main() { return 0; }'); + writeFile('src/utils.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers).toHaveLength(1); + expect(providers[0].contractId).toBe('include::src/utils.h'); + }); + }); + + // ---- Consumer detection ---- + + describe('consumer extraction', () => { + it('emits unresolved includes as consumers', async () => { + writeFile( + 'src/main.cpp', + `#include "map/base/view.h" +#include "map/base/types.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(2); + const ids = consumers.map((c) => c.contractId).sort(); + expect(ids).toEqual(['include::map/base/types.h', 'include::map/base/view.h']); + expect(consumers[0].type).toBe('include'); + expect(consumers[0].confidence).toBe(0.85); + }); + + it('skips locally resolved includes', async () => { + writeFile('map/base/view.h', '#pragma once\nclass View {};'); + writeFile( + 'src/main.cpp', + `#include "map/base/view.h" +#include "external/lib.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Only external/lib.h should be a consumer — map/base/view.h resolves locally + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::external/lib.h'); + }); + + it('skips angle-bracket includes', async () => { + writeFile( + 'src/main.cpp', + `#include +#include +#include "app/interface.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::app/interface.h'); + }); + + it('skips well-known system headers in quotes', async () => { + writeFile( + 'src/main.cpp', + `#include "stdio.h" +#include "stdlib.h" +#include "app/config.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::app/config.h'); + }); + + it('skips system path prefixes', async () => { + writeFile( + 'src/main.c', + `#include "sys/types.h" +#include "linux/input.h" +#include "mylib/types.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::mylib/types.h'); + }); + }); + + // ---- Cross-repo matching scenario ---- + + describe('cross-repo matching', () => { + it('provider and consumer produce matching contractIds', async () => { + // Simulate provider repo (header-only) + const providerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-provider-')); + const providerFile = path.join(providerDir, 'map/base/dice_map_view.h'); + fs.mkdirSync(path.dirname(providerFile), { recursive: true }); + fs.writeFileSync(providerFile, '#pragma once\nclass DiceMapView {};'); + + // Simulate consumer repo + const consumerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-consumer-')); + const consumerFile = path.join(consumerDir, 'src/controller.cpp'); + fs.mkdirSync(path.dirname(consumerFile), { recursive: true }); + fs.writeFileSync(consumerFile, '#include "map/base/dice_map_view.h"\nvoid init() {}'); + + try { + const providerContracts = await extractor.extract(null, providerDir, makeRepo(providerDir)); + const consumerContracts = await extractor.extract(null, consumerDir, makeRepo(consumerDir)); + + const providers = providerContracts.filter((c) => c.role === 'provider'); + const consumers = consumerContracts.filter((c) => c.role === 'consumer'); + + expect(providers.length).toBeGreaterThanOrEqual(1); + expect(consumers.length).toBeGreaterThanOrEqual(1); + + const providerIds = new Set(providers.map((p) => normalizeContractId(p.contractId))); + const consumerIds = consumers.map((c) => normalizeContractId(c.contractId)); + + // The consumer's include path should match a provider's file path + expect(providerIds.has(consumerIds[0])).toBe(true); + } finally { + fs.rmSync(providerDir, { recursive: true, force: true }); + fs.rmSync(consumerDir, { recursive: true, force: true }); + } + }); + }); + + // ---- Review finding #4: suffixResolve ambiguity ---- + + describe('finding #4: suffix-ambiguity does not silently suppress cross-repo include', () => { + it('emits a cross-repo contract when the include path does not match any local file (even if a shorter suffix does)', async () => { + // local repo has `internal/api.h` but NOT `ext/api.h` + writeFile('internal/api.h', '#pragma once'); + writeFile( + 'src/main.cpp', + `#include "ext/api.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Previously suffixResolve would match `api.h` against `internal/api.h` + // and drop the cross-repo contract. After finding #4 fix, we only + // accept exact full-path matches — so `ext/api.h` must still be + // emitted as a consumer contract. + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::ext/api.h'); + }); + + it('still suppresses a local include when the FULL path matches', async () => { + writeFile('ext/api.h', '#pragma once'); + writeFile('src/main.cpp', '#include "ext/api.h"\nint main(){return 0;}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + + it('resolves locally when include omits extension and a matching .h exists', async () => { + writeFile('foo/bar.h', '#pragma once'); + writeFile('src/main.cpp', '#include "foo/bar"\nint main(){return 0;}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + }); + + // ---- Review finding #5: regex fallback must strip block comments ---- + + describe('finding #5: regex fallback ignores block-commented includes', () => { + it('does not emit a contract for an #include inside /* ... */', async () => { + // Force regex fallback by producing a file larger than tree-sitter's + // 32 KB hard cap. The include we care about lives inside a block + // comment that spans the file. + const filler = 'int dummy_' + 'x'.repeat(32) + ' = 0;\n'.repeat(1200); + const content = `/* + * Historical include, kept for reference only: + * #include "legacy/old-api.h" + */ +${filler} +#include "real/api.h" +int main(){return 0;}`; + writeFile('src/huge.cpp', content); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + const ids = consumers.map((c) => c.contractId); + + // The live include should appear; the commented-out one must NOT. + expect(ids).toContain('include::real/api.h'); + expect(ids).not.toContain('include::legacy/old-api.h'); + }); + }); + + // ---- Review finding #6: meta.source must reflect which extraction path ran ---- + + describe('finding #6: meta.source reflects extraction path', () => { + it('stamps `tree_sitter` on contracts produced via AST walking', async () => { + writeFile('src/main.cpp', '#include "app/small.h"\nint main(){return 0;}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect((consumers[0].meta as { source?: string } | undefined)?.source).toBe('tree_sitter'); + }); + + it('meta.source is one of the two documented values (tree_sitter | regex_fallback)', async () => { + // Regex fallback is a defensive branch that only fires if + // parser.setLanguage() or parser.parse() throws. In practice + // tree-sitter-c/cpp handles realistic inputs, so we only assert + // the meta.source contract: it is always present and always one of + // the two documented values. This guards against future regressions + // that might hard-code the wrong string. + writeFile('src/main.cpp', '#include "ext/whatever.h"\nint main(){return 0;}'); + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumer = contracts.find((c) => c.role === 'consumer'); + expect(consumer).toBeDefined(); + const src = (consumer?.meta as { source?: string } | undefined)?.source; + expect(['tree_sitter', 'regex_fallback']).toContain(src); + }); + }); + + // ---- Review finding #3: provider id collision on case-sensitive FS ---- + + describe('finding #3: case-folding is documented and deterministic', () => { + it('collapses `Foo.h` and `foo.h` onto the same provider contract-id (documented trade-off)', async () => { + writeFile('Foo.h', '#pragma once\n// Capital Foo'); + // On case-insensitive filesystems (macOS default) the second writeFile + // will overwrite the first, so we only create this when distinct files + // can coexist (case-sensitive FS, e.g. Linux CI). + try { + fs.writeFileSync(path.join(tmpDir, 'foo.h'), '#pragma once\n// lowercase foo'); + } catch { + // Ignore — some FS won't allow both names to coexist. + } + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + const ids = providers.map((p) => p.contractId); + + // Both files (if they coexist) must normalize to the same id. + // dedupe() keeps only one; caller code must be aware of this. + expect(ids).toContain('include::foo.h'); + // Never see a mixed-case contract-id leak out. + expect(ids.every((id) => id === id.toLowerCase())).toBe(true); + }); + }); + + // ---- Deduplication ---- + + describe('deduplication', () => { + it('deduplicates same include from multiple source files', async () => { + writeFile('src/a.cpp', '#include "ext/api.h"\nvoid a() {}'); + writeFile('src/b.cpp', '#include "ext/api.h"\nvoid b() {}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Both files include "ext/api.h" — each should produce a separate + // consumer contract (different symbolRef.filePath) + expect(consumers).toHaveLength(2); + const files = consumers.map((c) => c.symbolRef.filePath).sort(); + expect(files).toEqual(['src/a.cpp', 'src/b.cpp']); + }); + }); + + // ---- normalizeContractId ---- + + describe('normalizeContractId for include', () => { + it('lowercases the path', () => { + expect(normalizeContractId('include::Map/Base/Foo.h')).toBe('include::map/base/foo.h'); + }); + + it('normalizes backslashes', () => { + expect(normalizeContractId('include::map\\base\\foo.h')).toBe('include::map/base/foo.h'); + }); + + it('strips leading ./', () => { + expect(normalizeContractId('include::./foo.h')).toBe('include::foo.h'); + }); + + it('collapses consecutive slashes', () => { + expect(normalizeContractId('include::map//base///foo.h')).toBe('include::map/base/foo.h'); + }); + }); + + // ---- PR #1156 follow-up: `../` relative includes ---- + + describe('follow-up: `../` relative includes are skipped', () => { + it('does not emit a consumer contract for `#include "../foo.h"`', async () => { + // Producer: a header that exists locally but only via parent reference + writeFile('include/foo.h', '#pragma once'); + writeFile( + 'src/sub/main.cpp', + `#include "../../include/foo.h" +#include "real/cross_repo.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Only `real/cross_repo.h` should remain — the `..`-prefixed include + // is intra-repo noise that no provider can ever satisfy. + expect(consumers.map((c) => c.contractId)).toEqual(['include::real/cross_repo.h']); + }); + + it('skips backslash-form `..\\` for completeness', async () => { + writeFile( + 'src/main.cpp', + `#include "..\\\\sibling\\\\foo.h" +#include "remote/header.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + const ids = consumers.map((c) => c.contractId); + expect(ids).toContain('include::remote/header.h'); + expect(ids.some((id) => id.includes('..'))).toBe(false); + }); + }); + + // ---- PR #1156 follow-up: macro-style includes ---- + + describe('follow-up: macro-style #include emits no consumer contract', () => { + it('does not emit a consumer contract for `#include PLATFORM_HEADER` (no separator, no dot)', async () => { + // `#include PLATFORM_HEADER` parses under tree-sitter as an identifier + // node, slips past the existing system-header / `..` filters, and used + // to leak through as a permanently orphaned consumer contract because + // no file is ever named `PLATFORM_HEADER`. Verify the macro guard + // suppresses it while preserving the real cross-repo include. + writeFile( + 'src/main.cpp', + `#include PLATFORM_HEADER +#include "real/api.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers.map((c) => c.contractId)).toEqual(['include::real/api.h']); + }); + + it('skips multiple macro identifiers in the same translation unit', async () => { + writeFile( + 'src/cfg.cpp', + `#include CONFIG_HEADER +#include PLATFORM_HEADER +#include ASSERT_H_ +int main(){return 0;}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + }); + + // ---- PR #1156 follow-up: graph provider absolute paths ---- + + describe('follow-up: extractProvidersGraph strips repo root from absolute paths', () => { + it('produces repo-relative contract IDs when the graph returns absolute paths', async () => { + writeFile('map/base/view.h', '#pragma once\nclass View {};'); + writeFile('utils/types.hpp', '#pragma once'); + + // Stub the Cypher executor to return absolute paths the way + // gitnexus analyze actually persists them. + const absolute1 = path.join(tmpDir, 'map/base/view.h'); + const absolute2 = path.join(tmpDir, 'utils/types.hpp'); + const stubDb = async () => [ + { filePath: absolute1, fileId: 'File:abs:1' }, + { filePath: absolute2, fileId: 'File:abs:2' }, + ]; + + const contracts = await extractor.extract(stubDb, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + const ids = providers.map((p) => p.contractId).sort(); + expect(ids).toEqual(['include::map/base/view.h', 'include::utils/types.hpp']); + expect(providers.every((p) => p.meta?.source === 'graph')).toBe(true); + }); + + it('drops graph rows whose path resolves outside the repo root', async () => { + writeFile('local/header.h', '#pragma once'); + const absoluteLocal = path.join(tmpDir, 'local/header.h'); + const stubDb = async () => [ + { filePath: absoluteLocal, fileId: 'File:1' }, + // Stale absolute path from a different machine — must be skipped. + { filePath: '/some/other/repo/foreign.h', fileId: 'File:2' }, + ]; + + const contracts = await extractor.extract(stubDb, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers.map((p) => p.contractId)).toEqual(['include::local/header.h']); + }); + }); + + // ---- PR #1156 Codex follow-up: discovery aligned with ingestion ---- + + describe('follow-up: file discovery honors createIgnoreFilter and getMaxFileSizeBytes', () => { + it('does not emit a provider contract for a header excluded by .gitignore', async () => { + writeFile('.gitignore', 'vendor-headers/\n'); + writeFile('vendor-headers/blocked.h', '#pragma once'); + writeFile('src/wanted.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::src/wanted.h'); + expect(providerIds).not.toContain('include::vendor-headers/blocked.h'); + }); + + it('does not emit a provider contract for a header excluded by .gitnexusignore', async () => { + writeFile('.gitnexusignore', 'legacy/\n'); + writeFile('legacy/old.h', '#pragma once'); + writeFile('src/current.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::src/current.h'); + expect(providerIds).not.toContain('include::legacy/old.h'); + }); + + it('does not parse #include directives in a source file excluded by .gitignore', async () => { + // The ignored source file references a header that would otherwise be + // a cross-repo consumer. After alignment, the ignored file is invisible + // to the consumer scan — no consumer contract should appear. + writeFile('.gitignore', 'generated/\n'); + writeFile( + 'generated/auto.cpp', + `#include "remote/should_not_appear.h" +int auto_main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumerIds = contracts.filter((c) => c.role === 'consumer').map((c) => c.contractId); + + expect(consumerIds).not.toContain('include::remote/should_not_appear.h'); + }); + + it('skips a provider header whose size exceeds GITNEXUS_MAX_FILE_SIZE', async () => { + const previous = process.env.GITNEXUS_MAX_FILE_SIZE; + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; // 1 KB cap + try { + // 4 KB header — comfortably exceeds the cap. + const oversized = '#pragma once\n' + 'x'.repeat(4 * 1024); + writeFile('huge/big.h', oversized); + writeFile('small/tiny.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::small/tiny.h'); + expect(providerIds).not.toContain('include::huge/big.h'); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_MAX_FILE_SIZE; + else process.env.GITNEXUS_MAX_FILE_SIZE = previous; + } + }); + + it('skips parsing #include directives in source files exceeding GITNEXUS_MAX_FILE_SIZE', async () => { + const previous = process.env.GITNEXUS_MAX_FILE_SIZE; + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; + try { + const oversized = + '#include "remote/should_not_appear.h"\n' + + '// padding to push the file past 1 KB\n' + + 'x'.repeat(4 * 1024); + writeFile('big/main.cpp', oversized); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumerIds = contracts.filter((c) => c.role === 'consumer').map((c) => c.contractId); + + expect(consumerIds).not.toContain('include::remote/should_not_appear.h'); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_MAX_FILE_SIZE; + else process.env.GITNEXUS_MAX_FILE_SIZE = previous; + } + }); + }); +}); diff --git a/gitnexus/test/unit/group/sync.test.ts b/gitnexus/test/unit/group/sync.test.ts index 88bc3af2d..9f14db231 100644 --- a/gitnexus/test/unit/group/sync.test.ts +++ b/gitnexus/test/unit/group/sync.test.ts @@ -3,6 +3,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; import { syncGroup, stableRepoPoolId } from '../../../src/core/group/sync.js'; +import { cleanupTempDir } from '../../helpers/test-db.js'; import { _captureLogger } from '../../../src/core/logger.js'; import type { GroupConfig, @@ -583,6 +584,47 @@ service OrderService { } }); + it('does not extract include contracts during real sync when includes detection is disabled', async () => { + // PR #1156 Codex follow-up: ce-code-review T1 — verifies the gate at + // sync.ts:174 honors `detect.includes: false`. Mirrors the existing + // thrift-off pattern at sync.test.ts:545. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-sync-includes-off-')); + const storageDir = path.join(tmpDir, '.gitnexus'); + fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); + fs.mkdirSync(storageDir, { recursive: true }); + fs.writeFileSync(path.join(tmpDir, 'src', 'view.h'), '#pragma once\nclass View {};'); + + const config = makeConfig({ 'app/cpp-lib': 'cpp-lib-repo' }); + config.detect.http = false; + config.detect.grpc = false; + config.detect.thrift = false; + config.detect.topics = false; + config.detect.includes = false; + + const poolAdapter = await import('../../../src/core/lbug/pool-adapter.js'); + const initSpy = vi.spyOn(poolAdapter, 'initLbug').mockResolvedValue(undefined); + const closeSpy = vi.spyOn(poolAdapter, 'closeLbug').mockResolvedValue(undefined); + + try { + const result = await syncGroup(config, { + resolveRepoHandle: async (_name, groupPath) => ({ + id: 'cpp-lib-repo', + path: groupPath, + repoPath: tmpDir, + storagePath: storageDir, + }), + skipWrite: true, + }); + + expect(result.missingRepos).toHaveLength(0); + expect(result.contracts.filter((c) => c.type === 'include')).toHaveLength(0); + } finally { + initSpy.mockRestore(); + closeSpy.mockRestore(); + await cleanupTempDir(tmpDir); + } + }); + it('dedupes duplicate wildcard cross-links during sync', async () => { const config = makeConfig({ 'app/provider': 'provider-repo', 'app/consumer': 'consumer-repo' }); const provider: StoredContract = { @@ -689,7 +731,12 @@ service OrderService { expect(registry.version).toBe(1); expect(registry.contracts).toHaveLength(0); } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); + // syncGroup now writes bridge.lbug + WAL/shadow sidecars when + // skipWrite is false. On Windows, LadybugDB's checkpoint thread can + // briefly outlive closeBridgeDb, holding a Win32 lock on the file. + // cleanupTempDir tolerates the documented Windows-native lock codes + // (EBUSY/EPERM/EACCES/ENOTEMPTY) with bounded retries. + await cleanupTempDir(tmpDir); } }); diff --git a/gitnexus/test/unit/ignore-service.test.ts b/gitnexus/test/unit/ignore-service.test.ts index b4e5cdca1..1e1908137 100644 --- a/gitnexus/test/unit/ignore-service.test.ts +++ b/gitnexus/test/unit/ignore-service.test.ts @@ -28,6 +28,8 @@ describe('shouldIgnorePath', () => { it.each([ 'node_modules', 'vendor', + 'third_party', + '3rdparty', 'venv', '.venv', '__pycache__', From d91428ad9deba2d1363b12431a6ceae9f6ae1a76 Mon Sep 17 00:00:00 2001 From: Alex Macdonald-Smith Date: Sat, 9 May 2026 04:52:26 -0400 Subject: [PATCH 03/11] feat(cli): add `gitnexus publish` for opt-in understand-quickly registry (#1425) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cli): add `gitnexus publish` for opt-in understand-quickly registry Adds a small, opt-in command that fires a single `repository_dispatch` event at `looptech-ai/understand-quickly` to ask the registry for an instant resync of the current repo's entry. No graph file is uploaded; the registry pulls from raw.githubusercontent.com per the protocol at https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md. - Pure helpers (id parsing, payload construction, validation) live in `gitnexus-shared/src/integrations/understand-quickly.ts` so the package stays Node-free and the same logic is testable in isolation. - The CLI command lives in `gitnexus/src/cli/publish.ts`. Without `UNDERSTAND_QUICKLY_TOKEN` it is a no-op (exits 0 with one informational line); with the token it POSTs the dispatch and surfaces 204 / 401 / 404 / 5xx distinctly. - The id defaults to `/` parsed from the `origin` remote and can be overridden with `--id`. - Refuses to publish when no `.gitnexus/` index exists, with a `gitnexus analyze` hint. Tests: a new vitest unit covers the pure helpers (8 + 8 + 2 cases) and the no-token no-op path with a `fetch` spy that fails the test if the network is touched. README gets a one-paragraph "Publishing to understand-quickly" section near the existing CLI docs. * fix(uq-publish): address review blockers + high-severity items Addresses CodeQL polynomial-regex (HIGH), token-gate ordering, distinct 401/403/404/422 response branches, fetch timeout, expanded test coverage, tightened owner/repo validation, and non-GitHub remote rejection. See response thread on PR #1425 for the per-finding rationale. Signed-off-by: amacsmith * fix(publish): address Claude review on PR #1425 - AbortError → TimeoutError: AbortSignal.timeout() throws a DOMException with name 'TimeoutError', not Error{name:'AbortError'}. Match the pattern used in core/embeddings/http-client.ts so the user-facing "timed out after 15000ms" message actually fires. Update the regression test to throw a real DOMException — the previous fake was a false-green. - isValidOwnerRepo: forbid trailing hyphen in the owner segment. GitHub rejects this at account-creation time; allowing it here meant hand-typed --id values like 'my-org-/repo' would pass our regex and 422 from GitHub. - Add publish-command coverage to cli-index-help.test.ts (asserts on --id, --skip-git, the registry name, and the token env var) and cli-commands.test.ts (asserts publishCommand is exported as a function). Catches accidental command-registration deletion. --------- Signed-off-by: amacsmith Co-authored-by: Gergő Magyar --- README.md | 7 + gitnexus-shared/src/index.ts | 12 + .../src/integrations/understand-quickly.ts | 151 +++++++++ gitnexus/src/cli/index.ts | 12 + gitnexus/src/cli/publish.ts | 232 +++++++++++++ gitnexus/test/unit/cli-commands.test.ts | 10 + gitnexus/test/unit/cli-index-help.test.ts | 13 + gitnexus/test/unit/publish.test.ts | 316 ++++++++++++++++++ 8 files changed, 753 insertions(+) create mode 100644 gitnexus-shared/src/integrations/understand-quickly.ts create mode 100644 gitnexus/src/cli/publish.ts create mode 100644 gitnexus/test/unit/publish.test.ts diff --git a/README.md b/README.md index f5f3c5a88..6beadb63d 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ gitnexus clean --all --force # Delete all indexes gitnexus wiki [path] # Generate repository wiki from knowledge graph gitnexus wiki --model # Wiki with custom LLM model (default: gpt-4o-mini) gitnexus wiki --base-url # Wiki with custom LLM API base URL +gitnexus publish # Notify the understand-quickly registry (opt-in, see below) # Repository groups (multi-repo / monorepo service tracking) gitnexus group create # Create a repository group @@ -228,6 +229,12 @@ gitnexus group status # Check staleness of repos in a group If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. +#### Publishing to understand-quickly (opt-in) + +[`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) is a public registry of code-knowledge graphs that lists `gitnexus@1` as a first-class format. After registering your repo once (`npx @understand-quickly/cli add` or the [wizard](https://looptech-ai.github.io/understand-quickly/add.html)), `gitnexus publish` fires a single `repository_dispatch` event so the registry resyncs your entry on demand instead of waiting for the nightly job. + +It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained GitHub PAT with `Repository dispatches: write` on the registry repo. Nothing else happens; no graph file is uploaded. See the [protocol spec](https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md) for the full contract. + ### What Your AI Agent Gets **16 tools** exposed via MCP (11 per-repo + 5 group): diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index ea66c3855..faf136fe7 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -143,6 +143,18 @@ export type { ScopeTree } from './scope-resolution/scope-tree.js'; export { buildPositionIndex } from './scope-resolution/position-index.js'; export type { PositionIndex } from './scope-resolution/position-index.js'; +// Understand-Quickly registry integration (opt-in) +export { + UNDERSTAND_QUICKLY_DISPATCH_URL, + UNDERSTAND_QUICKLY_EVENT_TYPE, + UNDERSTAND_QUICKLY_TOKEN_ENV, + buildUqDispatchPayload, + isValidOwnerRepo, + parseOwnerRepoFromRemote, + stripGitSuffix, +} from './integrations/understand-quickly.js'; +export type { UqDispatchPayload } from './integrations/understand-quickly.js'; + // Shadow-mode diff + aggregation (RFC §6.3; Ring 2 SHARED #918) export { diffResolutions } from './scope-resolution/shadow/diff.js'; export type { diff --git a/gitnexus-shared/src/integrations/understand-quickly.ts b/gitnexus-shared/src/integrations/understand-quickly.ts new file mode 100644 index 000000000..f30e7461b --- /dev/null +++ b/gitnexus-shared/src/integrations/understand-quickly.ts @@ -0,0 +1,151 @@ +/** + * Understand-Quickly registry integration helpers. + * + * Pure, runtime-agnostic logic for opting in to publishing a GitNexus + * index to the [`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) + * registry. Lives in `gitnexus-shared` so both the Node CLI and any + * future browser-side surface can construct identical dispatch payloads. + * + * Network I/O lives in the CLI command (`gitnexus/src/cli/publish.ts`) + * to keep this module free of Node-only imports — see the comment at + * the top of `gitnexus-shared/src/graph/types.ts`. + * + * The protocol contract (single dispatch event, no graph upload) is + * documented at: + * https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md + */ + +/** + * URL of the registry repo's repository_dispatch endpoint. Hardcoded + * because the registry is the canonical home for this integration — + * users who want a private registry can fork and patch. + */ +export const UNDERSTAND_QUICKLY_DISPATCH_URL = + 'https://api.github.com/repos/looptech-ai/understand-quickly/dispatches'; + +/** + * Event type the registry's sync workflow listens for. + * See `looptech-ai/understand-quickly/.github/workflows/sync.yml`. + */ +export const UNDERSTAND_QUICKLY_EVENT_TYPE = 'sync-entry'; + +/** Environment variable that gates the dispatch. */ +export const UNDERSTAND_QUICKLY_TOKEN_ENV = 'UNDERSTAND_QUICKLY_TOKEN'; + +export interface UqDispatchPayload { + event_type: typeof UNDERSTAND_QUICKLY_EVENT_TYPE; + client_payload: { + /** `/` shape — must match the registered entry. */ + id: string; + }; +} + +/** + * Build the JSON body for the `repository_dispatch` ping. Pure — no + * env reads, no network. Validates that `id` looks like `owner/repo` + * (one slash, no whitespace, both halves non-empty) so a misconfigured + * caller fails loudly before the round-trip. + */ +export function buildUqDispatchPayload(id: string): UqDispatchPayload { + if (!isValidOwnerRepo(id)) { + throw new Error( + `[understand-quickly] expected id of the form "owner/repo", got "${id}". ` + + `The registry uses this string to look up your entry in registry.json — ` + + `it must match the GitHub owner/repo of the source code, not a local path.`, + ); + } + return { + event_type: UNDERSTAND_QUICKLY_EVENT_TYPE, + client_payload: { id }, + }; +} + +/** + * `owner/repo` validation. Conservative on purpose: GitHub's actual + * naming rules are looser, but we want to catch local paths + * (`/Users/...`), bare slugs (`my-repo`), and accidental whitespace. + * + * Matches GitHub's published slug rules: + * owner: starts with alnum, then alnum/hyphen only, must end with + * alnum (no trailing hyphen — GitHub rejects this at account + * creation, so a `my-org-/repo` input would otherwise pass us + * and 422 from GitHub). No underscore, no dot. Length cap 39. + * repo: any of alnum/dot/hyphen/underscore. Length cap 100. + */ +export function isValidOwnerRepo(id: string): boolean { + return /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?\/[A-Za-z0-9._-]{1,100}$/.test(id); +} + +/** + * Strip a single trailing `.git` (case-insensitive) and any trailing + * slashes from a URL-ish string. Bounded linear: each character is + * visited at most twice, no backtracking. + * + * Replaces `s.replace(/\.git\/*$/i, '').replace(/\/+$/, '')` which + * CodeQL's polynomial-regex check (codeql/js/polynomial-redos) flags as + * a worst-case O(n²) on adversarial input like "////.../x". + */ +export function stripGitSuffix(input: string): string { + let end = input.length; + // Trim trailing '/'. + while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--; + // Drop one trailing '.git' (case-insensitive). + if (end >= 4) { + const tail = input.slice(end - 4, end).toLowerCase(); + if (tail === '.git') end -= 4; + } + // Trim trailing '/' that may have sat between '.git' and the rest. + while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--; + return input.slice(0, end); +} + +/** + * Parse `owner/repo` out of a git remote URL. Mirrors the heuristic in + * `gitnexus/src/storage/git.ts:parseRepoNameFromUrl` but keeps both + * halves so we can build a registry id. Returns `null` on shapes we + * don't recognise. + * + * Examples: + * git@github.com:looptech-ai/understand-quickly.git + * https://github.com/looptech-ai/understand-quickly + * ssh://git@github.com/looptech-ai/understand-quickly.git + */ +export function parseOwnerRepoFromRemote(url: string | null | undefined): string | null { + if (!url) return null; + const trimmed = url.trim(); + if (!trimmed) return null; + // Strip a trailing `.git` (case-insensitive) and any trailing slashes + // so https://h/o/r and https://h/o/r.git collapse to the same id. + // Bounded-linear helper avoids the polynomial-regex CodeQL alert. + const stripped = stripGitSuffix(trimmed); + + // SCP-form SSH (`git@host:owner/repo`). Capture host so we can reject + // non-GitHub remotes — a GitLab origin like + // `https://gitlab.example.com/group/sub/project.git` would otherwise + // silently dispatch the wrong id (LOW 9). + const ssh = stripped.match(/^[^@]+@([^:]+):([^/]+)\/([^/]+)$/); + if (ssh) { + const host = ssh[1].toLowerCase(); + if (host !== 'github.com' && host !== 'www.github.com') return null; + return `${ssh[2]}/${ssh[3]}`; + } + + // URL forms (https://, ssh://, git://, file://) — last two path segments. + const url2 = stripped.match(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/([^/]+)\/(.+)$/); + if (url2) { + // Strip optional `userinfo@` (e.g. `ssh://git@github.com/...`). + const authority = url2[1]; + const atIdx = authority.lastIndexOf('@'); + const hostAndPort = atIdx >= 0 ? authority.slice(atIdx + 1) : authority; + // Strip `:port` suffix if present. + const colonIdx = hostAndPort.indexOf(':'); + const host = (colonIdx >= 0 ? hostAndPort.slice(0, colonIdx) : hostAndPort).toLowerCase(); + if (host !== 'github.com' && host !== 'www.github.com') return null; + const segments = url2[2].split('/').filter(Boolean); + if (segments.length >= 2) { + const [owner, repo] = segments.slice(-2); + return `${owner}/${repo}`; + } + } + return null; +} diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index b89b40db0..e4a455d40 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -160,6 +160,18 @@ program .description('Augment a search pattern with knowledge graph context (used by hooks)') .action(createLazyAction(() => import('./augment.js'), 'augmentCommand')); +program + .command('publish [path]') + .description( + 'Notify the understand-quickly registry that this repo has a fresh GitNexus index. ' + + 'Opt-in: requires UNDERSTAND_QUICKLY_TOKEN (fine-grained PAT with ' + + '`Repository dispatches: write` on looptech-ai/understand-quickly). ' + + 'No-op without the token. See https://github.com/looptech-ai/understand-quickly.', + ) + .option('--id ', 'Override the registry id (defaults to the origin remote)') + .option('--skip-git', 'Treat cwd as the repo root and skip parent git-root discovery') + .action(createLazyAction(() => import('./publish.js'), 'publishCommand')); + // ─── Direct Tool Commands (no MCP overhead) ──────────────────────── // These invoke LocalBackend directly for use in eval, scripts, and CI. diff --git a/gitnexus/src/cli/publish.ts b/gitnexus/src/cli/publish.ts new file mode 100644 index 000000000..8aedc9c35 --- /dev/null +++ b/gitnexus/src/cli/publish.ts @@ -0,0 +1,232 @@ +/** + * `gitnexus publish` — opt-in ping to the understand-quickly registry. + * + * Fires a single `repository_dispatch` event at + * `looptech-ai/understand-quickly` so the registry knows to refresh its + * entry for the current repo. Does NOT upload anything: per the + * understand-quickly protocol, the registry pulls the graph from a + * raw-GitHub URL the user controls. + * + * https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md + * + * Defaults: + * - Without `UNDERSTAND_QUICKLY_TOKEN` in the env, this is a no-op + * (prints one informational line, exit 0). Same shape as the + * `--publish` patterns in sibling tools. + * - With the token, fires the dispatch and reports the response code. + * + * The `id` is derived from the repo's `origin` remote unless the caller + * passes `--id ` explicitly. We deliberately do NOT auto-add + * the repo to the registry — registration is one-time and uses the + * `npx @understand-quickly/cli add` path documented in the protocol. + */ + +import path from 'path'; +import { + UNDERSTAND_QUICKLY_DISPATCH_URL, + UNDERSTAND_QUICKLY_TOKEN_ENV, + buildUqDispatchPayload, + isValidOwnerRepo, + parseOwnerRepoFromRemote, +} from 'gitnexus-shared'; +import { getGitRoot, getRemoteOriginUrl, getCurrentCommit } from '../storage/git.js'; +import { hasIndex } from '../storage/repo-manager.js'; +import { cliInfo, cliError } from './cli-message.js'; + +export interface PublishOptions { + /** Override the auto-derived `owner/repo` id. */ + id?: string; + /** Treat the cwd as the repo root (skip git-root walk). */ + skipGit?: boolean; +} + +const REGISTER_HINT = + 'Register your repo once with: npx @understand-quickly/cli add\n' + + 'Or use the wizard: https://looptech-ai.github.io/understand-quickly/add.html'; + +/** + * Hard cap on the dispatch fetch to keep CI publish steps from stalling + * for the OS TCP timeout (~2 min) when api.github.com is unreachable. + * Matches the pattern used in `src/core/embeddings/http-client.ts`. + */ +const DISPATCH_TIMEOUT_MS = 15_000; + +export const publishCommand = async ( + inputPath?: string, + options: PublishOptions = {}, +): Promise => { + // ── 0. Token gate FIRST — guarantees true no-op without the token. ── + // The README, CLI --help, and PR body all promise "exit 0 without + // UNDERSTAND_QUICKLY_TOKEN". Doing the index/repo-root checks before + // the token gate would make those promises false for users who haven't + // run `gitnexus analyze` yet but want to verify the command is wired. + const token = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + if (!token) { + cliInfo( + `[understand-quickly] ${UNDERSTAND_QUICKLY_TOKEN_ENV} is not set — skipping dispatch.\n` + + `Set it to a fine-grained PAT with "Repository dispatches: write" on ` + + `looptech-ai/understand-quickly to enable instant resync.\n` + + `(Without the token, the registry's nightly sync still picks up your entry.)`, + { skipped: 'no-token' }, + ); + return; + } + + // ── 1. Resolve the repo root (same precedence as `analyze`) ────────── + let repoPath: string; + if (inputPath) { + repoPath = path.resolve(inputPath); + } else if (options.skipGit) { + repoPath = path.resolve(process.cwd()); + } else { + const gitRoot = getGitRoot(process.cwd()); + if (!gitRoot) { + cliError( + '[understand-quickly] not inside a git repository.\n' + + 'Run from a repo, or pass --skip-git to publish from the current directory.', + ); + process.exitCode = 1; + return; + } + repoPath = gitRoot; + } + + // ── 2. Confirm a GitNexus index exists ─────────────────────────────── + // Publishing without an index is almost always a mistake — the + // registry's nightly sync would fetch a stale or missing graph file + // and mark the entry `missing`. Refuse loudly with a fix-it hint. + if (!(await hasIndex(repoPath))) { + cliError( + `[understand-quickly] no GitNexus index found at ${repoPath}/.gitnexus.\n` + + 'Run `gitnexus analyze` first, then re-run `gitnexus publish`.', + ); + process.exitCode = 1; + return; + } + + // ── 3. Derive the registry id ───────────────────────────────────────── + const id = + options.id ?? parseOwnerRepoFromRemote(getRemoteOriginUrl(repoPath) ?? undefined) ?? null; + if (!id || !isValidOwnerRepo(id)) { + cliError( + `[understand-quickly] could not derive a registry id from this repo.\n` + + `Pass --id explicitly (e.g. --id looptech-ai/${path.basename(repoPath)}).\n` + + REGISTER_HINT, + ); + process.exitCode = 1; + return; + } + + // ── 4. Fire the dispatch ───────────────────────────────────────────── + const payload = buildUqDispatchPayload(id); + let response: Response; + try { + response = await fetch(UNDERSTAND_QUICKLY_DISPATCH_URL, { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + 'User-Agent': 'gitnexus-cli', + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(DISPATCH_TIMEOUT_MS), + }); + } catch (err) { + // `AbortSignal.timeout()` throws a `DOMException` with `name === + // 'TimeoutError'` on Node 18.14+ (and on browsers/Bun). It is NOT + // a plain `AbortError`. Match the pattern used in + // gitnexus/src/core/embeddings/http-client.ts so the user sees the + // targeted "timed out" message instead of a generic "operation + // was aborted". + const isTimeout = err instanceof DOMException && err.name === 'TimeoutError'; + if (isTimeout) { + cliError( + `[understand-quickly] dispatch timed out after ${DISPATCH_TIMEOUT_MS}ms. ` + + `Check network access to api.github.com and retry.`, + { id }, + ); + } else { + const msg = err instanceof Error ? err.message : String(err); + cliError(`[understand-quickly] dispatch network error: ${msg}`, { id }); + } + process.exitCode = 1; + return; + } + + // GitHub returns 204 on success. Distinct branches for 401/403/404/422 + // so users debug without checking the docs. + if (response.status === 204) { + await response.body?.cancel().catch(() => {}); + // `getCurrentCommit` is only meaningful in the success path — moving + // it inside this branch removes a wasted child-process spawn on every + // error response (LOW 7). + const commit = getCurrentCommit(repoPath); + cliInfo( + `[understand-quickly] dispatched sync-entry for ${id}` + + (commit ? ` @ ${commit.slice(0, 7)}` : '') + + '.\n' + + `Note: a 204 only confirms GitHub accepted the dispatch. Whether the ` + + `registry workflow finds an entry for "${id}" is logged at ` + + `https://github.com/looptech-ai/understand-quickly/actions/workflows/sync.yml`, + { id, commit, status: response.status }, + ); + return; + } + + if (response.status === 401) { + cliError( + `[understand-quickly] dispatch returned 401 — the ${UNDERSTAND_QUICKLY_TOKEN_ENV} value is invalid or expired.\n` + + `Regenerate a fine-grained PAT at https://github.com/settings/personal-access-tokens ` + + `with Repository access scoped to looptech-ai/understand-quickly and the ` + + `"Repository dispatches: write" permission, then retry.`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + if (response.status === 403) { + cliError( + `[understand-quickly] dispatch returned 403 — the token authenticated but ` + + `lacks the "Repository dispatches: write" permission on ` + + `looptech-ai/understand-quickly. Edit the PAT scopes and retry.`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + if (response.status === 404) { + cliError( + `[understand-quickly] dispatch returned 404 — the token cannot reach ` + + `looptech-ai/understand-quickly. Verify the PAT has Repository access to ` + + `that exact repo (not just your own org).`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + if (response.status === 422) { + // Malformed event_type / client_payload — a code bug in this CLI, + // not a user mistake. Surface so we get bug reports. + const body422 = await response.text().catch(() => ''); + cliError( + `[understand-quickly] dispatch returned 422 (this is a CLI bug; please report).\n` + + `Body: ${body422 || '(empty)'}`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + // 5xx and anything else → bubble the body so the user has something to act on. + const body = await response.text().catch(() => ''); + cliError( + `[understand-quickly] dispatch failed with HTTP ${response.status}: ${body || '(empty body)'}`, + { id, status: response.status }, + ); + process.exitCode = 1; +}; diff --git a/gitnexus/test/unit/cli-commands.test.ts b/gitnexus/test/unit/cli-commands.test.ts index a059a37bc..a42afb25c 100644 --- a/gitnexus/test/unit/cli-commands.test.ts +++ b/gitnexus/test/unit/cli-commands.test.ts @@ -10,6 +10,9 @@ vi.mock('../../src/cli/mcp.js', () => ({ vi.mock('../../src/cli/setup.js', () => ({ setupCommand: vi.fn(), })); +vi.mock('../../src/cli/publish.js', () => ({ + publishCommand: vi.fn(), +})); describe('CLI commands', () => { describe('version', () => { @@ -84,4 +87,11 @@ describe('CLI commands', () => { expect(typeof setupCommand).toBe('function'); }); }); + + describe('publishCommand', () => { + it('is a function', async () => { + const { publishCommand } = await import('../../src/cli/publish.js'); + expect(typeof publishCommand).toBe('function'); + }); + }); }); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index 59109c8d9..889a7473f 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -63,4 +63,17 @@ describe('CLI help surface', () => { expect(result.stdout).toContain('--model '); expect(result.stdout).toContain('--gist'); }); + + it('publish help names the registry, the token env var, and the opt-out behaviour', () => { + const result = runHelp('publish'); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('--id '); + expect(result.stdout).toContain('--skip-git'); + // Discoverability contract: a contributor scanning `--help` must see + // (a) which registry this dispatches to, and (b) the env var that + // gates the opt-in. Both are part of the no-token contract. + expect(result.stdout).toContain('understand-quickly'); + expect(result.stdout).toContain('UNDERSTAND_QUICKLY_TOKEN'); + }); }); diff --git a/gitnexus/test/unit/publish.test.ts b/gitnexus/test/unit/publish.test.ts new file mode 100644 index 000000000..2ba767594 --- /dev/null +++ b/gitnexus/test/unit/publish.test.ts @@ -0,0 +1,316 @@ +import { afterEach, beforeEach, describe, expect, it, test, vi } from 'vitest'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { performance } from 'node:perf_hooks'; +import { + buildUqDispatchPayload, + isValidOwnerRepo, + parseOwnerRepoFromRemote, + stripGitSuffix, + UNDERSTAND_QUICKLY_TOKEN_ENV, +} from 'gitnexus-shared'; + +describe('understand-quickly helpers (gitnexus-shared)', () => { + describe('isValidOwnerRepo', () => { + it.each([ + ['looptech-ai/understand-quickly', true], + ['abhigyanpatwari/GitNexus', true], + // LOW 8: GitHub user/org slugs are alnum/hyphen only — no underscore. + ['Some_Org/Some.Repo-2', false], + ['', false], + ['just-a-name', false], + ['/Users/me/code/repo', false], + ['org/with spaces', false], + ['org//double', false], + // LOW 8 additions: + ['some_org/repo', false], // underscore in owner — invalid + ['-org/repo', false], // leading hyphen — invalid + ['org-/repo', false], // trailing hyphen — GitHub rejects at account creation; we mirror that here + ['org/repo_with_underscore', true], + ['org/.dotfile', true], // repos may start with dot + ])('returns %s for %j', (id, expected) => { + expect(isValidOwnerRepo(id as string)).toBe(expected); + }); + }); + + describe('stripGitSuffix (BLOCKER 1 — ReDoS-safe)', () => { + it.each([ + ['https://github.com/o/r.git', 'https://github.com/o/r'], + ['https://github.com/o/r.git/', 'https://github.com/o/r'], + ['https://github.com/o/r/', 'https://github.com/o/r'], + ['https://github.com/o/r', 'https://github.com/o/r'], + ['https://github.com/o/r.GIT', 'https://github.com/o/r'], + ['https://github.com/o/r//', 'https://github.com/o/r'], + ['', ''], + ['/', ''], + ])('strips %j -> %j', (input, expected) => { + expect(stripGitSuffix(input)).toBe(expected); + }); + + test('linear time on adversarial trailing slashes (regression for ReDoS)', () => { + const adversarial = 'https://github.com/o/r' + '/'.repeat(10_000); + const start = performance.now(); + const result = stripGitSuffix(adversarial); + const elapsed = performance.now() - start; + expect(result).toBe('https://github.com/o/r'); + expect(elapsed).toBeLessThan(50); // generous; should be sub-millisecond + }); + + test('parseOwnerRepoFromRemote terminates quickly on adversarial input', () => { + const adversarial = 'https://github.com/o/r.git' + '/'.repeat(10_000); + const start = performance.now(); + const result = parseOwnerRepoFromRemote(adversarial); + const elapsed = performance.now() - start; + expect(result).toBe('o/r'); + expect(elapsed).toBeLessThan(50); + }); + }); + + describe('parseOwnerRepoFromRemote', () => { + it.each([ + ['git@github.com:looptech-ai/understand-quickly.git', 'looptech-ai/understand-quickly'], + ['https://github.com/looptech-ai/understand-quickly', 'looptech-ai/understand-quickly'], + ['https://github.com/looptech-ai/understand-quickly.git', 'looptech-ai/understand-quickly'], + ['ssh://git@github.com/abhigyanpatwari/GitNexus.git', 'abhigyanpatwari/GitNexus'], + ])('parses %s -> %s', (url, expected) => { + expect(parseOwnerRepoFromRemote(url)).toBe(expected); + }); + + // LOW 9: non-GitHub remotes must be rejected — a wrong id is worse + // than no id, since the user can always pass --id explicitly. + it.each([ + ['https://gitlab.example.com/group/sub/project.git'], + ['git@gitlab.example.com:group/sub/project.git'], + ['https://bitbucket.org/team/repo.git'], + ])('returns null for non-GitHub host %j', (input) => { + expect(parseOwnerRepoFromRemote(input)).toBeNull(); + }); + + it.each([null, undefined, '', ' ', 'not-a-url', 'https://github.com/'])( + 'returns null for %j', + (input) => { + expect(parseOwnerRepoFromRemote(input as string | null | undefined)).toBeNull(); + }, + ); + }); + + describe('buildUqDispatchPayload', () => { + it('wraps the id in the registry-expected event shape', () => { + expect(buildUqDispatchPayload('looptech-ai/understand-quickly')).toEqual({ + event_type: 'sync-entry', + client_payload: { id: 'looptech-ai/understand-quickly' }, + }); + }); + + it('throws on a malformed id rather than building an invalid payload', () => { + expect(() => buildUqDispatchPayload('just-a-name')).toThrow(/owner\/repo/); + expect(() => buildUqDispatchPayload('/Users/me/repo')).toThrow(/owner\/repo/); + }); + }); +}); + +describe('publishCommand (no-token no-op)', () => { + let tempDir: string; + let originalToken: string | undefined; + let exitCodeBefore: number | undefined; + + beforeEach(async () => { + vi.resetModules(); + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publish-test-')); + // Simulate an existing index so hasIndex() returns true. + await fs.mkdir(path.join(tempDir, '.gitnexus'), { recursive: true }); + await fs.writeFile( + path.join(tempDir, '.gitnexus', 'meta.json'), + JSON.stringify({ repoPath: tempDir, lastCommit: '', indexedAt: '' }), + 'utf-8', + ); + originalToken = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + delete process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + exitCodeBefore = process.exitCode; + process.exitCode = 0; + }); + + afterEach(async () => { + if (originalToken !== undefined) { + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = originalToken; + } else { + delete process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + } + process.exitCode = exitCodeBefore; + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it('exits 0 without firing a network call when the token is unset', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(() => { + throw new Error('publishCommand should NOT call fetch when the token is missing'); + }); + + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { id: 'looptech-ai/understand-quickly', skipGit: true }); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(process.exitCode ?? 0).toBe(0); + fetchSpy.mockRestore(); + }); + + it('exits 0 with no token even when no index/repo exists (BLOCKER 2)', async () => { + // Per the README, CLI --help, and PR body: without a token, the + // command must be a no-op even if the repo lacks `.gitnexus/`. + const noIndexDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publish-noidx-')); + try { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(() => { + throw new Error('publishCommand should NOT call fetch when the token is missing'); + }); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(noIndexDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(process.exitCode ?? 0).toBe(0); + fetchSpy.mockRestore(); + } finally { + await fs.rm(noIndexDir, { recursive: true, force: true }); + } + }); +}); + +describe('publishCommand response branches (MEDIUM 5)', () => { + let tempDir: string; + let originalToken: string | undefined; + let exitCodeBefore: number | undefined; + let fetchSpy: ReturnType; + + beforeEach(async () => { + vi.resetModules(); + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publish-resp-')); + await fs.mkdir(path.join(tempDir, '.gitnexus'), { recursive: true }); + await fs.writeFile( + path.join(tempDir, '.gitnexus', 'meta.json'), + JSON.stringify({ repoPath: tempDir, lastCommit: '', indexedAt: '' }), + 'utf-8', + ); + originalToken = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = 'pat_test'; + exitCodeBefore = process.exitCode; + process.exitCode = 0; + fetchSpy = vi.spyOn(globalThis, 'fetch'); + }); + + afterEach(async () => { + if (originalToken !== undefined) { + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = originalToken; + } else { + delete process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + } + process.exitCode = exitCodeBefore; + vi.restoreAllMocks(); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + function mockResponse(status: number, body = '') { + fetchSpy.mockResolvedValueOnce({ + status, + ok: status >= 200 && status < 300, + text: async () => body, + body: { cancel: async () => {} }, + headers: new Headers(), + } as unknown as Response); + } + + it('204 → exit 0 with success message', async () => { + mockResponse(204); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(process.exitCode ?? 0).toBe(0); + }); + + it('401 → exit 1 with PAT-invalid hint', async () => { + mockResponse(401, '{"message":"Bad credentials"}'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('403 → exit 1 with scope-missing hint', async () => { + mockResponse(403, '{"message":"Resource not accessible"}'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('404 → exit 1 with repo-access hint', async () => { + mockResponse(404, '{"message":"Not Found"}'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('5xx → exit 1 with raw body', async () => { + mockResponse(503, 'gateway timeout'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('network throw → exit 1', async () => { + fetchSpy.mockRejectedValueOnce(new Error('ECONNRESET')); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('TimeoutError (HIGH 4 — fetch timeout) → exit 1 with timed-out message', async () => { + // `AbortSignal.timeout()` throws a real `DOMException` with + // `name === 'TimeoutError'`. Faking it as `Error{name:'AbortError'}` + // (the previous shape of this test) hid a mismatch in publish.ts — + // the catch branch only matched 'AbortError' and the user-facing + // "timed out" message never fired in production. + const abort = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + fetchSpy.mockRejectedValueOnce(abort); + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + const written = errSpy.mock.calls.map((c) => String(c[0])).join(''); + expect(written).toMatch(/timed out/i); + errSpy.mockRestore(); + }); + + it('token never appears in any logged output', async () => { + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = 'pat_secret_value'; + mockResponse(401, ''); + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + const written = errSpy.mock.calls.map((c) => String(c[0])).join(''); + expect(written).not.toContain('pat_secret_value'); + errSpy.mockRestore(); + }); +}); From 3daf8c9984597265c799589fc6312cdb395d1505 Mon Sep 17 00:00:00 2001 From: Kareem Date: Sat, 9 May 2026 07:28:32 -0400 Subject: [PATCH 04/11] feat(extractors): strip Unreal Engine reflection macros before C++ parsing (#1439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(extractors): strip Unreal Engine reflection macros before C++ parsing Tree-sitter does not expand C preprocessor macros, so Unreal Engine reflection markers (UCLASS, UFUNCTION, UPROPERTY, MODULENAME_API, GENERATED_BODY, ...) are parsed verbatim. The result is mis-parsed UE class/function declarations: in 'class BRAWLUI_API UMyClass : public UObject', tree-sitter-cpp captures BRAWLUI_API as the class name, leaving the actual class without an entry in the graph. This patch adds an optional 'preprocessSource' hook to LanguageProvider and implements it for C++ via a new 'stripUeMacros' module. The transform is length-preserving (each elided byte becomes a space, newlines preserved) so byte offsets and line/column positions tree-sitter reports remain identical to the original file -- symbol locations in the graph stay accurate. A cheap detection guard short-circuits files that don't look like UE sources, so non-UE C++ codebases pay no cost (single regex test then bail). 27 unit tests cover the detection guard, length preservation across multiple UE samples, macro removal for UCLASS/UFUNCTION/UPROPERTY/USTRUCT/GENERATED_BODY/MODULE_API/DECLARE_*_DELEGATE/UE_DEPRECATED, false-positive guards (substring matches, balanced parens inside string literals, Qt macros left alone), and class-name extraction sanity. Full unit suite still passes (5337 tests, 0 regressions). Verified end-to-end against an Unreal Engine 5.7 game project (Brawl). * fix(extractors): address PR review findings on UE macro preprocessor Resolves three blocking issues raised by automated review: 1. Prettier format: ran prettier --write on call-processor.ts, heritage-processor.ts, import-processor.ts (the three sites where the cache-miss reparse hook insertion landed unformatted). 2. Byte-length contract narrowed: language-provider.ts docblock now states the contract precisely (UTF-16 .length + newline-position preservation, not UTF-8 byte length). Notes that startIndex byte offsets only match the original file when the elided range is pure ASCII -- which is the practical UE case (reflection macros and module-export tokens are ASCII-only). 3. Tree-sitter extraction tests added: new end-to-end tests parse the preprocessed source with tree-sitter-cpp and assert the captured class/struct name is the real UClass identifier (UMyClass, FMyData), never the MODULE_API export macro. Also asserts source positions (startPosition.row) survive the transform. Plus one moderate fix: 4. _API stripping is now scoped to UE files only. The HAS_UE_HINT guard previously included [A-Z]_API tokens, which would fire on non-UE codebases that use REST_API / HTTP_API / MY_LIB_API as constants or enum values, silently erasing them. The guard now requires a strong UE marker (UCLASS|UFUNCTION|UPROPERTY|USTRUCT|UENUM|UINTERFACE|GENERATED_BODY|UE_DEPRECATED|DECLARE_*_DELEGATE) to be present before any stripping runs. Two new tests confirm REST_API and DECLARE_HANDLER style identifiers in non-UE files are left untouched. Plus one minor fix: 5. stripUeMacros signature now accepts (source, _filePath?) to match the LanguageProvider.preprocessSource hook contract exactly. The filePath argument is unused; UE detection is purely content-based. Verification: 34/34 preprocessor tests pass (was 27, +7 new for non-ASCII preservation, REST_API safety, tree-sitter extraction, struct extraction, source position preservation). Full unit suite 5349 pass, 0 regressions. Typecheck clean. Prettier --check clean on all 9 changed files. --------- Co-authored-by: Gergő Magyar --- gitnexus/src/core/ingestion/call-processor.ts | 10 +- .../src/core/ingestion/cpp-ue-preprocessor.ts | 265 +++++++++++++++++ .../src/core/ingestion/heritage-processor.ts | 13 +- .../src/core/ingestion/import-processor.ts | 5 +- .../src/core/ingestion/language-provider.ts | 33 +++ .../src/core/ingestion/languages/c-cpp.ts | 2 + .../src/core/ingestion/parsing-processor.ts | 5 + .../core/ingestion/workers/parse-worker.ts | 5 + .../test/unit/cpp-ue-preprocessor.test.ts | 272 ++++++++++++++++++ 9 files changed, 600 insertions(+), 10 deletions(-) create mode 100644 gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts create mode 100644 gitnexus/test/unit/cpp-ue-preprocessor.test.ts diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 6c59578b0..bf0206057 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -769,9 +769,10 @@ export const processCalls = async ( let tree = astCache.get(file.path); if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch (parseError) { continue; @@ -3280,9 +3281,10 @@ export const extractFetchCallsFromFiles = async ( let tree = astCache.get(file.path); if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch { continue; diff --git a/gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts b/gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts new file mode 100644 index 000000000..baa5d17ff --- /dev/null +++ b/gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts @@ -0,0 +1,265 @@ +/** + * Unreal Engine reflection-macro preprocessor for C++ source. + * + * Tree-sitter does not expand C preprocessor macros, so Unreal's reflection + * markers (`UCLASS(...)`, `UFUNCTION(...)`, `MODULENAME_API`, ...) are parsed + * verbatim. The result is mis-parsed declarations: in `class BRAWLUI_API + * UMyClass : public UObject`, tree-sitter-cpp captures `BRAWLUI_API` as the + * class name and the rest of the declaration becomes structurally wrong. + * + * This module elides those macros from the source text BEFORE tree-sitter + * parses it. Replacement is **length-preserving** (each elided byte becomes + * a space, newlines preserved) so byte offsets and line/column positions + * tree-sitter reports remain identical to the original file. Symbol + * locations in the graph stay accurate. + * + * A cheap detection guard short-circuits files that don't look like UE + * sources, so non-UE C++ codebases pay no cost. + * + * Pure function — no tree-sitter dependency, safe for worker threads. + */ +/** + * Strong UE markers — reflection macros that only Unreal Engine projects use. + * Presence of one of these is sufficient evidence that the file is a UE source + * and that `MODULENAME_API` tokens in it are intended as export macros. + * + * Importantly, `_API` tokens are NOT in this guard — `REST_API`, `HTTP_API`, + * `MY_LIB_API` and similar identifiers appear in plenty of non-UE C++ codebases + * as constants/enums/parameter names. We must not erase them just because the + * file mentions an `_API` token. + */ +const HAS_UE_HINT = + /\b(?:UCLASS|UFUNCTION|UPROPERTY|USTRUCT|UENUM|UINTERFACE|GENERATED_BODY|GENERATED_[A-Z_]+_BODY|UE_DEPRECATED|DECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE)/; + +const SIMPLE_MACROS_NO_ARGS: readonly string[] = [ + 'GENERATED_BODY', + 'GENERATED_UCLASS_BODY', + 'GENERATED_USTRUCT_BODY', + 'GENERATED_UINTERFACE_BODY', + 'GENERATED_IINTERFACE_BODY', + 'DECLARE_CLASS', + 'GENERATED_BODY_LEGACY', +]; + +const PARENTHESIZED_MACROS: readonly string[] = [ + 'UCLASS', + 'UFUNCTION', + 'UPROPERTY', + 'USTRUCT', + 'UENUM', + 'UINTERFACE', + 'UMETA', + 'UE_DEPRECATED', +]; + +const DELEGATE_MACRO_RE = + /\bDECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE(?:_(?:RetVal_OneParam|RetVal_TwoParams|RetVal_ThreeParams|RetVal_FourParams|RetVal_FiveParams|RetVal_SixParams|RetVal_SevenParams|RetVal_EightParams|RetVal_NineParams|RetVal|OneParam|TwoParams|ThreeParams|FourParams|FiveParams|SixParams|SevenParams|EightParams|NineParams|TenParams))?(?=\s*\()/g; + +/** + * Module export tokens like `BRAWLUI_API`, `ENGINE_API`, `COREUOBJECT_API`. + * Pattern: ALL_CAPS identifier ending in `_API`. The leading word boundary + * (`\b`) prevents matching mid-identifier. + */ +const API_MACRO_RE = /\b[A-Z][A-Z0-9_]*_API\b/g; + +/** Replace `[start, end)` of `chars` with spaces, preserving newlines. */ +function eraseRange(chars: string[], start: number, end: number): void { + for (let i = start; i < end; i++) { + if (chars[i] !== '\n' && chars[i] !== '\r') { + chars[i] = ' '; + } + } +} + +/** + * Find the matching close paren for an opening paren at index `openIdx`. + * Returns the index of `)` (inclusive end), or -1 if unbalanced. + * + * Handles nested parens and string/char literals so commas/parens inside + * strings don't throw off the match. Does not attempt to handle raw string + * literals (`R"(...)"`); UE reflection-macro arguments do not use them in + * practice. + */ +function findMatchingParen(source: string, openIdx: number): number { + if (source.charCodeAt(openIdx) !== 0x28) return -1; + let depth = 1; + let i = openIdx + 1; + const len = source.length; + while (i < len && depth > 0) { + const ch = source.charCodeAt(i); + // String literal + if (ch === 0x22) { + i++; + while (i < len) { + const c = source.charCodeAt(i); + if (c === 0x5c) { + i += 2; + continue; + } + if (c === 0x22) { + i++; + break; + } + i++; + } + continue; + } + // Char literal + if (ch === 0x27) { + i++; + while (i < len) { + const c = source.charCodeAt(i); + if (c === 0x5c) { + i += 2; + continue; + } + if (c === 0x27) { + i++; + break; + } + i++; + } + continue; + } + // Line comment + if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2f) { + while (i < len && source.charCodeAt(i) !== 0x0a) i++; + continue; + } + // Block comment + if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2a) { + i += 2; + while (i < len) { + if (source.charCodeAt(i) === 0x2a && source.charCodeAt(i + 1) === 0x2f) { + i += 2; + break; + } + i++; + } + continue; + } + if (ch === 0x28) depth++; + else if (ch === 0x29) { + depth--; + if (depth === 0) return i; + } + i++; + } + return -1; +} + +/** Match a whole-word identifier at `idx`. Returns the byte after the identifier, or -1 on miss. */ +function matchIdentifierAt(source: string, idx: number, name: string): number { + if (idx > 0) { + const prev = source.charCodeAt(idx - 1); + if ( + (prev >= 0x30 && prev <= 0x39) || + (prev >= 0x41 && prev <= 0x5a) || + (prev >= 0x61 && prev <= 0x7a) || + prev === 0x5f + ) { + return -1; + } + } + for (let k = 0; k < name.length; k++) { + if (source.charCodeAt(idx + k) !== name.charCodeAt(k)) return -1; + } + const after = idx + name.length; + if (after < source.length) { + const next = source.charCodeAt(after); + if ( + (next >= 0x30 && next <= 0x39) || + (next >= 0x41 && next <= 0x5a) || + (next >= 0x61 && next <= 0x7a) || + next === 0x5f + ) { + return -1; + } + } + return after; +} + +/** Skip ASCII whitespace forward from `idx`. Returns the next non-whitespace byte index. */ +function skipWhitespace(source: string, idx: number): number { + const len = source.length; + while (idx < len) { + const ch = source.charCodeAt(idx); + if (ch === 0x20 || ch === 0x09 || ch === 0x0a || ch === 0x0d) { + idx++; + continue; + } + break; + } + return idx; +} + +/** + * Strip Unreal Engine reflection macros from C++ source, length-preserving. + * + * Returns the original string unchanged if no strong UE marker is detected, + * so non-UE C++ files (including ones that contain `*_API`-suffixed + * identifiers like `REST_API` or `HTTP_API`) incur only a single regex test. + * + * The `_filePath` parameter is part of the `LanguageProvider.preprocessSource` + * contract but is unused — UE detection is purely content-based. Accepted and + * ignored here so the function matches the hook signature exactly. + */ +export function stripUeMacros(source: string, _filePath?: string): string { + if (!HAS_UE_HINT.test(source)) return source; + + const chars: string[] = source.split(''); + + for (const macro of PARENTHESIZED_MACROS) { + let searchFrom = 0; + while (true) { + const hit = source.indexOf(macro, searchFrom); + if (hit < 0) break; + searchFrom = hit + 1; + const after = matchIdentifierAt(source, hit, macro); + if (after < 0) continue; + const parenIdx = skipWhitespace(source, after); + if (source.charCodeAt(parenIdx) !== 0x28) continue; + const close = findMatchingParen(source, parenIdx); + if (close < 0) continue; + eraseRange(chars, hit, close + 1); + } + } + + for (const macro of SIMPLE_MACROS_NO_ARGS) { + let searchFrom = 0; + while (true) { + const hit = source.indexOf(macro, searchFrom); + if (hit < 0) break; + searchFrom = hit + 1; + const after = matchIdentifierAt(source, hit, macro); + if (after < 0) continue; + const parenIdx = skipWhitespace(source, after); + if (source.charCodeAt(parenIdx) === 0x28) { + const close = findMatchingParen(source, parenIdx); + if (close < 0) continue; + eraseRange(chars, hit, close + 1); + } else { + eraseRange(chars, hit, after); + } + } + } + + for (const re of [DELEGATE_MACRO_RE, API_MACRO_RE]) { + re.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = re.exec(source)) !== null) { + const start = match.index; + let end = start + match[0].length; + if (re === DELEGATE_MACRO_RE) { + const parenIdx = skipWhitespace(source, end); + if (source.charCodeAt(parenIdx) === 0x28) { + const close = findMatchingParen(source, parenIdx); + if (close >= 0) end = close + 1; + } + } + eraseRange(chars, start, end); + } + } + + return chars.join(''); +} diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts index 2c973ad8e..f8628e651 100644 --- a/gitnexus/src/core/ingestion/heritage-processor.ts +++ b/gitnexus/src/core/ingestion/heritage-processor.ts @@ -219,9 +219,13 @@ export const processHeritage = async ( let tree = astCache.get(file.path); if (!tree) { // Use larger bufferSize for files > 32KB + // Per-language source preprocessor (length-preserving, e.g. UE macro + // stripping for C++). MUST mirror parsing-processor on cache miss so + // re-parses see the same input as the cached AST. + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch (parseError) { // Skip files that can't be parsed @@ -413,9 +417,10 @@ export async function extractExtractedHeritageFromFiles( let tree = astCache.get(file.path); if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch { continue; diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts index 03cbed5b7..6f0b40b6f 100644 --- a/gitnexus/src/core/ingestion/import-processor.ts +++ b/gitnexus/src/core/ingestion/import-processor.ts @@ -305,9 +305,10 @@ export const processImports = async ( let wasReparsed = false; if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch (parseError) { continue; diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index 12ce0839b..e139cf5f3 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -116,6 +116,39 @@ interface LanguageProviderConfig { * Required for tree-sitter languages; empty string for standalone processors. */ readonly treeSitterQueries: string; + /** + * Optional source-text transform that runs **before** tree-sitter parses the file. + * + * Used to elide language constructs that confuse the grammar without affecting + * source-position fidelity — e.g., Unreal Engine reflection macros (`UCLASS`, + * `UFUNCTION`, `MODULENAME_API`) in C++ headers that prevent the parser from + * recognising class/function names correctly. + * + * **Length / position preservation:** the returned string MUST have the same + * JavaScript `.length` as the input AND preserve every newline (`\n`/`\r`) + * position byte-for-byte. Implementations replace elided characters with + * ASCII spaces while leaving newlines untouched. With this contract: + * + * - tree-sitter's reported `startPosition.row`/`startPosition.column` + * match the original file exactly (line/column come from newline counts) + * - `startIndex`/`endIndex` byte offsets match the original file exactly + * **when the elided range is pure ASCII** (UTF-16 `.length` equals UTF-8 + * byte length only for ASCII). + * + * Implementations targeting languages where elided ranges may contain + * non-ASCII content must therefore preserve byte length, not just `.length`, + * if downstream code uses `startIndex` to slice the original UTF-8 bytes. + * The current C++ UE-macro preprocessor relies on the practical fact that + * UE reflection macros and module-export tokens are ASCII-only. + * + * Must be a pure function — same input always yields the same output. Called + * once per file, on every code path that re-parses (parsing-processor, import + * processor, heritage processor, call processor, parse worker). + * + * Default: undefined (no preprocessing — `file.content` is parsed verbatim). + */ + readonly preprocessSource?: (sourceText: string, filePath: string) => string; + // ── Core (required) ─────────────────────────────────────────────── /** Type extraction: declarations, initializers, for-loop bindings */ readonly typeConfig: LanguageTypeConfig; diff --git a/gitnexus/src/core/ingestion/languages/c-cpp.ts b/gitnexus/src/core/ingestion/languages/c-cpp.ts index a5b3e5729..693e8cec2 100644 --- a/gitnexus/src/core/ingestion/languages/c-cpp.ts +++ b/gitnexus/src/core/ingestion/languages/c-cpp.ts @@ -45,6 +45,7 @@ import { cVariableConfig, cppVariableConfig } from '../variable-extractors/confi import { createCallExtractor } from '../call-extractors/generic.js'; import { cCallConfig, cppCallConfig } from '../call-extractors/configs/c-cpp.js'; import { createHeritageExtractor } from '../heritage-extractors/generic.js'; +import { stripUeMacros } from '../cpp-ue-preprocessor.js'; const C_BUILT_INS: ReadonlySet = new Set([ 'printf', @@ -410,6 +411,7 @@ export const cppProvider = defineLanguage({ }, ] satisfies AstFrameworkPatternConfig[], treeSitterQueries: CPP_QUERIES, + preprocessSource: stripUeMacros, typeConfig: cCppConfig, exportChecker: cCppExportChecker, importResolver: createImportResolver(cppImportConfig), diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 8803ec023..98036fbe8 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -371,6 +371,11 @@ const processParsingSequential = async ( isVueSetup = extracted.isSetup; } + // Per-language source-text transform (e.g., UE macro stripping for C++). + // Length-preserving — see LanguageProvider.preprocessSource contract. + parseContent = + getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent; + try { await loadLanguage(language, file.path); } catch { diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 5c6712562..4442b5a65 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -1407,6 +1407,11 @@ const processFileGroup = ( isVueSetup = extracted.isSetup; } + // Per-language source-text transform (e.g., UE macro stripping for C++). + // Length-preserving — see LanguageProvider.preprocessSource contract. + parseContent = + getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent; + clearCaches(); // Reset memoization before each new file let tree; diff --git a/gitnexus/test/unit/cpp-ue-preprocessor.test.ts b/gitnexus/test/unit/cpp-ue-preprocessor.test.ts new file mode 100644 index 000000000..087c5ba2a --- /dev/null +++ b/gitnexus/test/unit/cpp-ue-preprocessor.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import CPP from 'tree-sitter-cpp'; +import { stripUeMacros } from '../../src/core/ingestion/cpp-ue-preprocessor.js'; + +describe('stripUeMacros — detection guard', () => { + it('returns input unchanged when no UE markers are present', () => { + const src = `class Plain {\npublic:\n int Get() const;\n};`; + expect(stripUeMacros(src)).toBe(src); + }); + + it('returns input unchanged for STL-style code', () => { + const src = `#include \nstd::vector v;`; + expect(stripUeMacros(src)).toBe(src); + }); +}); + +describe('stripUeMacros — length preservation', () => { + const ueSamples: string[] = [ + `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject { GENERATED_BODY() public: UFUNCTION() void Run(); };`, + `UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Combat") int32 Health;`, + `USTRUCT(BlueprintType)\nstruct ENGINE_API FMyData { GENERATED_BODY() float Value; };`, + `DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMyDelegate, int32, A, FString, B);`, + `UE_DEPRECATED(5.0, "Use NewThing instead") void OldThing();`, + ]; + + for (const src of ueSamples) { + it(`preserves byte length: ${src.slice(0, 40).replace(/\n/g, '\\n')}…`, () => { + const out = stripUeMacros(src); + expect(out.length).toBe(src.length); + }); + + it(`preserves newline positions: ${src.slice(0, 40).replace(/\n/g, '\\n')}…`, () => { + const out = stripUeMacros(src); + const inputNewlines: number[] = []; + const outputNewlines: number[] = []; + for (let i = 0; i < src.length; i++) { + if (src.charCodeAt(i) === 0x0a) inputNewlines.push(i); + if (out.charCodeAt(i) === 0x0a) outputNewlines.push(i); + } + expect(outputNewlines).toEqual(inputNewlines); + }); + } +}); + +describe('stripUeMacros — macro removal', () => { + it('elides UCLASS(...) with arguments', () => { + const src = `UCLASS(BlueprintType, Category="Foo")\nclass UFoo {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('UCLASS'); + expect(out).not.toContain('BlueprintType'); + expect(out).toContain('class UFoo {};'); + }); + + it('elides UCLASS() with empty parens', () => { + const src = `UCLASS()\nclass UBar {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('UCLASS'); + expect(out).toContain('class UBar {};'); + }); + + it('elides MODULE_API export macros (BRAWLUI_API style) when paired with a UE marker', () => { + const src = `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('BRAWLUI_API'); + expect(out).toContain('class'); + expect(out).toContain('UMyClass'); + expect(out).toContain('public UObject'); + }); + + it('elides multiple distinct *_API tokens in same file when UE marker is present', () => { + const src = `UCLASS()\nclass CORE_API A {};\nUCLASS()\nclass UMG_API B : public A {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('CORE_API'); + expect(out).not.toContain('UMG_API'); + expect(out).toContain('class'); + expect(out).toContain('A {};'); + }); + + it('elides GENERATED_BODY() inside class body', () => { + const src = `class UThing { GENERATED_BODY() public: void Foo(); };`; + const out = stripUeMacros(src); + expect(out).not.toContain('GENERATED_BODY'); + expect(out).toContain('public:'); + expect(out).toContain('void Foo();'); + }); + + it('elides UFUNCTION(...) before method declarations', () => { + const src = `class X { UFUNCTION(BlueprintCallable, Server, Reliable) void DoThing(); };`; + const out = stripUeMacros(src); + expect(out).not.toContain('UFUNCTION'); + expect(out).not.toContain('BlueprintCallable'); + expect(out).toContain('void DoThing();'); + }); + + it('elides UPROPERTY(...) before field declarations', () => { + const src = `class X { UPROPERTY(EditAnywhere) int32 Health; };`; + const out = stripUeMacros(src); + expect(out).not.toContain('UPROPERTY'); + expect(out).not.toContain('EditAnywhere'); + expect(out).toContain('int32 Health;'); + }); + + it('elides DECLARE_DYNAMIC_MULTICAST_DELEGATE_*Params(...)', () => { + const src = `DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMyDelegate, int32, Value);\nclass X {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('DECLARE_DYNAMIC_MULTICAST_DELEGATE'); + expect(out).not.toContain('FMyDelegate'); + expect(out).toContain('class X {};'); + }); + + it('elides UE_DEPRECATED(...) before function declarations', () => { + const src = `UE_DEPRECATED(5.1, "Reason") void Old();`; + const out = stripUeMacros(src); + expect(out).not.toContain('UE_DEPRECATED'); + expect(out).not.toContain('5.1'); + expect(out).toContain('void Old();'); + }); +}); + +describe('stripUeMacros — non-UE files left alone', () => { + it('does NOT strip standalone *_API identifiers when no UE marker is present', () => { + const src = `enum class Status { REST_API = 1, HTTP_API = 2, MY_LIB_API = 3 };\nvoid handle(REST_API status);`; + expect(stripUeMacros(src)).toBe(src); + }); + + it('does NOT strip _API tokens in a file that only mentions DECLARE_DELEGATE-like macros from non-UE codebases', () => { + const src = `// Custom delegate framework, not UE\n#define DECLARE_HANDLER(x) void x()\nDECLARE_HANDLER(MyHandler);\nint REST_API = 0;`; + expect(stripUeMacros(src)).toBe(src); + }); +}); + +describe('stripUeMacros — non-ASCII content preservation', () => { + it('leaves non-ASCII content outside elided ranges intact and at the same .length offset', () => { + const src = `// Comment with non-ASCII: café résumé naïve\nUCLASS()\nclass UMyClass : public UObject\n{\n GENERATED_BODY()\n // Trailing: 日本語 αβγ\n};`; + const out = stripUeMacros(src); + expect(out.length).toBe(src.length); + expect(out).toContain('café résumé naïve'); + expect(out).toContain('日本語 αβγ'); + expect(out).toContain('class UMyClass : public UObject'); + expect(out).not.toContain('UCLASS'); + expect(out).not.toContain('GENERATED_BODY'); + }); + + it('preserves newline positions when the file contains non-ASCII characters', () => { + const src = `// café\nUPROPERTY()\nint32 Health;\n// résumé\nUFUNCTION()\nvoid Run();`; + const out = stripUeMacros(src); + const inputNewlines: number[] = []; + const outputNewlines: number[] = []; + for (let i = 0; i < src.length; i++) { + if (src.charCodeAt(i) === 0x0a) inputNewlines.push(i); + if (out.charCodeAt(i) === 0x0a) outputNewlines.push(i); + } + expect(outputNewlines).toEqual(inputNewlines); + }); +}); + +describe('stripUeMacros — false-positive guards', () => { + it('does NOT strip identifiers that merely contain UCLASS as a substring', () => { + const src = `void NotUCLASSAtAll(); int MyUCLASS = 0;`; + const out = stripUeMacros(src); + expect(out).toBe(src); + }); + + it('does NOT strip _API substrings inside larger identifiers', () => { + const src = `class MY_APIName {};\nint not_my_API_thing = 0;`; + const out = stripUeMacros(src); + expect(out).toContain('MY_APIName'); + expect(out).toContain('not_my_API_thing'); + }); + + it('does not eat parens balanced inside string literals', () => { + const src = `UFUNCTION(meta=(DisplayName="Foo (Bar)")) void Z();`; + const out = stripUeMacros(src); + expect(out).not.toContain('UFUNCTION'); + expect(out).not.toContain('DisplayName'); + expect(out).toContain('void Z();'); + }); + + it('handles UCLASS with deeply nested parens in arguments', () => { + const src = `UCLASS(meta=(Categories=("A.B", "C.D")), Within=Foo) class UDeep {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('UCLASS'); + expect(out).not.toContain('Categories'); + expect(out).toContain('class UDeep {};'); + }); + + it('leaves Qt macros alone (only UE markers stripped)', () => { + const src = `class QFoo { Q_OBJECT public: void Bar(); };`; + const out = stripUeMacros(src); + expect(out).toContain('Q_OBJECT'); + }); +}); + +describe('stripUeMacros — class-name extraction sanity', () => { + it('after stripping, "class UMyClass" appears immediately after "class "', () => { + const src = `UCLASS(BlueprintType)\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n};`; + const out = stripUeMacros(src); + const classIdx = out.indexOf('class '); + expect(classIdx).toBeGreaterThanOrEqual(0); + const tail = out.slice(classIdx + 'class '.length).trimStart(); + expect(tail.startsWith('UMyClass')).toBe(true); + }); +}); + +describe('stripUeMacros — tree-sitter extraction (end-to-end)', () => { + /** + * Walk the parse tree and return the captured class name(s). Works against + * the actual tree-sitter-cpp grammar so this is a true integration check + * for the core PR claim: the indexer now sees `UMyClass`, not `BRAWLUI_API`. + */ + function extractClassNames(source: string): string[] { + const parser = new Parser(); + parser.setLanguage(CPP as unknown as Parser.Language); + const tree = parser.parse(source); + const names: string[] = []; + const stack: Parser.SyntaxNode[] = [tree.rootNode]; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.type === 'class_specifier' || node.type === 'struct_specifier') { + const nameNode = node.childForFieldName('name'); + if (nameNode) names.push(nameNode.text); + } + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child) stack.push(child); + } + } + return names; + } + + it('tree-sitter-cpp captures UMyClass as the class name (not BRAWLUI_API)', () => { + const src = `UCLASS(BlueprintType)\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n public:\n UFUNCTION()\n void Run();\n};`; + const out = stripUeMacros(src); + const names = extractClassNames(out); + expect(names).toContain('UMyClass'); + expect(names).not.toContain('BRAWLUI_API'); + }); + + it('tree-sitter-cpp captures struct name correctly through USTRUCT + MODULE_API', () => { + const src = `USTRUCT(BlueprintType)\nstruct ENGINE_API FMyData : public FBase\n{\n GENERATED_BODY()\n float Value;\n};`; + const out = stripUeMacros(src); + const names = extractClassNames(out); + expect(names).toContain('FMyData'); + expect(names).not.toContain('ENGINE_API'); + }); + + it('tree-sitter-cpp source positions are preserved across stripping (line numbers match)', () => { + const src = `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n public:\n void Run();\n};`; + const out = stripUeMacros(src); + const parser = new Parser(); + parser.setLanguage(CPP as unknown as Parser.Language); + const tree = parser.parse(out); + const stack: Parser.SyntaxNode[] = [tree.rootNode]; + let runLine: number | undefined; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.type === 'function_declarator') { + const declarator = node.childForFieldName('declarator'); + if (declarator?.text === 'Run') { + runLine = node.startPosition.row; + break; + } + } + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child) stack.push(child); + } + } + expect(runLine).toBe(5); // 0-indexed: "void Run();" is on line 6 (index 5) + }); +}); From b5627f27d8973ffc23f40e1994e3939e2da67745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 9 May 2026 13:35:04 +0100 Subject: [PATCH 05/11] ci: add fork-safe PR autofix pipeline (#1446) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: add fork-safe PR autofix pipeline Two-workflow split posts prettier + eslint --fix output as inline review-comment suggestions on PRs (including fork PRs) without running fork-controlled ESLint plugins under a privileged token. - pr-autofix.yml: untrusted, runs lint:fix/format with permissions: {}, uploads diff artifact. paths-ignore on lockfiles/snapshots/dist to avoid reviewdog 406 on >3k-line diffs. - pr-autofix-publish.yml: trusted workflow_run consumer. Validates every metadata.json field with regex allowlists before exporting to GITHUB_OUTPUT (closes head_ref newline-injection vector). Concurrency keyed on PR number with fork fallback to head-repo+branch. Reviewdog pinned to v0.21.0. Sticky comment posts only when patch is non-empty (no noise on clean PRs); body carries a fenced gitnexus-autofix JSON block under a stable HTML marker for agent parsing. gh API calls go through a small retry helper for transient 5xx. Branch protection should enable merge queue + 'require branches up to date' to handle PR freshness; chinthakagodawita/autoupdate is dropped (unmaintained since 2023). * ci(autofix): close zizmor template-injection findings Move fork-controlled values (head.ref, head.repo.full_name, head.sha, pr.number, github.repository) into the step's env: block instead of interpolating them with `${{ }}` directly into the bash run body. The job has permissions:{} today so this is defence-in-depth, but a future scope grant on the untrusted half would otherwise turn a malicious branch name into shell injection. Add pr-autofix-publish.yml to the documented dangerous-triggers ignore list — workflow_run is required to post sticky comments on fork PRs and the file's structural defences (no fork checkout, allowlist on metadata.json, base_repo equality check) match the existing ci-report.yml exemption. * ci(autofix): close remaining review findings - Add an actionlint job to workflow-lint.yml. Catches YAML syntax, expression typing, shellcheck-inside-run, and deprecated runner labels on every .github/** PR — closes the gap that let pr-autofix's YAML literal-block bug reach review on this branch. - pr-autofix-publish.yml emits a `gitnexus/autofix` Check Run on the PR head SHA: conclusion `success` for clean, `neutral` (with distinct output titles) for suggestions-posted vs. skipped-too-large. Stable name lets agents read the outcome via `gh pr checks` without parsing the sticky comment. - Document the autofix signal contract in CONTRIBUTING.md — sticky marker, fenced gitnexus-autofix JSON schema, Check Run name. One source of truth so the marker / schema fields don't drift across the workflow files and consumers. * ci: fix actionlint/shellcheck findings on PR #1446 Closes the actionlint warnings the new lint job (workflow-lint.yml's actionlint runner) surfaced once it was wired into CI. Mostly shellcheck-style cleanups across three workflows. pr-autofix-publish.yml - SC2170: `[ "${{ steps.meta.outputs.changed_lines }}" -gt 3000 ]` interpolates a literal string into bash, breaking shellcheck's arithmetic-comparison parse. Move `changed_lines` through env: as `CHANGED_LINES` and reference as `$CHANGED_LINES` inside bash. ci-report.yml (Read PR metadata step) - SC2002 ×2: `cat file | tr` -> `tr < file`. - SC2129: three consecutive `>> "$GITHUB_OUTPUT"` redirects collapsed into one `{ ...; } >> "$GITHUB_OUTPUT"` group. ci-report.yml (Build report step) - SC2162 ×2: `read VAR1 VAR2` -> `read -r VAR1 VAR2` so backslashes in test-results.json output aren't mangled. - SC2034: drop unused `SUITES` aggregate. The per-framework suite counts (CLI_SU, WEB_SU) are now read into `_` placeholders since the report doesn't surface them anywhere. release-candidate.yml - SC2129 ×2: collapse consecutive `>> "$GITHUB_OUTPUT"` redirects in the rc-version computation step and the tag-push step into one grouped block each. --- .github/workflows/ci-report.yml | 25 +- .github/workflows/pr-autofix-publish.yml | 314 +++++++++++++++++++++++ .github/workflows/pr-autofix.yml | 146 +++++++++++ .github/workflows/release-candidate.yml | 16 +- .github/workflows/workflow-lint.yml | 34 ++- .github/zizmor.yml | 9 + CONTRIBUTING.md | 18 ++ 7 files changed, 541 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/pr-autofix-publish.yml create mode 100644 .github/workflows/pr-autofix.yml diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 03908bb2b..03c933ad0 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -95,31 +95,33 @@ jobs: # Validate PR number is a positive integer (artifact comes from # untrusted fork code, so treat contents defensively). - PR_NUM=$(cat "$DIR/pr_number" | tr -d '[:space:]') + PR_NUM=$(tr -d '[:space:]' < "$DIR/pr_number") if ! [[ "$PR_NUM" =~ ^[0-9]+$ ]]; then echo "skip=true" >> "$GITHUB_OUTPUT" echo "::error::Invalid PR number in artifact: '$PR_NUM'" exit 0 fi - echo "skip=false" >> "$GITHUB_OUTPUT" - echo "pr_number=$PR_NUM" >> "$GITHUB_OUTPUT" # Validate job-result strings against known GitHub Actions values. # Artifact contents come from the PR workflow (potentially untrusted # fork code), so we whitelist to prevent newline injection into # GITHUB_OUTPUT. validate_result() { local val - val=$(cat "$1" | tr -d '[:space:]') + val=$(tr -d '[:space:]' < "$1") case "$val" in success|failure|cancelled|skipped) echo "$val" ;; *) echo "unknown" ;; esac } - echo "quality=$(validate_result "$DIR/quality_result")" >> "$GITHUB_OUTPUT" - echo "tests=$(validate_result "$DIR/tests_result")" >> "$GITHUB_OUTPUT" - echo "e2e=$(validate_result "$DIR/e2e_result")" >> "$GITHUB_OUTPUT" + { + echo "skip=false" + echo "pr_number=$PR_NUM" + echo "quality=$(validate_result "$DIR/quality_result")" + echo "tests=$(validate_result "$DIR/tests_result")" + echo "e2e=$(validate_result "$DIR/e2e_result")" + } >> "$GITHUB_OUTPUT" - name: Checkout (for vitest config) if: steps.meta.outputs.skip != 'true' @@ -279,14 +281,17 @@ jobs: fi } - read CLI_T CLI_P CLI_F CLI_S CLI_SU CLI_D <<< "$(sum_results "$RESULTS_FILE")" - read WEB_T WEB_P WEB_F WEB_S WEB_SU WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")" + # `_` placeholder for the suite-count column — positional + # readability for sum_results' 6-field output, but the value + # isn't surfaced in the report (suites are tracked per-test + # framework, not as a top-line metric). + read -r CLI_T CLI_P CLI_F CLI_S _ CLI_D <<< "$(sum_results "$RESULTS_FILE")" + read -r WEB_T WEB_P WEB_F WEB_S _ WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")" TOTAL=$((CLI_T + WEB_T)) PASSED=$((CLI_P + WEB_P)) FAILED=$((CLI_F + WEB_F)) SKIPPED=$((CLI_S + WEB_S)) - SUITES=$((CLI_SU + WEB_SU)) DURATION=$((CLI_D > WEB_D ? CLI_D : WEB_D)) # ── Status helpers ── diff --git a/.github/workflows/pr-autofix-publish.yml b/.github/workflows/pr-autofix-publish.yml new file mode 100644 index 000000000..a22f2cc7a --- /dev/null +++ b/.github/workflows/pr-autofix-publish.yml @@ -0,0 +1,314 @@ +name: PR Autofix (publish) + +# TRUSTED HALF of the autofix pipeline. +# +# Triggered by `pr-autofix.yml` completing on a PR (including fork PRs). +# Downloads the diff artifact produced by the untrusted job and posts +# inline review-comment suggestions to the PR using `reviewdog`. This +# job NEVER checks out fork code — it only consumes the diff (data) and +# calls the GitHub API. That isolation is what makes it safe to run +# under `pull-requests: write` on fork-triggered events. +# +# Also posts (or edits) a single sticky summary comment so contributors +# and AI agents have one stable, machine-readable signal that says +# whether autofix had anything to suggest. Look for the heading +# "## :sparkles: PR Autofix" in the PR's top-level comments. +# +# Reviewdog reporter: `github-pr-review` reads $REVIEWDOG_GITHUB_API_TOKEN +# and posts via the GraphQL/REST PR-review API. It does not need a +# checkout because the diff itself encodes file paths + line numbers. + +on: + workflow_run: + workflows: ['PR Autofix'] + types: [completed] + +concurrency: + # Key on PR identity, NOT workflow_run.id — workflow_run.id is per-run + # unique, which would defeat serialization and let two parallel + # publishes both POST a sticky summary comment. CONTRIBUTING.md + # § GitHub Actions — Concurrency Convention names this anti-pattern + # explicitly. For fork PRs, `pull_requests[]` is empty in the + # workflow_run payload, so we fall back to head-repo + head-branch. + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }} + cancel-in-progress: false + +permissions: {} + +jobs: + publish: + name: publish-autofix + if: >- + github.event.workflow_run.event == 'pull_request' + && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + # Required by actions/download-artifact to fetch artifacts produced + # by a different workflow run. + actions: read + # Required to create the `gitnexus/autofix` Check Run that reports + # the outcome (clean / suggestions-posted / skipped-too-large) to + # the PR's Checks tab. Branch protection or agents can grep the + # conclusion + output title without parsing the sticky comment. + checks: write + steps: + # Pinned to v8.0.1. Verify SHA via: + # gh api repos/actions/download-artifact/git/refs/tags/v8.0.1 + - name: Download autofix artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: autofix + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: autofix-in + + - name: Read and validate metadata + id: meta + shell: bash + run: | + set -euo pipefail + test -f autofix-in/metadata.json + jq . autofix-in/metadata.json + + # The artifact comes from the untrusted half running fork code. + # Every field is allowlist-validated before it can flow into + # $GITHUB_OUTPUT. A newline in head_ref would otherwise let a + # malicious branch name inject a second `pr_number=N` line and + # redirect this job's reviewdog suggestions / sticky summary + # comment onto a victim PR under github-actions[bot] with + # pull-requests: write. + assert_field() { + local key="$1" pattern="$2" value + value=$(jq -r ".${key} // empty" autofix-in/metadata.json) + if [ -z "$value" ] || ! [[ "$value" =~ $pattern ]]; then + echo "::error::metadata.${key} failed allowlist (got: $(printf '%q' "$value"))" + exit 1 + fi + printf '%s' "$value" + } + + SCHEMA=$(assert_field schema '^gitnexus\.pr-autofix/v[0-9]+$') + PR_NUMBER=$(assert_field pr_number '^[0-9]+$') + HEAD_SHA=$(assert_field head_sha '^[0-9a-f]{40}$') + HEAD_REF=$(assert_field head_ref '^[A-Za-z0-9._/-]+$') + HEAD_REPO=$(assert_field head_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') + BASE_REPO=$(assert_field base_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') + CHANGED=$(assert_field changed_lines '^[0-9]+$') + + # Defence-in-depth: refuse to act if the artifact claims to + # belong to a different repo than the one that triggered us. + if [ "$BASE_REPO" != "${GITHUB_REPOSITORY}" ]; then + echo "::error::Artifact base_repo does not match \$GITHUB_REPOSITORY — refusing to publish." + exit 1 + fi + + { + echo "schema=${SCHEMA}" + echo "pr_number=${PR_NUMBER}" + echo "head_sha=${HEAD_SHA}" + echo "head_ref=${HEAD_REF}" + echo "head_repo=${HEAD_REPO}" + echo "base_repo=${BASE_REPO}" + echo "changed_lines=${CHANGED}" + } >> "$GITHUB_OUTPUT" + + # Pinned to v1.5.0. Verify SHA via: + # gh api repos/reviewdog/action-setup/git/refs/tags/v1.5.0 + # (annotated tag — resolve via .../git/tags/ --jq .object) + - name: Install reviewdog + if: steps.meta.outputs.changed_lines != '0' + uses: reviewdog/action-setup@d8a7baabd7f3e8544ee4dbde3ee41d0011c3a93f # v1.5.0 + with: + # Pin the binary, not just the action SHA — a bad reviewdog + # release otherwise breaks every PR with no rollback. Bump + # this knob deliberately when validating a new release. + reviewdog_version: v0.21.0 + + - name: Post inline suggestions + id: suggest + if: steps.meta.outputs.changed_lines != '0' + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CI_REPO_OWNER: ${{ github.repository_owner }} + CI_REPO_NAME: ${{ github.event.repository.name }} + CI_PULL_REQUEST: ${{ steps.meta.outputs.pr_number }} + CI_COMMIT: ${{ steps.meta.outputs.head_sha }} + # Pull `changed_lines` through env so bash gets a real + # variable (and shellcheck SC2170 doesn't fire on `-gt` against + # a `${{ }}`-interpolated literal). + CHANGED_LINES: ${{ steps.meta.outputs.changed_lines }} + shell: bash + run: | + set -euo pipefail + patch=autofix-in/autofix.patch + if [ ! -s "$patch" ]; then + echo "Empty patch — nothing to suggest." + echo "posted=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # GitHub's review-comment API returns 406 on diffs above ~3k + # changed lines. Bail out gracefully and let the summary + # comment carry the signal instead. + if [ "$CHANGED_LINES" -gt 3000 ]; then + echo "Diff too large ($CHANGED_LINES lines) — skipping inline suggestions." + echo "posted=skipped-too-large" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # `-f.diff.strip=1` matches `git diff` output (a/foo b/foo). + # `-filter-mode=added` only suggests on lines the PR added, + # which avoids re-suggesting on already-resolved threads when + # the contributor re-adds the autoformat label. + reviewdog \ + -f=diff -f.diff.strip=1 \ + -name="prettier+eslint" \ + -reporter=github-pr-review \ + -filter-mode=added \ + -level=warning \ + -fail-on-error=false < "$patch" + + echo "posted=true" >> "$GITHUB_OUTPUT" + + - name: Upsert sticky summary comment + # Only post when ci-quality found something fixable (= the + # autofix patch is non-empty). When prettier/eslint are clean + # the patch is zero bytes and the sticky comment is pure noise, + # so we skip it. When the diff was too large for inline + # suggestions, the sticky is the only signal the contributor + # gets, so we still post in that case. + if: >- + always() + && steps.meta.outputs.pr_number != '' + && steps.meta.outputs.changed_lines != '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PR: ${{ steps.meta.outputs.pr_number }} + CHANGED: ${{ steps.meta.outputs.changed_lines }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + SCHEMA: ${{ steps.meta.outputs.schema }} + POSTED: ${{ steps.suggest.outputs.posted }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + + # Stable heading + marker — agents grep for these exact strings. + marker="" + heading="## :sparkles: PR Autofix" + + if [ "${POSTED}" = "skipped-too-large" ]; then + ui_state="skipped-too-large" + prose="Diff is **${CHANGED}** lines — too large for inline suggestions (GitHub caps the review-comment API at ~3000). Run locally: \`npm run lint:fix && npm run format\`." + else + ui_state="suggestions-posted" + prose="Posted formatting / unused-import suggestions inline. Click **Apply suggestion** on each, or run locally: \`npm run lint:fix && npm run format\`." + fi + + # Machine-readable JSON block — agents parse this instead of + # regexing English. Fenced code-block info string is + # `gitnexus-autofix` so agents can locate it without ambiguity. + json=$(jq -n -c \ + --arg schema "${SCHEMA}" \ + --arg state "${ui_state}" \ + --argjson pr_number "${PR}" \ + --argjson changed_lines "${CHANGED}" \ + --arg head_sha "${HEAD_SHA}" \ + --arg run_id "${RUN_ID}" \ + '{schema:$schema, state:$state, pr_number:$pr_number, changed_lines:$changed_lines, head_sha:$head_sha, run_id:$run_id}') + + # Multi-line quoted string instead of a column-0 heredoc — YAML's + # `run: |` block ends as soon as a content line dedents below the + # block's first-line indent, which would mis-parse the workflow. + body="${marker} + ${heading} + + ${prose} + + \`\`\`gitnexus-autofix + ${json} + \`\`\`" + # Strip the leading 10-space indent that the YAML block requires + # so the rendered comment body starts at column 0. + body="$(printf '%s\n' "$body" | sed 's/^ //')" + + # Small retry wrapper for transient 5xx / rate-limit responses + # on the GitHub REST API. Three tries with linear backoff. We + # only retry GET (idempotent) and PATCH on a known comment id + # (idempotent). POST is NOT wrapped — retrying a comment-create + # would create duplicates if the first attempt actually landed. + gh_retry() { + local n=0 max=3 + while true; do + if gh "$@"; then return 0; fi + n=$((n+1)) + if [ "$n" -ge "$max" ]; then return 1; fi + sleep $((n * 2)) + done + } + + # Find existing bot comment by the marker and edit-in-place; else create. + # CRITICAL: filter by `.user.login == "github-actions[bot]"`. A regular + # user posting a comment containing the marker would otherwise be the + # `head -n1` match; PATCH on someone else's comment 403s, `set -e` + # aborts, and the bot is permanently DoS'd for that PR. + existing=$(gh_retry api "repos/${GH_REPO}/issues/${PR}/comments" \ + --paginate --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"${marker}\"))) | .id" \ + | head -n1 || true) + + if [ -n "${existing}" ]; then + gh_retry api -X PATCH "repos/${GH_REPO}/issues/comments/${existing}" \ + -f body="${body}" >/dev/null + echo "Updated comment ${existing}." + else + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="${body}" >/dev/null + echo "Created summary comment." + fi + + - name: Emit gitnexus/autofix Check Run + # Stable check name `gitnexus/autofix` so PR-watching agents can + # `gh pr checks ` and read the conclusion + title without + # parsing the sticky comment. Three outcomes: + # clean → conclusion: success + # suggestions-posted → conclusion: neutral (review suggestions) + # skipped-too-large → conclusion: neutral (diff > 3000 lines) + # `neutral` does not block branch-protection required-checks but + # is visually distinct from a green pass. + if: always() && steps.meta.outputs.head_sha != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + CHANGED: ${{ steps.meta.outputs.changed_lines }} + POSTED: ${{ steps.suggest.outputs.posted }} + shell: bash + run: | + set -euo pipefail + + if [ "${CHANGED}" = "0" ]; then + conclusion="success" + title="Formatting clean" + summary="Prettier and ESLint --fix produced no changes." + elif [ "${POSTED}" = "skipped-too-large" ]; then + conclusion="neutral" + title="Diff too large for inline suggestions (${CHANGED} lines)" + summary="GitHub caps the review-comment API at ~3000 lines. Run \`npm run lint:fix && npm run format\` locally." + else + conclusion="neutral" + title="Suggestions posted" + summary="Inline review-comment suggestions posted. Click **Apply suggestion** on each, or run \`npm run lint:fix && npm run format\` locally." + fi + + gh api -X POST "repos/${GH_REPO}/check-runs" \ + -f name="gitnexus/autofix" \ + -f head_sha="${HEAD_SHA}" \ + -f status="completed" \ + -f conclusion="${conclusion}" \ + -f "output[title]=${title}" \ + -f "output[summary]=${summary}" \ + >/dev/null + echo "Posted check-run gitnexus/autofix=${conclusion} (${title})" diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml new file mode 100644 index 000000000..f15e8c0e6 --- /dev/null +++ b/.github/workflows/pr-autofix.yml @@ -0,0 +1,146 @@ +name: PR Autofix + +# UNTRUSTED HALF of the autofix pipeline. +# +# Runs `npm run lint:fix` + `npm run format` against the PR head +# (including fork heads) and uploads the resulting diff as an artifact. +# This job has NO privileged token and CANNOT post to the PR. The trusted +# `pr-autofix-publish.yml` workflow downloads the artifact via +# `workflow_run` and posts the inline review-comment suggestions. +# +# Why the split: +# ESLint loads plugins from fork-controlled `node_modules`, so running +# it in a job with `pull-requests: write` would let a malicious fork PR +# ship a poisoned eslint plugin and execute arbitrary code under that +# token. By keeping fork code execution in this job (token: read-only) +# and posting from a separate trusted job that never touches fork +# code, we get the inline-suggestion UX for fork PRs without the +# supply-chain hole. (See autofix.ci for the same pattern.) +# +# Removes unused imports via `eslint-plugin-unused-imports`, already in +# devDependencies and wired into the `lint` config. + +on: + pull_request: + types: [opened, synchronize, reopened] + # Skip lockfile / generated-file PRs entirely — `action-suggester` + # cannot post on diffs > ~3k lines (GitHub returns 406) and these + # paths produce massive diffs no human wants suggested back inline. + paths-ignore: + - '**/package-lock.json' + - '**/*.snap' + - '**/dist/**' + - '**/node_modules/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + # Don't cancel in-flight runs; the publish workflow may already be + # downloading the artifact and a cancelled untrusted run produces no + # signal at all (worse DX than waiting). + cancel-in-progress: false + +# This workflow runs untrusted fork code. Top-level deny-all and NO +# job-level grants — the job can only read its own checkout. +permissions: {} + +jobs: + autofix: + name: autofix + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # PR head commit (not the synthetic merge ref) — we need the + # exact tree the contributor pushed so suggestions line up. + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 20 + cache: npm + cache-dependency-path: package-lock.json + + # `--ignore-scripts` blocks pre/postinstall lifecycle hooks. ESLint + # plugins still load from node_modules (that is the actual escape + # hatch on a typical fork), but this job has no token to abuse — + # which is the whole point of the split. + - run: npm ci --ignore-scripts + + - name: ESLint --fix (removes unused imports) + run: npm run lint:fix + # Lint errors that --fix can't auto-resolve must not block the + # diff artifact — partial fixes are still useful as suggestions. + continue-on-error: true + + - name: Prettier --write + run: npm run format + continue-on-error: true + + - name: Capture diff and metadata + id: capture + # Pass GitHub-context values via env: rather than `${{ }}` + # interpolated directly into the bash body. `head.ref` and + # `head.repo.full_name` are fork-controlled strings; expanding + # them into shell source is the canonical template-injection + # vector zizmor flags. Even though this job has `permissions: {}`, + # routing through env: makes it impossible for a future scope + # grant to turn into RCE. Inside bash, reference as `$HEAD_REF` + # etc. — the values are then plain strings, not code. + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REPO: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + mkdir -p autofix-out + + # Produce a unified diff of the working tree vs. the PR head. + # Empty diff => nothing to suggest; the publish job short-circuits. + git diff --no-color > autofix-out/autofix.patch + + # NOTE: `changed_lines` is the line-count of the patch file, + # which includes hunk headers and context lines — NOT the + # added/removed source-line count. The 3000-line cap in + # pr-autofix-publish.yml is therefore conservative (fires + # before reviewdog hits GitHub's ~3k review-comment API + # ceiling). That bias is intentional. + changed_lines=$(wc -l < autofix-out/autofix.patch | tr -d ' ') + echo "changed_lines=${changed_lines}" >> "$GITHUB_OUTPUT" + + # Carry PR identity over to the trusted job. workflow_run + # context is base-repo-only, so the publish job needs these + # to call the GitHub PR API on the right resource. + # CONTRACT: keep this schema in sync with pr-autofix-publish.yml's + # `assert_field` validators and the agent-facing JSON block in + # the sticky comment. Bump `schema` when changing field names. + jq -n \ + --arg schema 'gitnexus.pr-autofix/v1' \ + --argjson pr_number "${PR_NUMBER}" \ + --arg head_sha "${HEAD_SHA}" \ + --arg head_ref "${HEAD_REF}" \ + --arg head_repo "${HEAD_REPO}" \ + --arg base_repo "${BASE_REPO}" \ + --argjson changed_lines "${changed_lines}" \ + '{schema:$schema, pr_number:$pr_number, head_sha:$head_sha, head_ref:$head_ref, head_repo:$head_repo, base_repo:$base_repo, changed_lines:$changed_lines}' \ + > autofix-out/metadata.json + + echo "--- metadata ---" + cat autofix-out/metadata.json + echo "--- diff (head) ---" + head -c 2000 autofix-out/autofix.patch || true + + # Pinned to v7.0.1. Verify SHA via: + # gh api repos/actions/upload-artifact/git/refs/tags/v7.0.1 + - name: Upload autofix artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: autofix + path: autofix-out/ + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index 9ab00c0bd..36f37b5c6 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -294,9 +294,11 @@ jobs: fi fi - echo "base=$BASE" >> "$GITHUB_OUTPUT" - echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT" - echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT" + { + echo "base=$BASE" + echo "rc_n=$NEXT_N" + echo "rc_version=$RC_VERSION" + } >> "$GITHUB_OUTPUT" - name: Apply rc version in-CI shell: bash @@ -354,9 +356,11 @@ jobs: # remote ref, the push fails and we stop before npm publish. git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" - echo "vtag=$VTAG" >> "$GITHUB_OUTPUT" - echo "marker=$MARKER" >> "$GITHUB_OUTPUT" - echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT" + { + echo "vtag=$VTAG" + echo "marker=$MARKER" + echo "release_sha=$RELEASE_SHA" + } >> "$GITHUB_OUTPUT" - name: Publish to npm (rc dist-tag) run: npm publish --provenance --access public --tag rc diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index 97f388d28..7b38b8ddd 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -1,10 +1,13 @@ -name: Workflow Lint (zizmor) +name: Workflow Lint -# Lints .github/workflows/** for known GitHub Actions security misconfigurations: -# unpinned Actions, dangerous ${{ ... }} interpolation in run: blocks, -# missing per-job permissions:, etc. +# Lints .github/workflows/** for both: +# - actionlint: YAML syntax, expression typing, shellcheck inside `run:` +# blocks, unknown contexts, deprecated runner labels. +# - zizmor: security misconfigurations — unpinned actions, dangerous +# `${{ }}` interpolation, missing per-job permissions, etc. # -# Scoped to PRs that touch .github/** only — keeps off the typical PR critical path. +# Scoped to PRs that touch .github/** only — keeps off the typical PR +# critical path. on: pull_request: @@ -17,6 +20,27 @@ concurrency: cancel-in-progress: true jobs: + actionlint: + name: actionlint + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # Pinned to v2.1.2. Verify SHA via: + # gh api repos/raven-actions/actionlint/git/refs/tags/v2.1.2 + # The action wraps the upstream `rhysd/actionlint` binary and emits + # GitHub-annotation-formatted findings on PRs. + - name: Run actionlint + uses: raven-actions/actionlint@205b530c5d9fa8f44ae9ed59f341a0db994aa6f8 # v2.1.2 + with: + fail-on-error: true + zizmor: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/zizmor.yml b/.github/zizmor.yml index c534679c1..b2f89e3ba 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -14,6 +14,15 @@ rules: # no checkout of fork code occurs. Header comment in the file documents. - ci-report.yml + # workflow_run is the trusted half of the autofix pipeline. The + # untrusted half (pr-autofix.yml) runs fork code with permissions:{} + # and produces only a diff artifact (data, not executable code). The + # publish job consumes the artifact, allowlist-validates every field + # of metadata.json before exporting to $GITHUB_OUTPUT, never checks + # out fork code, and never executes anything fork-controlled. Header + # comment in the file documents the split. + - pr-autofix-publish.yml + # pull_request_target needed by claude-code-action to access secrets # and post review comments on fork PRs. Mitigated by: PR checkouts pin # the fork's HEAD SHA (not the branch ref) to prevent TOCTOU races, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ed2aeba5..99930cf0a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,6 +103,24 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency: - When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape. +## CI automation contracts + +Two workflows produce machine-readable signals on every PR. Coding agents and humans alike can rely on the names and shapes below — change them with intent. + +### `gitnexus/autofix` + +`pr-autofix.yml` (untrusted) + `pr-autofix-publish.yml` (trusted) run `prettier --write` and `eslint --fix` against the PR head and surface the diff as inline review-comment suggestions. Three signals are emitted: + +| Surface | Where | Notes | +|---|---|---| +| Sticky PR comment | Top-level comment with the HTML marker `` and heading `## :sparkles: PR Autofix`. Only posted when there is something to fix; clean PRs stay silent. | Edit-in-place via marker; one comment per PR. | +| Fenced JSON block | Inside the sticky, fenced as `gitnexus-autofix`. Schema `gitnexus.pr-autofix/v1` with fields `state` (`suggestions-posted` \| `skipped-too-large`), `pr_number`, `head_sha`, `changed_lines`, `run_id`. | Parseable signal — preferred over regexing prose. | +| Check Run | Stable name `gitnexus/autofix` on the PR head SHA. Conclusion: `success` (clean) or `neutral` (suggestions-posted / skipped-too-large). The output title disambiguates the two `neutral` cases. | Surfaced under PR Checks; readable via `gh pr checks `. | + +To detect outcome from an agent: `gh pr checks --json name,conclusion,output | jq '.[] | select(.name == "gitnexus/autofix")'`. + +Forks are supported. The untrusted half runs fork code with `permissions: {}` and ships the diff as an artifact; the trusted publish job consumes only the diff (data, not code) and posts the comment + check run. + ## AI-assisted contributions If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes. From f26a35b17f4dcec66d6b66cd3aa7bc95b6406f92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 May 2026 13:53:07 +0100 Subject: [PATCH 06/11] chore(deps)(deps): bump @anthropic-ai/sdk (#1442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the npm_and_yarn group with 1 update in the /gitnexus-web directory: [@anthropic-ai/sdk](https://github.com/anthropics/anthropic-sdk-typescript). Updates `@anthropic-ai/sdk` from 0.90.0 to 0.91.1 - [Release notes](https://github.com/anthropics/anthropic-sdk-typescript/releases) - [Changelog](https://github.com/anthropics/anthropic-sdk-typescript/blob/main/CHANGELOG.md) - [Commits](https://github.com/anthropics/anthropic-sdk-typescript/compare/sdk-v0.90.0...sdk-v0.91.1) --- updated-dependencies: - dependency-name: "@anthropic-ai/sdk" dependency-version: 0.91.1 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar --- gitnexus-web/package-lock.json | 24 ++++++++++++------------ gitnexus-web/package.json | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 7fcc0d299..3a560ca5a 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -8,7 +8,7 @@ "name": "gitnexus", "version": "0.0.0", "dependencies": { - "@langchain/anthropic": "^1.3.28", + "@langchain/anthropic": "^1.3.29", "@langchain/core": "^1.1.44", "@langchain/google-genai": "^2.1.28", "@langchain/langgraph": "^1.2.9", @@ -95,9 +95,9 @@ } }, "node_modules/@anthropic-ai/sdk": { - "version": "0.90.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.90.0.tgz", - "integrity": "sha512-MzZtPabJF1b0FTDl6Z6H5ljphPwACLGP13lu8MTiB8jXaW/YXlpOp+Po2cVou3MPM5+f5toyLnul9whKCy7fBg==", + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", "license": "MIT", "dependencies": { "json-schema-to-ts": "^3.1.1" @@ -1396,25 +1396,25 @@ } }, "node_modules/@langchain/anthropic": { - "version": "1.3.28", - "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.28.tgz", - "integrity": "sha512-gOF8oXJL8xDdYes2KXNI9vFm/9TldBBBHOjuCdt27kganVaQKzLvTw5kV6R4mjbnFagV5CWteNH7APLZYCpdwg==", + "version": "1.3.29", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.29.tgz", + "integrity": "sha512-ep1qBIcV07bajsg3fDqMd39rYwoRLOEK/6lk+MCxlm1YB5SRoKKJAZANrblQ/4RYhZJnxf95c6BSQu8VoNbVAQ==", "license": "MIT", "dependencies": { - "@anthropic-ai/sdk": "^0.90.0", + "@anthropic-ai/sdk": "^0.91.1", "zod": "^3.25.76 || ^4" }, "engines": { "node": ">=20" }, "peerDependencies": { - "@langchain/core": "^1.1.42" + "@langchain/core": "^1.1.45" } }, "node_modules/@langchain/core": { - "version": "1.1.44", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.44.tgz", - "integrity": "sha512-RePW1IjGCHr9ua2vcby3aE8mOOz3EnwDZxMEGbNDT91kf14eqkJqxDXvaZFviGdcN9DTrxM5RPQNAHmwSm4tbg==", + "version": "1.1.45", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.45.tgz", + "integrity": "sha512-Y/wvuglLTMKJahkl4QD9dBIdF/z/CxZJWdTfHJF/q2jtlJtoFf6Mb5JpGxZfsi3mBY6NSG941FSLTcqhCKrhBA==", "license": "MIT", "dependencies": { "@cfworker/json-schema": "^4.0.2", diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 81d7fc5fa..51d7b0520 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "gitnexus-shared": "file:../gitnexus-shared", - "@langchain/anthropic": "^1.3.28", + "@langchain/anthropic": "^1.3.29", "@langchain/core": "^1.1.44", "@langchain/google-genai": "^2.1.28", "@langchain/langgraph": "^1.2.9", From 248cb1e634309e7b749458a7f7b1647c2cc71be8 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 14:26:06 +0100 Subject: [PATCH 07/11] Add regression coverage for `.gitnexusignore` behavior with `--skip-git` (#1450) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * test: cover --skip-git with .gitnexusignore regression Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/7ca9afd5-10b9-4f39-a260-60e60bde6874 * test: reuse cli path constant in skip-git tests Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/7ca9afd5-10b9-4f39-a260-60e60bde6874 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Gergő Magyar --- gitnexus/test/unit/skip-git-cli.test.ts | 57 ++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/gitnexus/test/unit/skip-git-cli.test.ts b/gitnexus/test/unit/skip-git-cli.test.ts index e82bb7b13..069a9686f 100644 --- a/gitnexus/test/unit/skip-git-cli.test.ts +++ b/gitnexus/test/unit/skip-git-cli.test.ts @@ -5,9 +5,11 @@ import os from 'os'; import fs from 'fs'; describe('--skip-git CLI flag', () => { + const cliPath = path.resolve(__dirname, '../../dist/cli/index.js'); + it('Commander maps --skip-git to options.skipGit (not --no-git inversion)', () => { // Verify the CLI defines --skip-git and --skip-agents-md in analyze help. - const helpOutput = execSync('node dist/cli/index.js analyze --help', { + const helpOutput = execSync(`node "${cliPath}" analyze --help`, { cwd: path.resolve(__dirname, '../..'), encoding: 'utf8', timeout: 10000, @@ -37,8 +39,59 @@ describe('--skip-git CLI flag', () => { } }); + it('still respects .gitnexusignore when run with --skip-git', () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-skip-git-ignore-')); + const gitnexusHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-skip-git-ignore-home-')); + fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); + fs.mkdirSync(path.join(tmpDir, 'customskip'), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, '.gitnexusignore'), 'customskip/\n'); + fs.writeFileSync(path.join(tmpDir, 'src', 'keep.ts'), 'export function keep() { return 1; }\n'); + fs.writeFileSync( + path.join(tmpDir, 'customskip', 'leaked.ts'), + 'export function leaked() { return 42; }\n', + ); + + const env = { + ...process.env, + HOME: gitnexusHome, + GITNEXUS_HOME: gitnexusHome, + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + }; + + try { + execSync(`node "${cliPath}" analyze "${tmpDir}" --skip-git --skip-agents-md`, { + encoding: 'utf8', + timeout: 60000, + env, + }); + + const keepContext = execSync( + `node "${cliPath}" context keep --repo "${path.basename(tmpDir)}"`, + { + encoding: 'utf8', + timeout: 60000, + env, + }, + ); + expect(keepContext).toContain('"status": "found"'); + expect(keepContext).toContain('"filePath": "src/keep.ts"'); + + const leakedContext = execSync( + `node "${cliPath}" context leaked --repo "${path.basename(tmpDir)}"`, + { + encoding: 'utf8', + timeout: 60000, + env, + }, + ); + expect(leakedContext).toContain(`"error": "Symbol 'leaked' not found"`); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.rmSync(gitnexusHome, { recursive: true, force: true }); + } + }); + describe('--skip-git does not walk up to parent git repo (#1232)', () => { - const cliPath = path.resolve(__dirname, '../../dist/cli/index.js'); let parentDir: string; let gitnexusHome: string; From 152a0506c93ee3930f6a24d34934adbff226dce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 9 May 2026 15:18:09 +0100 Subject: [PATCH 08/11] feat: shared resilient-fetch (retries + circuit breaker) (#1448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: shared resilient-fetch (retries + circuit breaker) Add a small, runtime-agnostic resilience layer in gitnexus-shared and migrate every backend HTTP outbound call (CLI, MCP, wiki LLM, web → backend) through it. Helpers (gitnexus-shared/src/integrations/): - retry.ts — withRetry(fn, opts) with caller-supplied retryability classification and full-jitter exponential backoff. - circuit-breaker.ts — closed/open/half-open per-process breaker with injectable clock, plus a keyed registry so callers targeting the same endpoint share state. - resilient-fetch.ts — composed wrapper: retries 5xx + 429 + retryable network throws, treats AbortSignal.timeout() and 4xx (other than 429) as terminal, honors Retry-After (capped at 30s), throws CircuitOpenError when the breaker opens. Migrations (no behaviour regression — all existing tests pass): - gitnexus/src/core/embeddings/http-client.ts (covers analyze + MCP query path) — replaces inline linear-backoff retry. - gitnexus/src/core/wiki/llm-client.ts — preserves Azure content-filter branch; resilientFetch handles 5xx/429. - gitnexus-web/src/services/backend-client.ts (fetchWithTimeout helper) — small retry budget (2 attempts, 250–1500 ms) so a dead local backend still fails fast for the user. - gitnexus-web/src/core/llm/settings-service.ts (OpenRouter model list). Deliberately not migrated: - gitnexus-web/src/services/backend-client.ts streamJob() — Server-Sent Events stream; the existing reconnect-with-Last-Event-ID logic is not unary-fetch shaped. - gitnexus-web/src/components/SettingsPanel.tsx checkOllamaStatus() — one-shot health probe; retrying delays the "Ollama not running" error rather than improving UX. 41 new helper tests cover backoff math, breaker state transitions, Retry-After parsing (delta-seconds + HTTP-date), 401/422 terminal classification, and breaker fail-fast on three exhausted retry batches. * fix(review): apply autofix feedback Address Claude's two MEDIUM blocking findings on PR #1448 plus the CodeQL SSRF false-positive flag. - backend-client `fetchWithTimeout` now uses `AbortSignal.timeout()` merged with the caller's signal via `AbortSignal.any()`. Timer-fired aborts surface as `DOMException(name='TimeoutError')` so resilientFetch routes them through the terminal-network branch (no retry, no breaker hit), instead of incrementing the breaker for user-side network slowness. - Method-aware retry budget in `fetchWithTimeout`: idempotent verbs (GET/HEAD/OPTIONS) keep the 2-attempt budget; POST/PATCH/PUT/DELETE default to single-attempt so a 5xx on `startAnalyze` cannot start a duplicate job. New `forceRetry` parameter for callers that know-idempotent mutations (e.g. DELETE of a known-deleted resource). - `resilient-fetch.ts` carries a documented suppression for CodeQL js/server-side-request-forgery on the inner fetch call. Every concrete caller passes a hardcoded URL constant or a value from configuration (env vars, saved settings); user request input never flows into the URL parameter. - New test file `backend-client-retry.test.ts` covers all three paths: GET retries on 503, POST does not retry, timeout does not increment the breaker. * fix(resilient-fetch): address Codex adversarial findings Closes the three blocking issues from Codex's review on PR #1448. U1 — Add `recordNeutral()` to CircuitBreaker. Third outcome path that's an explicit no-op for state and the consecutive-failure counter. Distinct from `recordSuccess` (closes the breaker) and `recordFailure` (may open it). Used for outcomes that are neither evidence of backend health nor evidence of backend failure. U2 — Route terminal-client / terminal-network through `recordNeutral`. Previously a 401 or local timeout called `recordSuccess`, which reset `consecutiveFailures` to 0. A 5xx → 401 → 5xx → 401 → 5xx sequence would NEVER trip the breaker because each 4xx in between erased the running count. Also classify external `AbortError` as terminal-network (was retryable-network), so caller-driven cancellation no longer retries against an already-aborted signal or counts toward breaker failures on exhaustion. U3 — Per-origin breaker key in web `fetchWithTimeout`. Was hardcoded to `'web-backend'` even though `_backendUrl` is mutable via `setBackendUrl`. Switching backend URLs after a circuit tripped on host-A would strand the user during the full cooldown. Key is now `web-backend:`, so each backend URL gets its own breaker state. Tests: +5 recordNeutral, +4 resilient-fetch (interleaved 4xx/5xx, external AbortError, prior-state preservation), +1 web switch-backend regression. All 70 gitnexus integration tests + 15 web tests green. * fix(resilient-fetch): tolerate header-less fetch mocks on 429 `classifyOutcome` called `resp.headers.get('Retry-After')` directly, which crashed when a test stubs `fetch` with a plain object like `{ ok: false, status: 429 }` (no `headers` field). Real `Response` always has Headers, so this surfaces only in test setups, but the helper has no business assuming caller-side correctness on this — the defensive guard is cheap and a missing `Retry-After` falls through to exponential-backoff retry like any 429 without the header. Surfaced by `gitnexus/test/unit/http-embedder.test.ts > retries on rate limit`, which the embeddings migration exercises against a plain-object 429 stub. Locked in with a new `classifies 429 from a header-less fetch mock without throwing` case. * fix(review): apply autofix feedback Closes findings from the third multi-agent review pass on PR #1448. #1 (P1) callLLM had no per-attempt timeout Wiki LLM calls passed no `signal` to resilientFetch; each of three retry attempts could hang indefinitely on a frozen TCP connection. Add `signal: AbortSignal.timeout(60_000)` so the per-attempt budget matches what http-client.ts and backend-client.ts already provide. #2 (P2) drop dead `lastRetryableResp` post-loop fallback Variable was set in one switch arm but only read in unreachable code after the loop. The retry loop always returns/throws on every iteration. Keep only the defensive `throw` so TypeScript's control-flow analysis still sees `Promise` as the return. #5 (P2) gate test-only exports behind a subpath `__resetBreakerRegistry__` and `classifyOutcome` were reachable from the main `gitnexus-shared` barrel — production code calling `__resetBreakerRegistry__` from a tool implementation would silently nuke every circuit breaker process-wide. Move to a new `gitnexus-shared/test-helpers` subpath export. Production callers see the cleaner public API; tests import via the explicit `gitnexus-shared/test-helpers` path. #6 (P2) exhaustiveness guard on Outcome switch Add a `default: const _: never = outcome` arm so a future sixth `Outcome.kind` won't compile silently — it'll surface at the switch site rather than fall through to a retry/no-retry default. #9 (P3) document cumulative wall-clock budget Add a "Cumulative wall-clock budget" paragraph to resilientFetch's JSDoc explaining the worst-case total wait (`maxAttempts × (per-attempt timeout + capDelayMs)` ≈ 60s with defaults) and pointing callers at outer `AbortSignal.timeout()` when they want a tighter bound. Deferred to follow-up PRs (per review's Auto-resolve recommendation): - #3 idempotency knob to shared API (forceRetry into ResilientFetchOptions) - #4 publish.ts migration to resilientFetch - #7 parseRetryAfter past-HTTP-date / negative-seconds asymmetry - #8 recordNeutral counter time-decay (documented breaker semantic) * fix(circuit-breaker): gate half-open to a single in-flight probe Closes the Codex adversarial-review finding on PR #1448 that flagged a recovery-time thundering herd: when cooldown expired, every concurrent caller transitioned the breaker to half-open and probed the still- recovering dependency in lockstep, defeating the breaker's "fail fast" promise. U1 — probe-permit gate in CircuitBreaker.check() Added a `probeInFlight: boolean` field. After cooldown expires, the first `check()` admits the probe and consumes the permit; subsequent callers throw `CircuitOpenError` with a configurable `halfOpenRetryAfterMs` (default 1000ms) until the probe resolves. Critical design point: `recordNeutral` now RELEASES the permit but does NOT transition state. Without that split, a single `TimeoutError` from per-attempt `AbortSignal.timeout` (which routes through neutral classification) would permanently park the breaker in half-open. By separating permit-release from state-resolution, we keep the "neutral doesn't claim health" semantic without creating that wedge. Other changes: - `halfOpenRetryAfterMs` is now a constructor option for consumers with long-running protected ops (LLM streaming, large uploads). - `getState()` is documented as a pure read; the implicit Open -> Half-Open transition lives in `check()` only, so tests that inspect state never inadvertently consume a probe permit. - `isProbeInFlight()` test-only accessor for assertion clarity. - JSDoc on `check()` records the JS event-loop atomicity dependency and the load-bearing `try/finally` pairing invariant. U2 — End-to-end concurrency regression through resilientFetch Three new scenarios in resilient-fetch.test.ts (26 -> 29): - 3 concurrent calls + probe gets 200 -> 1 hits fetch, 2 throw CircuitOpenError, breaker closes. - 3 concurrent calls + probe gets 503 -> ResilientFetchExhaustedError on probe; concurrent callers see halfOpenRetryAfterMs (1000ms); fresh caller after probe resolves sees the FULL new cooldown (10000ms), not the probe-in-flight default. - Probe cancelled mid-flight via AbortError -> permit released, state stays half-open, next caller becomes the new probe and succeeds. Plus 9 new circuit-breaker unit tests (16 -> 25) covering the permit gate, recordNeutral-releases-permit semantic, fresh-cooldown distinction, default vs configurable halfOpenRetryAfterMs, getState() purity, and the three-probes-via-neutrals chain. Total integration test count: 70 -> 82. All 106 gitnexus + 15 web tests pass; both packages typecheck. Maintainer decisions (deferred per plan 003 Open Questions): - Plan 002's deferral judgement was reversed on Codex's argument without new measurement / incident data. The reversal is defensible on principle (Hystrix / Resilience4j alignment) but lacks workload- driven evidence. - Probe-blocked callers throw silently (no log / event hook). R4's "no new public API" prevents adding observability; loosen if a debug log on probe-blocked is wanted. * refactor(embeddings): replace bespoke HF breaker with shared CircuitBreaker Deleted the local `HfDownloadCircuitBreaker` class and the manual retry loop in `withHfDownloadRetry`. Both are now backed by the shared `gitnexus-shared` primitives: - `hfDownloadCircuit` is `new CircuitBreaker({ failureThreshold, cooldownMs, key: 'hf-download' })` — same state machine as before PLUS the single-permit half-open gate that prevents recovery-time stampedes when CLI + MCP embedders concurrently re-load the model. - `withHfDownloadRetry` delegates the loop to `withRetry` from the shared package. Per-attempt timeout (`withDownloadTimeout`), network-vs-non-network classification, circuit recording, and the `onRetry` callback wire through `withRetry`'s `isRetryable` callback. Behaviour preserved: - Pre-flight `CIRCUIT_OPEN_TAG` rejection when the breaker is open. - Mid-loop `CIRCUIT_OPEN_TAG` "opened after N consecutive failures" when a network error trips the threshold. - Non-network errors (e.g. CUDA unavailable) bypass retry and go through `recordNeutral` instead of resetting the breaker's failure-count progress. - `onRetry(attempt+1, max, err)` fires only when there's a next attempt, matching the prior semantic. Generic CircuitBreaker gained two inspection accessors: - `getOpenedAt(): number | null` - `getCooldownMs(): number` Used by `withHfDownloadRetry` to compute `secsUntilReset` without consuming a probe permit (which `check()` would do). Test consolidation: the 7 bespoke `HfDownloadCircuitBreaker` state-machine tests in hf-env.test.ts were 1:1 duplicates of existing tests in `circuit-breaker.test.ts` and were deleted. Remaining 42 hf-env tests all pass; full integration sweep (148 gitnexus + 15 web) green. --- gitnexus-shared/package.json | 4 + gitnexus-shared/src/index.ts | 16 + .../src/integrations/circuit-breaker.ts | 273 +++++++++ .../src/integrations/resilient-fetch.ts | 279 +++++++++ gitnexus-shared/src/integrations/retry.ts | 105 ++++ gitnexus-shared/src/test-helpers.ts | 13 + gitnexus-web/src/core/llm/settings-service.ts | 6 +- gitnexus-web/src/services/backend-client.ts | 85 ++- .../test/unit/backend-client-retry.test.ts | 110 ++++ gitnexus/src/core/embeddings/hf-env.ts | 179 +++--- gitnexus/src/core/embeddings/http-client.ts | 64 +- gitnexus/src/core/wiki/llm-client.ts | 146 +++-- gitnexus/test/unit/hf-env.test.ts | 130 +--- .../unit/integrations/circuit-breaker.test.ts | 395 +++++++++++++ .../unit/integrations/resilient-fetch.test.ts | 558 ++++++++++++++++++ gitnexus/test/unit/integrations/retry.test.ts | 128 ++++ 16 files changed, 2173 insertions(+), 318 deletions(-) create mode 100644 gitnexus-shared/src/integrations/circuit-breaker.ts create mode 100644 gitnexus-shared/src/integrations/resilient-fetch.ts create mode 100644 gitnexus-shared/src/integrations/retry.ts create mode 100644 gitnexus-shared/src/test-helpers.ts create mode 100644 gitnexus-web/test/unit/backend-client-retry.test.ts create mode 100644 gitnexus/test/unit/integrations/circuit-breaker.test.ts create mode 100644 gitnexus/test/unit/integrations/resilient-fetch.test.ts create mode 100644 gitnexus/test/unit/integrations/retry.test.ts diff --git a/gitnexus-shared/package.json b/gitnexus-shared/package.json index 7c1e6847a..0a5d7a2db 100644 --- a/gitnexus-shared/package.json +++ b/gitnexus-shared/package.json @@ -10,6 +10,10 @@ ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./test-helpers": { + "types": "./dist/test-helpers.d.ts", + "default": "./dist/test-helpers.js" } }, "scripts": { diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index faf136fe7..3c82658f1 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -143,6 +143,22 @@ export type { ScopeTree } from './scope-resolution/scope-tree.js'; export { buildPositionIndex } from './scope-resolution/position-index.js'; export type { PositionIndex } from './scope-resolution/position-index.js'; +// Resilient fetch primitives — bounded retries + per-process circuit breaker. +// Test-only helpers (`__resetBreakerRegistry__`, `classifyOutcome`) are +// reachable via the separate `gitnexus-shared/test-helpers` subpath; do +// NOT add them here. Production consumers must not call them. +export { withRetry, computeBackoffMs } from './integrations/retry.js'; +export type { RetryOptions, RetryDecision } from './integrations/retry.js'; +export { CircuitBreaker, CircuitOpenError, getBreaker } from './integrations/circuit-breaker.js'; +export type { CircuitBreakerOptions } from './integrations/circuit-breaker.js'; +export { + resilientFetch, + ResilientFetchExhaustedError, + RETRY_AFTER_CAP_MS, + parseRetryAfter, +} from './integrations/resilient-fetch.js'; +export type { ResilientFetchOptions } from './integrations/resilient-fetch.js'; + // Understand-Quickly registry integration (opt-in) export { UNDERSTAND_QUICKLY_DISPATCH_URL, diff --git a/gitnexus-shared/src/integrations/circuit-breaker.ts b/gitnexus-shared/src/integrations/circuit-breaker.ts new file mode 100644 index 000000000..29782a8fb --- /dev/null +++ b/gitnexus-shared/src/integrations/circuit-breaker.ts @@ -0,0 +1,273 @@ +/** + * Per-process circuit breaker. + * + * Closed -> Open transition fires after `failureThreshold` consecutive + * failures. While Open, `check` throws `CircuitOpenError` until + * `cooldownMs` has elapsed since the breaker tripped. The first call + * after the cooldown enters Half-Open and consumes the *probe permit*: + * a recorded success returns to Closed; a recorded failure flips back + * to Open with a fresh timestamp. + * + * Half-open admits exactly one in-flight probe at a time. Concurrent + * callers attempting `check()` while a probe is outstanding receive + * `CircuitOpenError` with `retryAfterMs = halfOpenRetryAfterMs` (default + * 1000ms; configurable). This prevents the recovery-time thundering + * herd that defeats the breaker's "fail fast" promise. + * + * Outcome reporting splits permit-release from state-resolution: + * - `recordSuccess` — releases the probe permit, resets the failure + * counter, transitions to Closed. Reserved for true 2xx/3xx outcomes. + * - `recordFailure` — releases the probe permit, increments the + * consecutive-failure counter, transitions to Open with a fresh + * `openedAt` (when called from Half-Open or when the threshold + * trips from Closed). + * - `recordNeutral` — releases the probe permit, BUT leaves state and + * counter untouched. Used for outcomes that are neither evidence of + * backend health nor evidence of backend failure (caller-driven + * cancellation, local timeout, terminal 4xx client errors). Critical + * design point: if `recordNeutral` did not release the permit, a + * single `TimeoutError` from per-attempt `AbortSignal.timeout` would + * route through `recordNeutral` and permanently park the breaker in + * half-open until process restart. Releasing the permit while leaving + * state half-open keeps the "neutral doesn't claim health" semantic + * without creating that wedge. + * + * Pairing invariant: every successful `check()` MUST be paired with + * exactly one `record*()` on every code path including throws. Direct + * consumers should wrap the protected operation in `try/finally`: + * + * breaker.check(); + * try { + * const result = await operation(); + * breaker.recordSuccess(); + * return result; + * } catch (err) { + * // classify err and call recordFailure / recordNeutral / etc. + * throw err; + * } + * + * `resilientFetch`'s catch-all on `fetchImpl` already satisfies this + * for that consumer. + * + * Atomicity model: the half-open gate relies on JavaScript event-loop + * single-threadedness within a synchronous `check()` body. There is no + * `await` inside `check()`; concurrent callers serialize on microtask + * order, and exactly one observes `probeInFlight === false`. Do not + * introduce `await` inside `check()` without revisiting the gate. If + * this code is ever ported to a runtime with shared-memory threads + * (Node `worker_threads` with `SharedArrayBuffer`, Web Workers with + * shared registries), the boolean must become an atomic CAS — Resilience4j + * and Hystrix use atomic permits *because* they run in JVM thread pools. + * + * Runtime-agnostic: depends only on a `now()` clock and standard JS — + * no Node-only imports. Tests inject `now` to advance the clock + * deterministically without `vi.useFakeTimers()`. + */ + +export class CircuitOpenError extends Error { + override readonly name = 'CircuitOpenError'; + /** Approximate wait time before the breaker may transition to Half-Open + * (or before the in-flight probe is expected to resolve). */ + readonly retryAfterMs: number; + + constructor(retryAfterMs: number, key?: string) { + super( + key + ? `Circuit '${key}' is open; retry in ${Math.ceil(retryAfterMs / 1000)}s` + : `Circuit is open; retry in ${Math.ceil(retryAfterMs / 1000)}s`, + ); + this.retryAfterMs = retryAfterMs; + } +} + +export interface CircuitBreakerOptions { + /** Consecutive failures required to trip Closed -> Open. */ + failureThreshold?: number; + /** Milliseconds Open before the next call may probe (Half-Open). */ + cooldownMs?: number; + /** + * Milliseconds to suggest in `CircuitOpenError.retryAfterMs` when the + * breaker is Half-Open with the probe permit consumed. Default 1000ms. + * Consumers with long-running protected ops (LLM streaming, large + * uploads) should raise this — the cooldown clock is no longer the + * right answer because cooldown has elapsed. Returning 0 invites + * retry storms; returning the full cooldown misleads about wait. + */ + halfOpenRetryAfterMs?: number; + /** Optional key for error messages and registry lookups. */ + key?: string; + /** Clock override — defaults to `Date.now`. Tests inject deterministic time. */ + now?: () => number; +} + +type State = 'closed' | 'open' | 'half-open'; + +export class CircuitBreaker { + private readonly failureThreshold: number; + private readonly cooldownMs: number; + private readonly halfOpenRetryAfterMs: number; + private readonly key: string | undefined; + private readonly now: () => number; + + private state: State = 'closed'; + private consecutiveFailures = 0; + private openedAt: number | null = null; + /** + * True between a successful `check()` and the next `record*()` call + * during Half-Open. Gates concurrent callers from stampeding a still- + * recovering dependency. Boolean rather than counter — single-permit + * is the conservative end of the Hystrix/Resilience4j spectrum. + */ + private probeInFlight = false; + + constructor(opts: CircuitBreakerOptions = {}) { + this.failureThreshold = opts.failureThreshold ?? 3; + this.cooldownMs = opts.cooldownMs ?? 30_000; + this.halfOpenRetryAfterMs = opts.halfOpenRetryAfterMs ?? 1_000; + this.key = opts.key; + this.now = opts.now ?? (() => Date.now()); + } + + /** + * Throw `CircuitOpenError` if the breaker won't admit this call. + * Otherwise consume the half-open probe permit (if applicable) and + * return so the caller can attempt the protected work. + * + * Three rejection paths: + * 1. Open and still in cooldown → throws with `retryAfterMs` = + * remaining cooldown. + * 2. Open with cooldown elapsed AND a probe is already in flight + * (race: another caller transitioned to half-open and grabbed + * the permit on a microtask before us) → throws with + * `halfOpenRetryAfterMs`. + * 3. Half-Open with probe in flight → throws with `halfOpenRetryAfterMs`. + * + * **Pairing invariant**: every successful return from `check()` MUST + * be paired with exactly one `recordSuccess` / `recordFailure` / + * `recordNeutral` on every code path including thrown exceptions. + * Failing to pair leaves the probe permit consumed forever and + * wedges the breaker. See file-header JSDoc for the canonical + * try/finally pattern. + */ + check(): void { + if (this.state === 'open' && this.openedAt !== null) { + const elapsed = this.now() - this.openedAt; + if (elapsed < this.cooldownMs) { + throw new CircuitOpenError(this.cooldownMs - elapsed, this.key); + } + // Cooldown elapsed — transition to Half-Open. The very next + // `probeInFlight` check below decides whether THIS caller gets + // the permit or hits the gate. + this.state = 'half-open'; + } + + if (this.state === 'half-open') { + if (this.probeInFlight) { + throw new CircuitOpenError(this.halfOpenRetryAfterMs, this.key); + } + this.probeInFlight = true; + } + // Closed state falls through silently. + } + + recordSuccess(): void { + this.probeInFlight = false; + this.consecutiveFailures = 0; + this.state = 'closed'; + this.openedAt = null; + } + + recordFailure(): void { + this.probeInFlight = false; + this.consecutiveFailures += 1; + if (this.state === 'half-open' || this.consecutiveFailures >= this.failureThreshold) { + this.state = 'open'; + this.openedAt = this.now(); + } + } + + /** + * Releases the probe permit BUT leaves state and counter untouched. + * Use when an attempt produced a response or error that should not + * influence breaker health in either direction — caller-driven aborts, + * local AbortSignal timeouts, terminal 4xx client errors. + * + * Why permit-release-without-state-resolution: if `recordNeutral` did + * not clear `probeInFlight`, a single `TimeoutError` from per-attempt + * `AbortSignal.timeout` (which routes through neutral classification) + * would permanently park the breaker in half-open. Since timeouts are + * an *expected* outcome under flaky-dependency conditions, the cited + * "per-attempt timeout bounds the stuck state" mitigation would itself + * be the trigger for a permanent wedge. Releasing the permit closes + * that loop while keeping the "neutral doesn't claim dependency + * health" semantic. + * + * Calling `recordSuccess` for these would erase legitimate prior + * failure signal; calling `recordFailure` would trip the breaker for + * outcomes the backend isn't responsible for. + */ + recordNeutral(): void { + this.probeInFlight = false; + // State and consecutiveFailures are preserved by design. + } + + /** + * Pure read — no state mutation, no permit accounting. Returns the + * *would-be* state at the current instant: 'half-open' if the breaker + * is open with cooldown elapsed (regardless of whether a probe is in + * flight), 'open' if open and still in cooldown, 'closed' otherwise. + * + * Inspection-only; safe to call from tests without consuming a probe + * permit. The implicit Open -> Half-Open transition that mutates + * `state` lives in `check()` only. + */ + getState(): State { + if (this.state === 'open' && this.openedAt !== null) { + const elapsed = this.now() - this.openedAt; + if (elapsed >= this.cooldownMs) return 'half-open'; + } + return this.state; + } + getConsecutiveFailures(): number { + return this.consecutiveFailures; + } + /** Inspection-only test accessor for the half-open probe permit. */ + isProbeInFlight(): boolean { + return this.probeInFlight; + } + /** Timestamp (ms since epoch) when the breaker last transitioned to Open, + * or `null` if it's currently Closed. Useful for computing remaining + * cooldown without consuming a probe permit via `check()`. */ + getOpenedAt(): number | null { + return this.openedAt; + } + /** Configured cooldown duration in milliseconds. */ + getCooldownMs(): number { + return this.cooldownMs; + } +} + +// ─── Per-process registry ──────────────────────────────────────────── +// +// Single shared map keyed on caller-chosen strings. Used by +// `resilient-fetch.ts` so multiple call sites targeting the same logical +// endpoint share breaker state. Per-process only — not persisted. + +const registry = new Map(); + +export function getBreaker(key: string, opts?: CircuitBreakerOptions): CircuitBreaker { + let breaker = registry.get(key); + if (!breaker) { + breaker = new CircuitBreaker({ ...opts, key }); + registry.set(key, breaker); + } + return breaker; +} + +/** + * Test-only: clear all registered breakers. Tests must call this in + * `beforeEach` to prevent breaker state from leaking across test cases. + */ +export function __resetBreakerRegistry__(): void { + registry.clear(); +} diff --git a/gitnexus-shared/src/integrations/resilient-fetch.ts b/gitnexus-shared/src/integrations/resilient-fetch.ts new file mode 100644 index 000000000..c91b9db3a --- /dev/null +++ b/gitnexus-shared/src/integrations/resilient-fetch.ts @@ -0,0 +1,279 @@ +/** + * `resilientFetch` — fetch wrapped in retry + circuit breaker, with + * GitHub-flavoured retry classification baked in (Retry-After parsing, + * 401/403/404/422 treated as terminal client errors). + * + * Designed for the `gitnexus publish` GitHub `repository_dispatch` + * call, but the classification rules apply to any GitHub REST endpoint. + * Runtime-agnostic — no Node-only imports. + */ + +import { + CircuitBreaker, + CircuitOpenError, + getBreaker, + type CircuitBreakerOptions, +} from './circuit-breaker.js'; +import { computeBackoffMs, type RetryOptions } from './retry.js'; + +export { CircuitOpenError }; + +export interface ResilientFetchOptions { + /** Optional fetch implementation override. Defaults to `globalThis.fetch`. */ + fetchImpl?: typeof fetch; + /** + * Logical key for the breaker. Defaults to `` of the + * request URL — call sites targeting the same endpoint share breaker + * state regardless of query-string differences. + */ + breakerKey?: string; + /** Per-call breaker override. Used for tests and one-off configuration. */ + breaker?: CircuitBreaker; + /** Tuning knobs for the breaker registered under `breakerKey`. */ + breakerOptions?: CircuitBreakerOptions; + /** Tuning knobs for the retry helper. */ + retry?: Partial> & { + sleep?: RetryOptions['sleep']; + random?: RetryOptions['random']; + }; + /** Clock override propagated into Retry-After HTTP-date math and breaker. */ + now?: () => number; +} + +/** Cap on any single Retry-After wait — protects CLI from a buggy registry. */ +export const RETRY_AFTER_CAP_MS = 30_000; + +const DEFAULT_RETRY = { + maxAttempts: 3, + baseDelayMs: 500, + capDelayMs: 5_000, +}; + +/** + * Parse a `Retry-After` header value into milliseconds. + * Accepts either a delta-seconds integer (`"30"`) or an HTTP-date. + * Returns null on parse failure or negative deltas. + */ +export function parseRetryAfter(value: string | null, now: () => number = Date.now): number | null { + if (!value) return null; + const trimmed = value.trim(); + if (trimmed === '') return null; + + if (/^[0-9]+$/.test(trimmed)) { + const seconds = parseInt(trimmed, 10); + if (Number.isNaN(seconds) || seconds < 0) return null; + return seconds * 1000; + } + + const target = Date.parse(trimmed); + if (Number.isNaN(target)) return null; + const delta = target - now(); + return delta >= 0 ? delta : 0; +} + +/** Internal: outcome classification used by the resilientFetch loop. */ +type Outcome = + | { kind: 'success'; resp: Response } + | { kind: 'terminal-client'; resp: Response } // 4xx other than 429: no retry, breaker neutral + | { kind: 'retryable-status'; resp: Response; afterMs: number | undefined } // 5xx, 429 + | { kind: 'terminal-network'; err: unknown } // TimeoutError or AbortError: no retry, breaker neutral + | { kind: 'retryable-network'; err: unknown }; // DNS, ECONNRESET, etc. + +/** Exported for unit tests. */ +export function classifyOutcome( + result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response }, + now: () => number, +): Outcome { + if (result.kind === 'error') { + // Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`) + // and caller-driven aborts (`AbortController.abort()` → `AbortError`) + // are terminal: retrying against an already-aborted signal would + // fail again immediately, and neither outcome reflects backend + // health. They route through the breaker's neutral path. + if ( + result.err instanceof DOMException && + (result.err.name === 'TimeoutError' || result.err.name === 'AbortError') + ) { + return { kind: 'terminal-network', err: result.err }; + } + return { kind: 'retryable-network', err: result.err }; + } + const resp = result.resp; + if (resp.status >= 200 && resp.status < 400) return { kind: 'success', resp }; + if (resp.status === 429) { + // `resp.headers` is always present on a real `Response`, but tests + // sometimes stub `fetch` with a plain `{ ok, status }` object. Be + // defensive — a missing `Retry-After` falls through to exponential + // backoff, which is the correct behaviour anyway. + const retryAfterHeader = + typeof resp.headers?.get === 'function' ? resp.headers.get('Retry-After') : null; + const parsed = parseRetryAfter(retryAfterHeader, now); + return { + kind: 'retryable-status', + resp, + afterMs: parsed !== null ? Math.min(parsed, RETRY_AFTER_CAP_MS) : undefined, + }; + } + if (resp.status >= 500) return { kind: 'retryable-status', resp, afterMs: undefined }; + return { kind: 'terminal-client', resp }; +} + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +function defaultBreakerKey(input: string | URL): string { + try { + const url = typeof input === 'string' ? new URL(input) : input; + return `${url.host}${url.pathname}`; + } catch { + return String(input); + } +} + +/** Final error thrown when retries are exhausted on a 5xx / 429. */ +export class ResilientFetchExhaustedError extends Error { + override readonly name = 'ResilientFetchExhaustedError'; + constructor(public readonly response: Response) { + super(`Request failed after retries (HTTP ${response.status})`); + } +} + +/** + * Wrap `fetch` with bounded retries and a per-process circuit breaker. + * + * Semantics: + * - 5xx and 429 responses are retried; 429 honors `Retry-After` (capped). + * - Network throws are retried unless they are `TimeoutError` DOMExceptions. + * - Timeouts and 4xx (other than 429) are returned/thrown without retry + * AND without incrementing the breaker — they reflect caller config + * or local network state, not registry health. + * - Each `fetch` call carries the caller-supplied `signal` (e.g. an + * `AbortSignal.timeout()`) — that timeout bounds each individual + * attempt, not the whole retry sequence. + * - When the breaker is open, throws `CircuitOpenError` synchronously + * without invoking `fetch`. + * - When retries are exhausted on a 5xx / 429, throws + * `ResilientFetchExhaustedError` carrying the last response. + * + * Cumulative wall-clock budget: + * maxAttempts × (per-attempt-timeout + capDelayMs) + * With defaults (3, 500ms base, 5000ms cap) and a typical 15s per-attempt + * timeout from the caller's signal, worst case is ~3 × (15s + 5s) = 60s. + * Callers that want a tighter total bound should reduce `maxAttempts` or + * wrap `resilientFetch` in their own outer `AbortSignal.timeout()`. + */ +export async function resilientFetch( + input: string | URL, + init: RequestInit | undefined, + opts: ResilientFetchOptions = {}, +): Promise { + const fetchImpl = opts.fetchImpl ?? globalThis.fetch; + const now = opts.now ?? (() => Date.now()); + const breaker = + opts.breaker ?? getBreaker(opts.breakerKey ?? defaultBreakerKey(input), opts.breakerOptions); + + const retryConfig = { + maxAttempts: opts.retry?.maxAttempts ?? DEFAULT_RETRY.maxAttempts, + baseDelayMs: opts.retry?.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs, + capDelayMs: opts.retry?.capDelayMs ?? DEFAULT_RETRY.capDelayMs, + }; + const sleep = opts.retry?.sleep ?? defaultSleep; + const random = opts.retry?.random ?? Math.random; + + // Fail fast on an open breaker, before invoking fetch. + breaker.check(); + + for (let attempt = 0; attempt < retryConfig.maxAttempts; attempt++) { + let result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response }; + try { + // CodeQL js/server-side-request-forgery — flagged because `input` + // is caller-supplied. Suppressed: every concrete caller passes + // either a hardcoded URL constant (UNDERSTAND_QUICKLY_DISPATCH_URL, + // OpenRouter base URL) or a value derived from configuration + // (env vars, saved settings, the local backend URL). User-input + // request fields (e.g. PR title, repo name) never flow into + // `input`. Validating URL shape here would push false-positive + // rejection onto every caller — wrong layer for the check. + // lgtm[js/server-side-request-forgery] + // codeql[js/server-side-request-forgery] + const resp = await fetchImpl(input, init); + result = { kind: 'response', resp }; + } catch (err) { + result = { kind: 'error', err }; + } + + const outcome = classifyOutcome(result, now); + + switch (outcome.kind) { + case 'success': + breaker.recordSuccess(); + return outcome.resp; + + case 'terminal-client': + // 4xx: do not count as breaker failure (the server is healthy + // and rejecting our request — auth, scope, or routing). But + // also do NOT call recordSuccess: a 401 sandwiched between + // 5xx responses would otherwise erase the running outage + // signal. The breaker's neutral path leaves state untouched. + breaker.recordNeutral(); + return outcome.resp; + + case 'terminal-network': + // Either `AbortSignal.timeout()` fired locally OR an external + // caller cancelled the request via AbortController. The server + // never had a chance to answer; this reflects the user's + // network or an explicit cancel, not registry health. Don't + // punish the breaker AND don't reset its outage signal. + breaker.recordNeutral(); + throw outcome.err; + + case 'retryable-status': + if (attempt + 1 >= retryConfig.maxAttempts) { + breaker.recordFailure(); + throw new ResilientFetchExhaustedError(outcome.resp); + } + await sleep( + computeBackoffMs( + attempt, + retryConfig.baseDelayMs, + retryConfig.capDelayMs, + outcome.afterMs, + random, + ), + ); + break; + + case 'retryable-network': + if (attempt + 1 >= retryConfig.maxAttempts) { + breaker.recordFailure(); + throw outcome.err; + } + await sleep( + computeBackoffMs( + attempt, + retryConfig.baseDelayMs, + retryConfig.capDelayMs, + undefined, + random, + ), + ); + break; + + default: { + // Exhaustiveness guard. If a sixth `Outcome` kind is added in + // future, TypeScript will refuse to assign it to `never` and + // this line forces the maintainer to add an explicit arm + // rather than silently fall through to retry/no-retry behaviour. + const _exhaustive: never = outcome; + throw new Error(`resilientFetch: unhandled outcome ${JSON.stringify(_exhaustive)}`); + } + } + } + + // Unreachable: every iteration of the loop either returns (success + // / terminal-client) or throws (terminal-network / retry exhaustion). + // The throw is here purely so TypeScript's control-flow analysis sees + // the function never falls off the end without producing `Promise`. + /* c8 ignore next 2 */ + throw new Error('resilientFetch: retry loop terminated unexpectedly'); +} diff --git a/gitnexus-shared/src/integrations/retry.ts b/gitnexus-shared/src/integrations/retry.ts new file mode 100644 index 000000000..774ca2542 --- /dev/null +++ b/gitnexus-shared/src/integrations/retry.ts @@ -0,0 +1,105 @@ +/** + * Bounded retry helper with full-jitter exponential backoff. + * + * Runtime-agnostic: depends only on `setTimeout`, `Math.random`, and the + * Promise machinery — no Node-only imports. Safe to consume from CLI, + * server, or browser callers. + * + * Pattern reference: gitnexus/src/core/embeddings/http-client.ts. This + * helper is the upgraded form: classification is caller-supplied (so + * 4xx-vs-5xx-vs-timeout decisions live with the protocol that knows + * them), backoff is exponential with full jitter, and an optional + * `afterMs` lets callers honor `Retry-After` headers. + */ + +export interface RetryOptions { + /** Initial delay before the first retry attempt, in milliseconds. */ + baseDelayMs: number; + /** Upper bound on any single delay, in milliseconds. */ + capDelayMs: number; + /** Total attempts including the first call. Must be >= 1. */ + maxAttempts: number; + /** + * Decide whether to retry after a thrown error. + * Return `{retry:false}` to terminate immediately and rethrow. + * Return `{retry:true}` to retry with exponential-backoff jitter. + * Return `{retry:true, afterMs}` to wait at least `afterMs` (still + * subject to `capDelayMs`) — used by callers parsing `Retry-After`. + */ + isRetryable: (err: unknown, attempt: number) => RetryDecision; + /** Sleep override — defaults to `setTimeout`. Tests inject fake timers. */ + sleep?: (ms: number) => Promise; + /** Random override — defaults to `Math.random`. Tests inject seeded values. */ + random?: () => number; +} + +export type RetryDecision = { retry: false } | { retry: true; afterMs?: number }; + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Compute the delay before the next retry attempt. + * + * - When the caller specifies `afterMs` (e.g., from `Retry-After`), use + * `min(afterMs, capDelayMs)` so a misbehaving server can't pin the + * client for an arbitrarily long wait. + * - Otherwise compute full-jitter exponential backoff: + * `random() * min(cap, base * 2^attempt)`. Full jitter (rather than + * "equal jitter") avoids retry-storm thundering herd, per AWS + * guidance on backoff strategies. + */ +export function computeBackoffMs( + attempt: number, + baseDelayMs: number, + capDelayMs: number, + afterMs: number | undefined, + random: () => number, +): number { + if (afterMs !== undefined) { + return Math.min(Math.max(0, afterMs), capDelayMs); + } + const exponential = baseDelayMs * Math.pow(2, attempt); + const upper = Math.min(capDelayMs, exponential); + return Math.floor(random() * upper); +} + +/** + * Execute `fn` with bounded retries. + * + * The classification of "retryable" is the caller's responsibility — see + * `resilient-fetch.ts` for the GitHub-dispatch-specific rules. This + * helper is the mechanical retry loop only. + */ +export async function withRetry( + fn: (attempt: number) => Promise, + opts: RetryOptions, +): Promise { + if (opts.maxAttempts < 1) { + throw new Error(`withRetry: maxAttempts must be >= 1, got ${opts.maxAttempts}`); + } + const sleep = opts.sleep ?? defaultSleep; + const random = opts.random ?? Math.random; + + let lastError: unknown; + for (let attempt = 0; attempt < opts.maxAttempts; attempt++) { + try { + return await fn(attempt); + } catch (err) { + lastError = err; + const decision = opts.isRetryable(err, attempt); + if (!decision.retry) throw err; + // Don't sleep after the final attempt. + if (attempt + 1 >= opts.maxAttempts) break; + const delayMs = computeBackoffMs( + attempt, + opts.baseDelayMs, + opts.capDelayMs, + decision.afterMs, + random, + ); + if (delayMs > 0) await sleep(delayMs); + } + } + throw lastError; +} diff --git a/gitnexus-shared/src/test-helpers.ts b/gitnexus-shared/src/test-helpers.ts new file mode 100644 index 000000000..a92441878 --- /dev/null +++ b/gitnexus-shared/src/test-helpers.ts @@ -0,0 +1,13 @@ +/** + * Test-only helpers. + * + * Symbols here are reachable from `gitnexus-shared/test-helpers` so test + * suites can reset shared registries or exercise internal classifiers, + * but they are deliberately NOT re-exported from the main `gitnexus-shared` + * barrel. Production consumers should never import this module — calling + * `__resetBreakerRegistry__()` from a tool implementation would silently + * nuke every circuit breaker process-wide. + */ + +export { __resetBreakerRegistry__ } from './integrations/circuit-breaker.js'; +export { classifyOutcome } from './integrations/resilient-fetch.js'; diff --git a/gitnexus-web/src/core/llm/settings-service.ts b/gitnexus-web/src/core/llm/settings-service.ts index 5e49cb7af..86330d2a5 100644 --- a/gitnexus-web/src/core/llm/settings-service.ts +++ b/gitnexus-web/src/core/llm/settings-service.ts @@ -20,6 +20,7 @@ import { ProviderConfig, } from './types'; import { DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OLLAMA_BASE_URL } from '../../config/ui-constants'; +import { resilientFetch } from 'gitnexus-shared'; const STORAGE_KEY = 'gitnexus-llm-settings'; @@ -407,7 +408,10 @@ export const getAvailableModels = (provider: LLMProvider): string[] => { */ export const fetchOpenRouterModels = async (): Promise> => { try { - const response = await fetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`); + const response = await resilientFetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`, undefined, { + breakerKey: 'openrouter-models', + retry: { maxAttempts: 2, baseDelayMs: 500, capDelayMs: 2_000 }, + }); if (!response.ok) throw new Error('Failed to fetch models'); const data = await response.json(); return data.data.map((model: any) => ({ diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index ec8c1a964..506d48f38 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -7,6 +7,7 @@ */ import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; +import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; // ── Types ────────────────────────────────────────────────────────────────── @@ -237,29 +238,91 @@ export function normalizeServerUrl(input: string): string { const DEFAULT_TIMEOUT_MS = 30_000; const PROBE_TIMEOUT_MS = 2_000; +/** Idempotent HTTP methods. Other verbs (POST, PATCH, PUT, DELETE) get + * a single-attempt retry budget by default to avoid duplicate side + * effects on retry — a POST that 5xx'd may have already executed + * server-side. Callers that have idempotency keys or otherwise know + * their mutation is safe to retry can opt in via `forceRetry`. */ +const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']); + const fetchWithTimeout = async ( url: string, init: RequestInit = {}, timeoutMs: number = DEFAULT_TIMEOUT_MS, + /** + * Force a retry budget on non-idempotent methods. Default false. + * Pass true only when the endpoint is known-idempotent (e.g. DELETE + * of a known-deleted resource — second call is a 404 / no-op) AND + * the duplicate-side-effect window is acceptable. + */ + forceRetry = false, ): Promise => { - const controller = new AbortController(); - // Merge external signal if provided + // Merge the external caller signal (if any) with an + // `AbortSignal.timeout()` so a timer-fired abort produces a + // `DOMException` with `name === 'TimeoutError'` — which + // `resilientFetch` correctly classifies as terminal-network (no + // retry, no breaker hit). A manual `AbortController.abort()` would + // produce `name === 'AbortError'` and route through the + // retryable-network branch, which mis-penalizes the breaker for + // user-side network slowness. + const timeoutSignal = AbortSignal.timeout(timeoutMs); const externalSignal = init.signal; - if (externalSignal) { - externalSignal.addEventListener('abort', () => controller.abort()); + const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal; + + const method = (init.method ?? 'GET').toUpperCase(); + const isIdempotent = IDEMPOTENT_METHODS.has(method); + const maxAttempts = isIdempotent || forceRetry ? 2 : 1; + + // Key the breaker by the current backend origin so switching backend + // URLs (e.g. recovering from a flapping local server by pointing at + // a different host) gives the new origin a fresh breaker state. A + // single shared `'web-backend'` key would otherwise leave a user + // locked out for the full cooldown after one bad host trips the + // circuit. The malformed-URL fallback is defensive — `setBackendUrl` + // normalizes input, so this branch shouldn't fire in practice. + let breakerKey: string; + try { + breakerKey = `web-backend:${new URL(_backendUrl).origin}`; + } catch { + breakerKey = 'web-backend:invalid'; } - const timer = setTimeout(() => controller.abort(), timeoutMs); try { - const response = await fetch(url, { ...init, signal: controller.signal }); + // Bounded retries + 5xx/429 handling are delegated to resilientFetch. + // Method-aware budget: idempotent verbs retry once on transient + // backend failures; mutations (POST/PATCH/PUT/DELETE) default to + // single-attempt to avoid duplicate side effects. + const response = await resilientFetch( + url, + { ...init, signal }, + { + breakerKey, + retry: { maxAttempts, baseDelayMs: 250, capDelayMs: 1500 }, + }, + ); return response; } catch (error: unknown) { - if (error instanceof DOMException && error.name === 'AbortError') { - if (externalSignal?.aborted) { - throw new BackendError('Request aborted', 0, 'network'); - } + if (error instanceof CircuitOpenError) { + throw new BackendError( + `GitNexus backend at ${_backendUrl} is unhealthy; retry in ${Math.ceil(error.retryAfterMs / 1000)}s`, + 0, + 'network', + ); + } + if (error instanceof ResilientFetchExhaustedError) { + // Fall through to caller — surface the raw response so assertOk + // can craft the BackendError with the right code. + return error.response; + } + if (error instanceof DOMException && error.name === 'TimeoutError') { throw new BackendError(`Request to ${url} timed out after ${timeoutMs}ms`, 0, 'timeout'); } + if (error instanceof DOMException && error.name === 'AbortError') { + // External caller-driven cancellation — `timeoutSignal` would + // have surfaced as TimeoutError above, so this branch covers + // only the externally-aborted case. + throw new BackendError('Request aborted', 0, 'network'); + } if (error instanceof TypeError) { throw new BackendError( `Network error reaching GitNexus backend at ${_backendUrl}: ${error.message}`, @@ -268,8 +331,6 @@ const fetchWithTimeout = async ( ); } throw error; - } finally { - clearTimeout(timer); } }; diff --git a/gitnexus-web/test/unit/backend-client-retry.test.ts b/gitnexus-web/test/unit/backend-client-retry.test.ts new file mode 100644 index 000000000..ea0fcf3a7 --- /dev/null +++ b/gitnexus-web/test/unit/backend-client-retry.test.ts @@ -0,0 +1,110 @@ +/** + * Method-aware retry budget + timeout-as-TimeoutError verification for + * backend-client's `fetchWithTimeout`. + * + * Closes review findings on PR #1448: + * - Non-idempotent POST/DELETE must NOT be retried by default — + * a 5xx on `startAnalyze` could otherwise start a duplicate job. + * - Timer-fired timeout must surface as `DOMException(name='TimeoutError')`, + * not `AbortError`, so resilientFetch routes it through the + * terminal-network branch (no retry, no breaker hit). + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getBreaker } from 'gitnexus-shared'; +import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers'; +import { fetchRepos, setBackendUrl, startAnalyze } from '../../src/services/backend-client'; + +const BASE = 'http://localhost:4747'; + +describe('backend-client retry budget (method-aware)', () => { + beforeEach(() => { + __resetBreakerRegistry__(); + setBackendUrl(BASE); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('GET retries once on transient 503 (idempotent verb)', async () => { + let n = 0; + const fetchMock = vi.fn(async () => { + n += 1; + if (n === 1) return new Response('boom', { status: 503 }); + return new Response('[]', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const repos = await fetchRepos(); + expect(repos).toEqual([]); + // 1 retry budget on idempotent GET → 2 total fetch calls. + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('POST does NOT retry on 503 by default (non-idempotent verb)', async () => { + const fetchMock = vi.fn(async () => new Response('boom', { status: 503 })); + vi.stubGlobal('fetch', fetchMock); + + await expect(startAnalyze({ path: '/tmp/repo' })).rejects.toBeTruthy(); + // Single attempt — never duplicates a job-start POST. + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('switching backend URL after a circuit opens reaches a fresh breaker (U3)', async () => { + // Pre-open the breaker for host-A by directly recording 3 failures. + setBackendUrl('http://host-a.test:4747'); + const aKey = 'web-backend:http://host-a.test:4747'; + const breakerA = getBreaker(aKey); + breakerA.recordFailure(); + breakerA.recordFailure(); + breakerA.recordFailure(); + expect(breakerA.getState()).toBe('open'); + + // Switch to host-B and make a request — must succeed against the + // new origin without tripping the host-A circuit. Under the old + // single-key behaviour the call would throw CircuitOpenError. + setBackendUrl('http://host-b.test:4747'); + const fetchMock = vi.fn( + async () => + new Response('[]', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const repos = await fetchRepos(); + expect(repos).toEqual([]); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // Host-A's breaker is still open in cooldown. + expect(breakerA.getState()).toBe('open'); + // Host-B has its own (fresh) breaker. + const bKey = 'web-backend:http://host-b.test:4747'; + expect(getBreaker(bKey).getState()).toBe('closed'); + expect(getBreaker(bKey).getConsecutiveFailures()).toBe(0); + }); + + it('breaker not incremented when timeout fires (TimeoutError, not AbortError)', async () => { + // Reject directly with a TimeoutError DOMException, mimicking what + // `fetch` produces when its `AbortSignal.timeout()`-wired signal + // fires. The real-fetch path goes signal.reason → reject(reason); + // we shortcut that here so the test doesn't have to wait the + // 30-second default timeout. + const fetchMock = vi.fn(async () => { + throw new DOMException('aborted by timeout', 'TimeoutError'); + }); + vi.stubGlobal('fetch', fetchMock); + + await expect(fetchRepos()).rejects.toMatchObject({ code: 'timeout' }); + + // The breaker must not have been penalized for a local timeout. + expect(getBreaker(`web-backend:${BASE}`).getConsecutiveFailures()).toBe(0); + // Timeout is terminal — no retry attempted. + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/src/core/embeddings/hf-env.ts b/gitnexus/src/core/embeddings/hf-env.ts index 95548fff2..5ae6d89ae 100644 --- a/gitnexus/src/core/embeddings/hf-env.ts +++ b/gitnexus/src/core/embeddings/hf-env.ts @@ -1,6 +1,8 @@ import os from 'node:os'; import { join } from 'node:path'; +import { CircuitBreaker, withRetry } from 'gitnexus-shared'; + // --------------------------------------------------------------------------- // Download resilience defaults // --------------------------------------------------------------------------- @@ -108,70 +110,19 @@ export function isNetworkFetchError(message: string): boolean { /** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */ export const CIRCUIT_OPEN_TAG = 'hf-circuit-open'; -/** Circuit-breaker states. */ -type CircuitState = 'closed' | 'open' | 'half-open'; - /** - * Circuit breaker for HuggingFace model downloads. - * - * After `failureThreshold` consecutive network failures the circuit opens and - * all subsequent calls to `withHfDownloadRetry` fail immediately without - * issuing any network requests. After `resetTimeoutMs` the circuit enters the - * half-open state and the next call is attempted — if it succeeds the circuit - * closes again; if it fails the circuit re-opens. - * - * Exported for unit-testing; production code should use the module-level - * `hfDownloadCircuit` singleton. + * Module-level singleton shared by both embedder entry points + * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Per-process + * only — not persisted across restarts. Backed by the shared + * `CircuitBreaker` from `gitnexus-shared` (same state machine, same + * semantics, plus the single-permit half-open gate that prevents + * recovery-time stampedes). */ -export class HfDownloadCircuitBreaker { - private _state: CircuitState = 'closed'; - private _failures = 0; - /** Timestamp of the last recorded failure (ms since epoch). */ - lastFailureAt = 0; - - constructor( - readonly failureThreshold: number = CB_FAILURE_THRESHOLD, - readonly resetTimeoutMs: number = CB_RESET_TIMEOUT_MS, - ) {} - - /** Effective state, factoring in the reset-timeout transition. */ - get state(): CircuitState { - if (this._state === 'open' && Date.now() - this.lastFailureAt > this.resetTimeoutMs) { - this._state = 'half-open'; - } - return this._state; - } - - /** Returns true when the circuit is open and calls should be rejected. */ - isOpen(): boolean { - return this.state === 'open'; - } - - /** Record a successful call — resets the failure counter and closes the circuit. */ - recordSuccess(): void { - this._failures = 0; - this._state = 'closed'; - } - - /** Record a failed call — increments the counter and opens the circuit when the threshold is reached. */ - recordFailure(): void { - this._failures++; - this.lastFailureAt = Date.now(); - if (this._failures >= this.failureThreshold) { - this._state = 'open'; - } - } - - /** @internal Reset to initial state (used in tests). */ - reset(): void { - this._failures = 0; - this._state = 'closed'; - this.lastFailureAt = 0; - } -} - -/** Module-level singleton shared by both embedder entry points. */ -export const hfDownloadCircuit = new HfDownloadCircuitBreaker(); +export const hfDownloadCircuit = new CircuitBreaker({ + failureThreshold: CB_FAILURE_THRESHOLD, + cooldownMs: CB_RESET_TIMEOUT_MS, + key: 'hf-download', +}); // --------------------------------------------------------------------------- // Retry + timeout wrapper @@ -219,11 +170,6 @@ export function withDownloadTimeout(fn: () => Promise, timeoutMs: number): }); } -/** @internal Async sleep (exposed for testing). */ -export function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export interface HfRetryOptions { /** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */ maxAttempts?: number; @@ -235,7 +181,7 @@ export interface HfRetryOptions { * Circuit-breaker instance to use. Defaults to the module-level * `hfDownloadCircuit` singleton. Pass a fresh instance in tests. */ - circuit?: HfDownloadCircuitBreaker; + circuit?: CircuitBreaker; /** * Optional callback invoked before each retry (not the initial attempt). * @param attempt - 1-based retry number @@ -295,49 +241,74 @@ export async function withHfDownloadRetry( circuit = hfDownloadCircuit, onRetry, } = options; - if (circuit.isOpen()) { - const secsUntilReset = Math.ceil( - (circuit.resetTimeoutMs - (Date.now() - circuit.lastFailureAt)) / 1000, - ); + if (circuit.getState() === 'open') { + // Compute remaining cooldown without consuming a probe permit. + const openedAt = circuit.getOpenedAt(); + const secsUntilReset = + openedAt !== null ? Math.ceil((circuit.getCooldownMs() - (Date.now() - openedAt)) / 1000) : 0; throw new Error( `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` + (secsUntilReset > 0 ? ` — will reset in ~${secsUntilReset}s` : ''), ); } - let lastError: Error = new Error('unknown error'); + // Retry budget delegated to `withRetry` from gitnexus-shared. The + // HF-specific bits — per-attempt timeout, network-vs-non-network + // classification, circuit-breaker recording, onRetry callback — wire + // through the `isRetryable` callback. `circuitTripped` is the + // sentinel that lets us replace the final thrown error with a + // CIRCUIT_OPEN_TAG message when the breaker tripped mid-loop. + let circuitTripped = false; - for (let attempt = 0; attempt < maxAttempts; attempt++) { - try { - const result = await withDownloadTimeout(fn, timeoutMs); - circuit.recordSuccess(); - return result; - } catch (err) { - lastError = err instanceof Error ? err : new Error(String(err)); - - if (!isNetworkFetchError(lastError.message)) { - // Non-network error (e.g. CUDA unavailable) — propagate without retry - throw lastError; - } - - circuit.recordFailure(); - - if (circuit.isOpen()) { - // Circuit just tripped — fail fast, no more retries - throw new Error( - `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${circuit.failureThreshold} consecutive failures`, - ); - } - - if (attempt < maxAttempts - 1) { - const delay = baseDelayMs * Math.pow(2, attempt); - onRetry?.(attempt + 1, maxAttempts, lastError); - await sleep(delay); - } + try { + return await withRetry( + async () => { + const result = await withDownloadTimeout(fn, timeoutMs); + circuit.recordSuccess(); + return result; + }, + { + maxAttempts, + baseDelayMs, + // Disable the cap to match the bespoke pure-exponential + // progression. With the default `HF_MAX_ATTEMPTS_CAP = 10` and + // `baseDelayMs = 2000`, the largest possible delay is + // `2000 * 2^9 = ~17 minutes` — bounded enough not to need a cap. + capDelayMs: Number.MAX_SAFE_INTEGER, + isRetryable: (err, attempt) => { + const error = err instanceof Error ? err : new Error(String(err)); + if (!isNetworkFetchError(error.message)) { + // Non-network error (e.g. CUDA unavailable) — propagate + // without retry. Use recordNeutral so the breaker's existing + // failure-count progress isn't reset by a non-network failure + // that says nothing about the CDN's health. + circuit.recordNeutral(); + return { retry: false }; + } + circuit.recordFailure(); + if (circuit.getState() === 'open') { + // Circuit just tripped — fail fast, no more retries. + circuitTripped = true; + return { retry: false }; + } + // Mirror the bespoke onRetry contract: fire only when there's + // actually a next attempt. + if (attempt + 1 < maxAttempts) { + onRetry?.(attempt + 1, maxAttempts, error); + } + return { retry: true }; + }, + }, + ); + } catch (err) { + if (circuitTripped) { + throw new Error( + `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${CB_FAILURE_THRESHOLD} consecutive failures`, + ); } + // All retries exhausted — rethrow the last network error so + // isNetworkFetchError patterns in the calling code still match and + // surface HF_ENDPOINT guidance. + throw err; } - - // All retries exhausted — throw the last network error so isNetworkFetchError - // patterns in the calling code still match and surface HF_ENDPOINT guidance. - throw lastError; } diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 85ad79111..e3fb06045 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -3,13 +3,22 @@ * * Shared fetch+retry logic for OpenAI-compatible /v1/embeddings endpoints. * Imported by both the core embedder (batch) and MCP embedder (query). + * + * Network resilience is delegated to `resilientFetch` from + * `gitnexus-shared` — bounded retries with exponential-backoff jitter, + * `Retry-After` honored on 429, and an in-process circuit breaker that + * fails fast on a flapping endpoint. Per-attempt timeout is enforced + * via `AbortSignal.timeout` on the underlying fetch. */ +import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; + const HTTP_TIMEOUT_MS = 30_000; const HTTP_MAX_RETRIES = 2; const HTTP_RETRY_BACKOFF_MS = 1_000; const HTTP_BATCH_SIZE = 64; const DEFAULT_DIMS = 384; +const HTTP_BREAKER_KEY = 'embeddings-http'; interface HttpConfig { baseUrl: string; @@ -90,46 +99,51 @@ const httpEmbedBatch = async ( model: string, apiKey: string, batchIndex = 0, - attempt = 0, ): Promise => { let resp: Response; try { - resp = await fetch(url, { - method: 'POST', - signal: AbortSignal.timeout(HTTP_TIMEOUT_MS), - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, + resp = await resilientFetch( + url, + { + method: 'POST', + signal: AbortSignal.timeout(HTTP_TIMEOUT_MS), + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ input: batch, model }), }, - body: JSON.stringify({ input: batch, model }), - }); + { + breakerKey: HTTP_BREAKER_KEY, + retry: { maxAttempts: HTTP_MAX_RETRIES + 1, baseDelayMs: HTTP_RETRY_BACKOFF_MS }, + }, + ); } catch (err) { - // Timeouts should not be retried — the server is unresponsive. - // AbortSignal.timeout() throws DOMException with name 'TimeoutError'. - const isTimeout = err instanceof DOMException && err.name === 'TimeoutError'; - if (isTimeout) { + if (err instanceof CircuitOpenError) { + throw new Error( + `Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`, + ); + } + if (err instanceof DOMException && err.name === 'TimeoutError') { throw new Error( `Embedding request timed out after ${HTTP_TIMEOUT_MS}ms (${safeUrl(url)}, batch ${batchIndex})`, ); } - // DNS, connection errors — retry with backoff - if (attempt < HTTP_MAX_RETRIES) { - const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1); - await new Promise((r) => setTimeout(r, delay)); - return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1); + if (err instanceof ResilientFetchExhaustedError) { + throw new Error( + `Embedding endpoint returned ${err.response.status} (${safeUrl(url)}, batch ${batchIndex})`, + ); } const reason = err instanceof Error ? err.message : String(err); throw new Error(`Embedding request failed (${safeUrl(url)}, batch ${batchIndex}): ${reason}`); } if (!resp.ok) { - const status = resp.status; - if ((status === 429 || status >= 500) && attempt < HTTP_MAX_RETRIES) { - const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1); - await new Promise((r) => setTimeout(r, delay)); - return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1); - } - throw new Error(`Embedding endpoint returned ${status} (${safeUrl(url)}, batch ${batchIndex})`); + // resilientFetch already retried 5xx/429; any non-OK response here is + // a terminal client error (4xx other than 429). + throw new Error( + `Embedding endpoint returned ${resp.status} (${safeUrl(url)}, batch ${batchIndex})`, + ); } const data = (await resp.json()) as { data: EmbeddingItem[] }; diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts index 172b8b00f..7f9cc8312 100644 --- a/gitnexus/src/core/wiki/llm-client.ts +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -1,4 +1,5 @@ import { logger } from '../logger.js'; +import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; /** * LLM Client for Wiki Generation * @@ -170,86 +171,85 @@ export async function callLLM( ? { 'api-key': config.apiKey } : { Authorization: `Bearer ${config.apiKey}` }; - const MAX_RETRIES = 3; - let lastError: Error | null = null; - - for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { - try { - const response = await fetch(url, { + // Network resilience (bounded retries with exponential-backoff jitter, + // 5xx + 429 + Retry-After handling, in-process circuit breaker on the + // LLM endpoint) is delegated to resilientFetch. Provider-specific + // error parsing (Azure content filter, empty-content checks) stays + // here since it requires response-body inspection. + let response: Response; + try { + response = await resilientFetch( + url, + { method: 'POST', headers: { 'Content-Type': 'application/json', ...authHeaders, }, body: JSON.stringify(body), - }); - - if (!response.ok) { - const errorText = await response.text().catch(() => 'unknown error'); - - // Azure content filter — surface a clear message instead of a generic API error - if ( - azure && - response.status === 400 && - (errorText.includes('content_filter') || - errorText.includes('ResponsibleAIPolicyViolation')) - ) { - throw new Error( - `Azure content filter blocked this request. The prompt triggered content policy. Details: ${errorText.slice(0, 300)}`, - ); - } - - // Rate limit — wait with exponential backoff and retry - if (response.status === 429 && attempt < MAX_RETRIES - 1) { - const retryAfter = parseInt(response.headers.get('retry-after') || '0', 10); - const delay = retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 3000; - await sleep(delay); - continue; - } - - // Server error — retry with backoff - if (response.status >= 500 && attempt < MAX_RETRIES - 1) { - await sleep((attempt + 1) * 2000); - continue; - } - - throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`); - } - - // Streaming path - if (useStream && response.body) { - return await readSSEStream(response.body, options!.onChunk!); - } - - // Non-streaming path - const json = (await response.json()) as any; - const choice = json.choices?.[0]; - if (!choice?.message?.content) { - throw new Error('LLM returned empty response'); - } - - return { - content: choice.message.content, - promptTokens: json.usage?.prompt_tokens, - completionTokens: json.usage?.completion_tokens, - }; - } catch (err: any) { - lastError = err; - - // Network error — retry with backoff - if ( - attempt < MAX_RETRIES - 1 && - (err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.message?.includes('fetch')) - ) { - await sleep((attempt + 1) * 3000); - continue; - } - - throw err; + // Per-attempt timeout. Without this each retry can hang + // indefinitely on a frozen TCP connection — the per-call + // signal is the only timeout `resilientFetch` honors; + // `capDelayMs` only bounds the *backoff* between attempts. + // 60s matches typical LLM completion budgets. + signal: AbortSignal.timeout(60_000), + }, + { + breakerKey: `wiki-llm-${new URL(url).host}`, + retry: { maxAttempts: 3, baseDelayMs: 2_000, capDelayMs: 30_000 }, + }, + ); + } catch (err) { + if (err instanceof CircuitOpenError) { + throw new Error( + `LLM endpoint circuit open: retry in ${Math.ceil(err.retryAfterMs / 1000)}s. ${err.message}`, + ); } + if (err instanceof ResilientFetchExhaustedError) { + const errorText = await err.response.text().catch(() => 'unknown error'); + throw new Error( + `LLM API error (${err.response.status} after retries): ${errorText.slice(0, 500)}`, + ); + } + throw err; } - throw lastError || new Error('LLM call failed after retries'); + if (!response.ok) { + const errorText = await response.text().catch(() => 'unknown error'); + + // Azure content filter — surface a clear message instead of a generic API error. + if ( + azure && + response.status === 400 && + (errorText.includes('content_filter') || errorText.includes('ResponsibleAIPolicyViolation')) + ) { + throw new Error( + `Azure content filter blocked this request. The prompt triggered content policy. Details: ${errorText.slice(0, 300)}`, + ); + } + + // Any other non-OK response here is a terminal 4xx — resilientFetch + // already retried 5xx/429 to exhaustion and would have thrown above. + throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`); + } + + // Streaming path + if (useStream && response.body) { + return await readSSEStream(response.body, options!.onChunk!); + } + + // Non-streaming path + const json = (await response.json()) as any; + const choice = json.choices?.[0]; + if (!choice?.message?.content) { + throw new Error('LLM returned empty response'); + } + + return { + content: choice.message.content, + promptTokens: json.usage?.prompt_tokens, + completionTokens: json.usage?.completion_tokens, + }; } /** @@ -312,7 +312,3 @@ async function readSSEStream( return { content }; } - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/gitnexus/test/unit/hf-env.test.ts b/gitnexus/test/unit/hf-env.test.ts index fa8f23bd3..5999b34a2 100644 --- a/gitnexus/test/unit/hf-env.test.ts +++ b/gitnexus/test/unit/hf-env.test.ts @@ -1,12 +1,12 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import os from 'node:os'; import { join } from 'node:path'; +import { CircuitBreaker } from 'gitnexus-shared'; import { applyHfEnvOverrides, isNetworkFetchError, isHfDownloadFailure, isHfCircuitOpenError, - HfDownloadCircuitBreaker, withDownloadTimeout, withHfDownloadRetry, CIRCUIT_OPEN_TAG, @@ -154,85 +154,13 @@ describe('isHfDownloadFailure', () => { }); }); -describe('HfDownloadCircuitBreaker', () => { - it('starts in closed state', () => { - const cb = new HfDownloadCircuitBreaker(); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - }); - - it('opens after reaching the failure threshold', () => { - const cb = new HfDownloadCircuitBreaker(3); - cb.recordFailure(); - cb.recordFailure(); - expect(cb.isOpen()).toBe(false); - cb.recordFailure(); // threshold reached - expect(cb.isOpen()).toBe(true); - expect(cb.state).toBe('open'); - }); - - it('closes on recordSuccess after being open', () => { - const cb = new HfDownloadCircuitBreaker(1); - cb.recordFailure(); - expect(cb.isOpen()).toBe(true); - cb.recordSuccess(); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - }); - - it('transitions to half-open after the reset timeout', () => { - vi.useFakeTimers(); - try { - const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); - cb.recordFailure(); - expect(cb.isOpen()).toBe(true); - vi.advanceTimersByTime(200); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('half-open'); - } finally { - vi.useRealTimers(); - } - }); - - it('reset() restores closed state', () => { - const cb = new HfDownloadCircuitBreaker(1); - cb.recordFailure(); - expect(cb.isOpen()).toBe(true); - cb.reset(); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - }); - - it('re-opens when a failure is recorded in half-open state', () => { - vi.useFakeTimers(); - try { - const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); - cb.recordFailure(); // opens the circuit - vi.advanceTimersByTime(200); // advance past reset timeout - expect(cb.state).toBe('half-open'); // getter transitions _state to half-open - cb.recordFailure(); // failure in half-open → re-opens - expect(cb.isOpen()).toBe(true); - expect(cb.state).toBe('open'); - } finally { - vi.useRealTimers(); - } - }); - - it('closes the circuit when success is recorded in half-open state', () => { - vi.useFakeTimers(); - try { - const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); - cb.recordFailure(); // opens the circuit - vi.advanceTimersByTime(200); // advance past reset timeout - expect(cb.state).toBe('half-open'); - cb.recordSuccess(); // success in half-open → closes - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - } finally { - vi.useRealTimers(); - } - }); -}); +// CircuitBreaker state-machine tests live in +// `gitnexus/test/unit/integrations/circuit-breaker.test.ts` — that suite +// already covers the closed/open/half-open transitions, recordSuccess/ +// recordFailure semantics, half-open probe gating, and configurable +// thresholds. No need to duplicate here; this file's remaining tests +// focus on HF-specific composition (withHfDownloadRetry, env-var +// overrides, error classification). describe('withDownloadTimeout', () => { it('resolves when fn completes before the timeout', async () => { @@ -262,7 +190,7 @@ describe('withDownloadTimeout', () => { describe('withHfDownloadRetry', () => { it('returns the result on first success', async () => { const fn = vi.fn().mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(); + const cb = new CircuitBreaker(); const result = await withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 }); expect(result).toBe('ok'); expect(fn).toHaveBeenCalledTimes(1); @@ -270,7 +198,7 @@ describe('withHfDownloadRetry', () => { it('retries on network errors and succeeds on second attempt', async () => { const fn = vi.fn().mockRejectedValueOnce(new Error('fetch failed')).mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(); + const cb = new CircuitBreaker(); const result = await withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, @@ -282,7 +210,7 @@ describe('withHfDownloadRetry', () => { it('throws the last network error after all attempts are exhausted', async () => { const fn = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:443')); - const cb = new HfDownloadCircuitBreaker(99 /* high threshold */); + const cb = new CircuitBreaker({ failureThreshold: 99 }); await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0 }), ).rejects.toThrow('ECONNREFUSED'); @@ -291,7 +219,7 @@ describe('withHfDownloadRetry', () => { it('does not retry non-network errors', async () => { const fn = vi.fn().mockRejectedValue(new Error('Failed to initialize CUDA backend')); - const cb = new HfDownloadCircuitBreaker(); + const cb = new CircuitBreaker(); await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0 }), ).rejects.toThrow('Failed to initialize CUDA backend'); @@ -300,7 +228,7 @@ describe('withHfDownloadRetry', () => { it('fails immediately when the circuit is already open', async () => { const fn = vi.fn().mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(1); + const cb = new CircuitBreaker({ failureThreshold: 1 }); cb.recordFailure(); // open the circuit await expect(withHfDownloadRetry(fn, { circuit: cb })).rejects.toThrow(CIRCUIT_OPEN_TAG); expect(fn).not.toHaveBeenCalled(); @@ -308,12 +236,12 @@ describe('withHfDownloadRetry', () => { it('opens the circuit after failureThreshold failures and throws a circuit-open error', async () => { const fn = vi.fn().mockRejectedValue(new Error('ENOTFOUND huggingface.co')); - const cb = new HfDownloadCircuitBreaker(2 /* threshold */, 60_000); + const cb = new CircuitBreaker({ failureThreshold: 2, cooldownMs: 60_000 }); // First call: 2 attempts, threshold=2 → circuit opens on 2nd failure await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 2, baseDelayMs: 0 }), ).rejects.toThrow(CIRCUIT_OPEN_TAG); - expect(cb.isOpen()).toBe(true); + expect(cb.getState()).toBe('open'); }); it('calls onRetry with correct arguments on each retry', async () => { @@ -322,7 +250,7 @@ describe('withHfDownloadRetry', () => { .mockRejectedValueOnce(new Error('fetch failed')) .mockRejectedValueOnce(new Error('fetch failed')) .mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); const onRetry = vi.fn(); await withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0, onRetry }); expect(onRetry).toHaveBeenCalledTimes(2); @@ -342,11 +270,11 @@ describe('withHfDownloadRetry', () => { it('resets the circuit on success', async () => { const fn = vi.fn().mockResolvedValue('value'); - const cb = new HfDownloadCircuitBreaker(5); + const cb = new CircuitBreaker({ failureThreshold: 5 }); cb.recordFailure(); cb.recordFailure(); // 2 failures, circuit still closed await withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 }); - expect(cb.state).toBe('closed'); + expect(cb.getState()).toBe('closed'); }); }); @@ -371,7 +299,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=1 gives exactly 1 attempt', async () => { process.env.HF_MAX_ATTEMPTS = '1'; const fn = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:443')); - const cb = new HfDownloadCircuitBreaker(99_999 /* high threshold */); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'ECONNREFUSED', ); @@ -381,7 +309,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=2 gives exactly 2 attempts', async () => { process.env.HF_MAX_ATTEMPTS = '2'; const fn = vi.fn().mockRejectedValue(new Error('ENOTFOUND huggingface.co')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'ENOTFOUND', ); @@ -391,7 +319,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=abc falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = 'abc'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -401,7 +329,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=0 falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = '0'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -411,7 +339,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=-1 falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = '-1'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -421,7 +349,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS is clamped to HF_MAX_ATTEMPTS_CAP', async () => { process.env.HF_MAX_ATTEMPTS = '9999'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999 /* very high threshold */); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -431,7 +359,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=2.9 is floored to 2', async () => { process.env.HF_MAX_ATTEMPTS = '2.9'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -443,7 +371,7 @@ describe('withHfDownloadRetry env overrides', () => { try { process.env.HF_DOWNLOAD_TIMEOUT_MS = '50'; const neverResolves = () => new Promise(() => {}); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); const promise = withHfDownloadRetry(neverResolves, { circuit: cb, maxAttempts: 1 }); vi.advanceTimersByTime(100); await expect(promise).rejects.toThrow('ETIMEDOUT'); @@ -458,7 +386,7 @@ describe('withHfDownloadRetry env overrides', () => { // we just verify that the env var rejection causes options.timeoutMs to be // the default constant (not -1) by confirming the resolved value is used. const fn = vi.fn().mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); // Provide explicit timeoutMs to avoid the default 5-minute wait const result = await withHfDownloadRetry(fn, { circuit: cb, timeoutMs: 100 }); expect(result).toBe('ok'); @@ -470,7 +398,7 @@ describe('withHfDownloadRetry env overrides', () => { // Set an env value exceeding the 30-minute cap process.env.HF_DOWNLOAD_TIMEOUT_MS = String(HF_MAX_TIMEOUT_MS + 60_000); const neverResolves = () => new Promise(() => {}); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); const promise = withHfDownloadRetry(neverResolves, { circuit: cb, maxAttempts: 1 }); // Advance just past the 30-minute cap vi.advanceTimersByTime(HF_MAX_TIMEOUT_MS + 1); @@ -483,7 +411,7 @@ describe('withHfDownloadRetry env overrides', () => { it('explicit options override env vars', async () => { process.env.HF_MAX_ATTEMPTS = '5'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); // explicit maxAttempts: 2 must win over HF_MAX_ATTEMPTS=5 await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 2, baseDelayMs: 0 }), diff --git a/gitnexus/test/unit/integrations/circuit-breaker.test.ts b/gitnexus/test/unit/integrations/circuit-breaker.test.ts new file mode 100644 index 000000000..4f08cd4f8 --- /dev/null +++ b/gitnexus/test/unit/integrations/circuit-breaker.test.ts @@ -0,0 +1,395 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { CircuitBreaker, CircuitOpenError, getBreaker } from 'gitnexus-shared'; +import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers'; + +describe('CircuitBreaker', () => { + beforeEach(() => __resetBreakerRegistry__()); + + function makeClock(start = 1_700_000_000_000) { + let t = start; + return { + now: () => t, + advance: (ms: number) => { + t += ms; + }, + }; + } + + it('runs through check/recordSuccess in closed state', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 3, cooldownMs: 30_000, now: clock.now }); + expect(b.getState()).toBe('closed'); + b.check(); // does not throw + b.recordSuccess(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(0); + }); + + it('stays closed below the failure threshold', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 3, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + b.recordFailure(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(2); + }); + + it('opens after failureThreshold consecutive failures and check throws', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 3, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + b.recordFailure(); + b.recordFailure(); + expect(b.getState()).toBe('open'); + expect(() => b.check()).toThrow(CircuitOpenError); + }); + + it('CircuitOpenError.retryAfterMs decreases as time advances', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(30_000); + + clock.advance(10_000); + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(20_000); + }); + + it('transitions Open -> Half-Open after cooldown elapses (via check)', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + expect(b.getState()).toBe('open'); + clock.advance(31_000); + b.check(); // should not throw + // After check, internal state is half-open (next call probes). + expect(b.getConsecutiveFailures()).toBe(1); // unchanged until next outcome + }); + + it('half-open + recordSuccess -> closed and counter reset', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + b.check(); + b.recordSuccess(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(0); + }); + + it('half-open + recordFailure -> open with fresh openedAt', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + const firstOpen = clock.now(); + clock.advance(31_000); // cooldown expired + b.check(); // half-open + b.recordFailure(); + // Open with fresh timestamp — full cooldown again. + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + expect(caught?.retryAfterMs).toBe(30_000); + // Sanity: not the original openedAt (would be negative remaining). + expect(clock.now()).toBeGreaterThan(firstOpen); + }); + + it('recordSuccess from closed state with prior partial failures resets counter', () => { + const b = new CircuitBreaker({ failureThreshold: 5 }); + b.recordFailure(); + b.recordFailure(); + expect(b.getConsecutiveFailures()).toBe(2); + b.recordSuccess(); + expect(b.getConsecutiveFailures()).toBe(0); + expect(b.getState()).toBe('closed'); + }); + + describe('recordNeutral (U1)', () => { + it('is a no-op from closed state with zero prior failures', () => { + const b = new CircuitBreaker({ failureThreshold: 3 }); + b.recordNeutral(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(0); + }); + + it('preserves partial-failure progress (does not reset counter)', () => { + const b = new CircuitBreaker({ failureThreshold: 3 }); + b.recordFailure(); + b.recordFailure(); + b.recordNeutral(); + expect(b.getConsecutiveFailures()).toBe(2); + expect(b.getState()).toBe('closed'); + // Real third failure still trips the breaker — neutrals didn't + // erase the running count toward the threshold. + b.recordFailure(); + expect(b.getState()).toBe('open'); + }); + + it('does not reset openedAt or transition out of open state', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + expect(b.getState()).toBe('open'); + b.recordNeutral(); + // Still open; cooldown clock unchanged. + expect(() => b.check()).toThrow(CircuitOpenError); + }); + + it('leaves half-open state alone (next true outcome decides)', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + b.check(); // half-open + b.recordNeutral(); + // Still half-open; a subsequent recordFailure flips to open. + b.recordFailure(); + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + }); + + it('integration: 2 failures + 5 neutrals + 1 failure → opens on third real failure', () => { + const b = new CircuitBreaker({ failureThreshold: 3 }); + b.recordFailure(); + b.recordFailure(); + for (let i = 0; i < 5; i++) b.recordNeutral(); + expect(b.getConsecutiveFailures()).toBe(2); + expect(b.getState()).toBe('closed'); + b.recordFailure(); + expect(b.getState()).toBe('open'); + }); + }); + + describe('half-open probe permit gate (U1)', () => { + it('admits exactly one caller after cooldown; subsequent check() throws halfOpenRetryAfterMs', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: 30_000, + halfOpenRetryAfterMs: 1_000, + now: clock.now, + }); + b.recordFailure(); + clock.advance(31_000); + + // Caller A: gets the probe permit. + b.check(); + expect(b.isProbeInFlight()).toBe(true); + + // Caller B: blocked. + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + expect(caught?.retryAfterMs).toBe(1_000); + + // Caller A's recordSuccess clears the breaker. + b.recordSuccess(); + expect(b.isProbeInFlight()).toBe(false); + expect(b.getState()).toBe('closed'); + + // Caller C: succeeds in closed state. + b.check(); + expect(b.getState()).toBe('closed'); + }); + + it('recordFailure on probe re-opens with fresh cooldown (NOT halfOpenRetryAfterMs)', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: 30_000, + halfOpenRetryAfterMs: 1_000, + now: clock.now, + }); + b.recordFailure(); + clock.advance(31_000); + + b.check(); // A: probe + expect(() => b.check()).toThrow(CircuitOpenError); // B: blocked + + b.recordFailure(); // A reports failure → reopens with fresh openedAt + + // C: should see the fresh cooldown remaining, not the probe-in-flight 1s default. + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + // Fresh openedAt = current clock; cooldown is 30s; retryAfter ≈ 30s. + expect(caught?.retryAfterMs).toBe(30_000); + }); + + it('recordNeutral releases the probe permit but leaves state half-open', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + b.check(); // A: probe + expect(b.isProbeInFlight()).toBe(true); + + b.recordNeutral(); // A: neutral — permit released, state untouched + expect(b.isProbeInFlight()).toBe(false); + expect(b.getState()).toBe('half-open'); + + // B: succeeds (becomes the new probe), no longer blocked. + b.check(); + expect(b.isProbeInFlight()).toBe(true); + + // B's recordSuccess clears the breaker. + b.recordSuccess(); + expect(b.getState()).toBe('closed'); + }); + + it('three sequential probes via neutrals: A → A.neutral → B → B.neutral → C', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + const initialFailures = b.getConsecutiveFailures(); + clock.advance(31_000); + + for (let i = 0; i < 3; i++) { + b.check(); + b.recordNeutral(); + } + // Counter unchanged; state still half-open; permit released. + expect(b.getConsecutiveFailures()).toBe(initialFailures); + expect(b.getState()).toBe('half-open'); + expect(b.isProbeInFlight()).toBe(false); + }); + + it('5 same-tick sequential callers: exactly one passes, the other 4 throw', () => { + // `check()` is synchronous — these calls execute on a single + // microtask in declaration order. The first mutates probeInFlight + // = true; the next four observe the mutation and throw. This + // tests mutation ordering, not true concurrency (the actual + // interleaved-async-microtask scenario lives in U2). + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + const results: Array<'pass' | 'throw'> = []; + for (let i = 0; i < 5; i++) { + try { + b.check(); + results.push('pass'); + } catch { + results.push('throw'); + } + } + expect(results.filter((r) => r === 'pass').length).toBe(1); + expect(results.filter((r) => r === 'throw').length).toBe(4); + }); + + it('probe permit consumed; clock advances another full cooldown without record*; still throws', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + b.check(); // probe permit consumed + clock.advance(60_000); // another full cooldown elapses, no record* + + // Half-open semantics: wait for an outcome, not a timer. The + // permit-consumed state doesn't auto-resolve on time. + expect(() => b.check()).toThrow(CircuitOpenError); + }); + + it('halfOpenRetryAfterMs default is 1000 when not configured', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + b.check(); + + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(1_000); + }); + + it('halfOpenRetryAfterMs is configurable for long-running protected ops', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: 30_000, + halfOpenRetryAfterMs: 10_000, // LLM-streaming-friendly + now: clock.now, + }); + b.recordFailure(); + clock.advance(31_000); + b.check(); + + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(10_000); + }); + + it('getState() is a pure read — does not consume the probe permit', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + // Test calls getState() to inspect — must not consume the permit. + expect(b.getState()).toBe('half-open'); + expect(b.isProbeInFlight()).toBe(false); + // First check() still gets the permit. + b.check(); + expect(b.isProbeInFlight()).toBe(true); + }); + }); + + describe('getBreaker registry', () => { + it('returns the same instance for the same key', () => { + const a = getBreaker('endpoint-a'); + const b = getBreaker('endpoint-a'); + expect(a).toBe(b); + }); + + it('returns different instances for different keys', () => { + const a = getBreaker('endpoint-a'); + const b = getBreaker('endpoint-b'); + expect(a).not.toBe(b); + }); + + it('__resetBreakerRegistry__ clears all instances', () => { + const a = getBreaker('endpoint-a'); + __resetBreakerRegistry__(); + const a2 = getBreaker('endpoint-a'); + expect(a2).not.toBe(a); + }); + }); +}); diff --git a/gitnexus/test/unit/integrations/resilient-fetch.test.ts b/gitnexus/test/unit/integrations/resilient-fetch.test.ts new file mode 100644 index 000000000..05299c0ba --- /dev/null +++ b/gitnexus/test/unit/integrations/resilient-fetch.test.ts @@ -0,0 +1,558 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { + CircuitBreaker, + CircuitOpenError, + parseRetryAfter, + resilientFetch, + ResilientFetchExhaustedError, + RETRY_AFTER_CAP_MS, +} from 'gitnexus-shared'; +import { __resetBreakerRegistry__, classifyOutcome } from 'gitnexus-shared/test-helpers'; + +describe('parseRetryAfter', () => { + it('parses delta-seconds form', () => { + expect(parseRetryAfter('30')).toBe(30_000); + expect(parseRetryAfter('0')).toBe(0); + }); + it('returns null on negative or non-numeric garbage', () => { + expect(parseRetryAfter(null)).toBeNull(); + expect(parseRetryAfter('')).toBeNull(); + expect(parseRetryAfter(' ')).toBeNull(); + expect(parseRetryAfter('not-a-number')).toBeNull(); + }); + it('parses HTTP-date form against an injected clock', () => { + const now = () => Date.parse('Wed, 21 Oct 2025 07:28:00 GMT'); + expect(parseRetryAfter('Wed, 21 Oct 2025 07:28:30 GMT', now)).toBe(30_000); + }); + it('returns 0 (not negative) on past HTTP-date', () => { + const now = () => Date.parse('Wed, 21 Oct 2025 08:00:00 GMT'); + expect(parseRetryAfter('Wed, 21 Oct 2025 07:28:00 GMT', now)).toBe(0); + }); +}); + +describe('classifyOutcome', () => { + const now = () => 1_700_000_000_000; + + it('classifies 2xx as success', () => { + const resp = new Response(null, { status: 204 }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('success'); + }); + it('classifies 5xx as retryable-status without afterMs', () => { + const resp = new Response(null, { status: 503 }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('retryable-status'); + if (out.kind === 'retryable-status') expect(out.afterMs).toBeUndefined(); + }); + it('classifies 429 with Retry-After (capped) as retryable-status', () => { + const resp = new Response(null, { status: 429, headers: { 'Retry-After': '99999' } }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('retryable-status'); + if (out.kind === 'retryable-status') expect(out.afterMs).toBe(RETRY_AFTER_CAP_MS); + }); + it('classifies 429 from a header-less fetch mock without throwing', () => { + // Tests sometimes stub `fetch` with a plain `{ ok, status }` object + // (e.g. http-embedder.test.ts). Real `Response` always carries + // `Headers`, but the helper must not crash when the stub does not. + // Falls through to exponential-backoff retry like a 429 with no + // Retry-After header. + const resp = { ok: false, status: 429 } as unknown as Response; + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('retryable-status'); + if (out.kind === 'retryable-status') expect(out.afterMs).toBeUndefined(); + }); + it('classifies 401/403/404/422 as terminal-client', () => { + for (const status of [401, 403, 404, 422, 400]) { + const resp = new Response(null, { status }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('terminal-client'); + } + }); + it('classifies TimeoutError as terminal-network', () => { + const err = new DOMException('aborted', 'TimeoutError'); + const out = classifyOutcome({ kind: 'error', err }, now); + expect(out.kind).toBe('terminal-network'); + }); + it('classifies generic network throw as retryable-network', () => { + const err = new TypeError('fetch failed'); + const out = classifyOutcome({ kind: 'error', err }, now); + expect(out.kind).toBe('retryable-network'); + }); +}); + +describe('resilientFetch', () => { + const URL_STR = 'https://example.test/api/dispatch'; + + beforeEach(() => __resetBreakerRegistry__()); + + function jsonResp(status: number, headers?: Record): Response { + return new Response(null, { status, headers }); + } + + function makeBreaker(opts: Partial[0]> = {}) { + let t = 1_700_000_000_000; + const breaker = new CircuitBreaker({ + failureThreshold: 3, + cooldownMs: 30_000, + key: 'test', + now: () => t, + ...opts, + }); + return { breaker, advance: (ms: number) => (t += ms) }; + } + + it('204 returns immediately, no retries, breaker stays closed', async () => { + const fetchImpl = vi.fn(async () => jsonResp(204)); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }); + expect(resp.status).toBe(204); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + expect(breaker.getState()).toBe('closed'); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('one 503 then 204 → retried once, returns 204, breaker stays closed', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(503) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, random: () => 0.5, baseDelayMs: 100, capDelayMs: 1000 }, + }); + expect(resp.status).toBe(204); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledTimes(1); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('429 with Retry-After honored (capped at RETRY_AFTER_CAP_MS)', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(429, { 'Retry-After': '1' }) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }); + expect(sleep).toHaveBeenCalledWith(1000); // 1s + }); + + it('429 with absurd Retry-After is capped to RETRY_AFTER_CAP_MS', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(429, { 'Retry-After': '99999' }) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, capDelayMs: 999_999 }, // ensure cap comes from RETRY_AFTER_CAP_MS, not retry config + }); + expect(sleep).toHaveBeenCalledWith(RETRY_AFTER_CAP_MS); + }); + + it('429 without Retry-After falls back to exponential-backoff delay', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(429) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, baseDelayMs: 100, capDelayMs: 1000, random: () => 0.5 }, + }); + // attempt 0: full-jitter upper = min(1000, 100*1) = 100; floor(0.5*100) = 50 + expect(sleep).toHaveBeenCalledWith(50); + }); + + it('401 returned as Response, no retry, breaker not incremented', async () => { + const fetchImpl = vi.fn(async () => jsonResp(401)); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }); + expect(resp.status).toBe(401); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('422 returned as Response, no retry', async () => { + const fetchImpl = vi.fn(async () => jsonResp(422)); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {} }, + }); + expect(resp.status).toBe(422); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('TimeoutError rethrown immediately, no retry, breaker not incremented', async () => { + const fetchImpl = vi.fn(async () => { + throw new DOMException('aborted', 'TimeoutError'); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }), + ).rejects.toThrow(DOMException); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('three consecutive 503 throws ResilientFetchExhaustedError; breaker increments by 1', async () => { + const fetchImpl = vi.fn(async () => jsonResp(503)); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(ResilientFetchExhaustedError); + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect(breaker.getConsecutiveFailures()).toBe(1); + }); + + it('after three exhausted 503 batches, breaker opens and fails fast', async () => { + const fetchImpl = vi.fn(async () => jsonResp(503)); + const { breaker } = makeBreaker({ failureThreshold: 3, cooldownMs: 60_000 }); + for (let i = 0; i < 3; i++) { + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(ResilientFetchExhaustedError); + } + expect(breaker.getState()).toBe('open'); + // 4th call: breaker open, no fetch invoked. + const fetchCallsBefore = fetchImpl.mock.calls.length; + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(CircuitOpenError); + expect(fetchImpl.mock.calls.length).toBe(fetchCallsBefore); + }); + + it('retryable-network error retries, breaker counts only on exhaustion', async () => { + const fetchImpl = vi.fn(async () => { + throw new TypeError('fetch failed'); + }); + const { breaker } = makeBreaker(); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(TypeError); + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect(breaker.getConsecutiveFailures()).toBe(1); + }); + + describe('U2: terminal outcomes route through recordNeutral', () => { + it('401 does not erase prior partial-failure progress on the breaker', async () => { + const { breaker } = makeBreaker(); + // Pre-seed the breaker with 2 failures (still closed; threshold 3). + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.getConsecutiveFailures()).toBe(2); + + const fetchImpl = vi.fn(async () => jsonResp(401)); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {} }, + }); + + // Counter MUST stay at 2 — under the old behaviour recordSuccess + // would have reset to 0 and the next 5xx batch would have started + // from scratch instead of tipping over the threshold. + expect(breaker.getConsecutiveFailures()).toBe(2); + expect(breaker.getState()).toBe('closed'); + }); + + it('TimeoutError does not erase prior partial-failure progress', async () => { + const { breaker } = makeBreaker(); + breaker.recordFailure(); + breaker.recordFailure(); + + const fetchImpl = vi.fn(async () => { + throw new DOMException('aborted by timeout', 'TimeoutError'); + }); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {} }, + }), + ).rejects.toBeInstanceOf(DOMException); + + expect(breaker.getConsecutiveFailures()).toBe(2); + }); + + it('external AbortError is terminal: no retry, breaker untouched', async () => { + const { breaker } = makeBreaker(); + breaker.recordFailure(); + + const fetchImpl = vi.fn(async () => { + throw new DOMException('aborted by caller', 'AbortError'); + }); + const sleep = vi.fn(async () => {}); + + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, maxAttempts: 3 }, + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + // Counter unchanged — neither incremented (no failure) nor reset + // (no synthetic success). + expect(breaker.getConsecutiveFailures()).toBe(1); + }); + + it('interleaved 5xx + 401 + 5xx + 401 + 5xx opens breaker on third real failure', async () => { + const { breaker } = makeBreaker({ failureThreshold: 3 }); + const sequence = [503, 401, 503, 401, 503]; + let i = 0; + const fetchImpl = vi.fn(async () => jsonResp(sequence[i++])); + + // Each call uses maxAttempts:1 so each surfaces a single response + // (5xx → ResilientFetchExhaustedError; 4xx → returned Response). + const driveOne = () => + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + + await expect(driveOne()).rejects.toBeInstanceOf(ResilientFetchExhaustedError); // 5xx fail #1 + await driveOne(); // 401 neutral + await expect(driveOne()).rejects.toBeInstanceOf(ResilientFetchExhaustedError); // 5xx fail #2 + await driveOne(); // 401 neutral + await expect(driveOne()).rejects.toBeInstanceOf(ResilientFetchExhaustedError); // 5xx fail #3 → opens + + expect(breaker.getState()).toBe('open'); + expect(fetchImpl).toHaveBeenCalledTimes(5); + }); + }); + + describe('half-open single-probe gating (U2)', () => { + /** Test helper: a fetch mock whose Response is controlled by the test. */ + function deferredFetch(): { + promise: Promise; + resolve: (resp: Response) => void; + reject: (err: unknown) => void; + } { + let resolve!: (resp: Response) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; + } + + /** Builds a clock-injected breaker pre-opened with cooldown elapsed. */ + function preOpenedBreaker(opts: { cooldownMs: number; halfOpenRetryAfterMs?: number }): { + breaker: CircuitBreaker; + advance: (ms: number) => void; + } { + let t = 1_700_000_000_000; + const breaker = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: opts.cooldownMs, + halfOpenRetryAfterMs: opts.halfOpenRetryAfterMs ?? 1_000, + key: 'test', + now: () => t, + }); + breaker.recordFailure(); + t += opts.cooldownMs + 1; // cooldown elapsed + return { breaker, advance: (ms) => (t += ms) }; + } + + it('happy: 3 concurrent calls — exactly 1 hits fetch, others throw CircuitOpenError', async () => { + const { breaker } = preOpenedBreaker({ cooldownMs: 10 }); + const deferred = deferredFetch(); + const fetchImpl = vi.fn(() => deferred.promise); + + // Synchronous portion of each `resilientFetch` runs eagerly up to + // the first await, so by the time r2/r3 are constructed the probe + // permit is already consumed by r1 and they reject synchronously. + const r1 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r2 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r3 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + + // Resolve the probe with 200; r1 should now settle. + deferred.resolve(new Response(null, { status: 200 })); + + const results = await Promise.allSettled([r1, r2, r3]); + + expect(results[0].status).toBe('fulfilled'); + if (results[0].status === 'fulfilled') { + expect(results[0].value.status).toBe(200); + } + expect(results[1].status).toBe('rejected'); + if (results[1].status === 'rejected') { + expect(results[1].reason).toBeInstanceOf(CircuitOpenError); + } + expect(results[2].status).toBe('rejected'); + if (results[2].status === 'rejected') { + expect(results[2].reason).toBeInstanceOf(CircuitOpenError); + } + + // Only ONE underlying fetch was invoked. + expect(fetchImpl).toHaveBeenCalledTimes(1); + // Breaker closed after the probe's success. + expect(breaker.getState()).toBe('closed'); + }); + + it('error: probe gets 503 — exhausted error; subsequent caller sees fresh full cooldown', async () => { + const { breaker } = preOpenedBreaker({ cooldownMs: 10_000, halfOpenRetryAfterMs: 1_000 }); + const deferred = deferredFetch(); + const fetchImpl = vi.fn(() => deferred.promise); + + const r1 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r2 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r3 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + + // Probe fails with 503 → exhausted (maxAttempts: 1) → recordFailure → reopen. + deferred.resolve(new Response(null, { status: 503 })); + + const results = await Promise.allSettled([r1, r2, r3]); + + expect(results[0].status).toBe('rejected'); + if (results[0].status === 'rejected') { + expect(results[0].reason).toBeInstanceOf(ResilientFetchExhaustedError); + } + expect(results[1].status).toBe('rejected'); + if (results[1].status === 'rejected') { + expect(results[1].reason).toBeInstanceOf(CircuitOpenError); + // Blocked-while-half-open used the halfOpenRetryAfterMs default. + expect((results[1].reason as CircuitOpenError).retryAfterMs).toBe(1_000); + } + + // Breaker has re-opened with a fresh openedAt. + expect(breaker.getState()).toBe('open'); + + // r4: should see the fresh full cooldown, NOT the probe-in-flight 1000ms. + let r4Caught: CircuitOpenError | null = null; + try { + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + } catch (err) { + r4Caught = err as CircuitOpenError; + } + expect(r4Caught).toBeInstanceOf(CircuitOpenError); + expect(r4Caught?.retryAfterMs).toBe(10_000); + }); + + it('cancellation: probe AbortError releases permit; next caller becomes new probe', async () => { + const { breaker } = preOpenedBreaker({ cooldownMs: 10_000 }); + const deferred1 = deferredFetch(); + const deferred2 = deferredFetch(); + let callIdx = 0; + const fetchImpl = vi.fn(() => (callIdx++ === 0 ? deferred1.promise : deferred2.promise)); + + // r1 admitted as the probe; r2 blocked while r1 still in flight. + const r1 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r2 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + await expect(r2).rejects.toBeInstanceOf(CircuitOpenError); + + // Cancel the probe — `AbortError` routes through terminal-network → + // `recordNeutral` → permit released, state stays half-open. + deferred1.reject(new DOMException('aborted by caller', 'AbortError')); + await expect(r1).rejects.toMatchObject({ name: 'AbortError' }); + + expect(breaker.isProbeInFlight()).toBe(false); + expect(breaker.getState()).toBe('half-open'); + + // r3: now succeeds and becomes the new probe. + const r3 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + deferred2.resolve(new Response(null, { status: 200 })); + const r3Resp = await r3; + expect(r3Resp.status).toBe(200); + expect(breaker.getState()).toBe('closed'); + // Two fetches total: the cancelled probe + the recovery probe. + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/gitnexus/test/unit/integrations/retry.test.ts b/gitnexus/test/unit/integrations/retry.test.ts new file mode 100644 index 000000000..2dd4f4cf3 --- /dev/null +++ b/gitnexus/test/unit/integrations/retry.test.ts @@ -0,0 +1,128 @@ +import { describe, it, expect, vi } from 'vitest'; +import { computeBackoffMs, withRetry, type RetryOptions } from 'gitnexus-shared'; + +describe('computeBackoffMs', () => { + it('returns afterMs (capped) when caller supplies it', () => { + expect(computeBackoffMs(0, 500, 5000, 1500, () => 0.5)).toBe(1500); + expect(computeBackoffMs(0, 500, 5000, 99_999, () => 0.5)).toBe(5000); + expect(computeBackoffMs(0, 500, 5000, 0, () => 0.5)).toBe(0); + expect(computeBackoffMs(0, 500, 5000, -1, () => 0.5)).toBe(0); + }); + + it('full-jitter delay falls within [0, min(cap, base * 2^attempt)]', () => { + // attempt 0: upper = min(5000, 500 * 1) = 500 + expect(computeBackoffMs(0, 500, 5000, undefined, () => 0)).toBe(0); + expect(computeBackoffMs(0, 500, 5000, undefined, () => 0.999)).toBeLessThan(500); + // attempt 1: upper = min(5000, 500 * 2) = 1000 + expect(computeBackoffMs(1, 500, 5000, undefined, () => 0.5)).toBe(500); + // attempt 4: 500 * 16 = 8000, capped at 5000 + expect(computeBackoffMs(4, 500, 5000, undefined, () => 0.5)).toBe(2500); + expect(computeBackoffMs(4, 500, 5000, undefined, () => 0.999)).toBeLessThan(5000); + }); +}); + +describe('withRetry', () => { + function makeOpts(overrides: Partial = {}): RetryOptions { + return { + maxAttempts: 3, + baseDelayMs: 10, + capDelayMs: 100, + isRetryable: () => ({ retry: true }), + sleep: vi.fn(async () => {}), + random: () => 0.5, + ...overrides, + }; + } + + it('returns immediately when fn succeeds first try', async () => { + const sleep = vi.fn(async () => {}); + const fn = vi.fn(async () => 'ok'); + const result = await withRetry(fn, makeOpts({ sleep })); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('retries when isRetryable returns retry:true and second call succeeds', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls === 1) throw new Error('boom'); + return 'ok'; + }; + const result = await withRetry(fn, makeOpts({ sleep })); + expect(result).toBe('ok'); + expect(calls).toBe(2); + expect(sleep).toHaveBeenCalledTimes(1); + }); + + it('honors afterMs returned by isRetryable', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls === 1) throw new Error('throttle'); + return 'ok'; + }; + await withRetry( + fn, + makeOpts({ + sleep, + isRetryable: () => ({ retry: true, afterMs: 1500 }), + capDelayMs: 5000, + }), + ); + expect(sleep).toHaveBeenCalledWith(1500); + }); + + it('caps afterMs at capDelayMs', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls === 1) throw new Error('throttle'); + return 'ok'; + }; + await withRetry( + fn, + makeOpts({ + sleep, + isRetryable: () => ({ retry: true, afterMs: 10_000 }), + capDelayMs: 3000, + }), + ); + expect(sleep).toHaveBeenCalledWith(3000); + }); + + it('rethrows immediately when isRetryable returns retry:false', async () => { + const sleep = vi.fn(async () => {}); + const fn = vi.fn(async () => { + throw new Error('terminal'); + }); + await expect( + withRetry(fn, makeOpts({ sleep, isRetryable: () => ({ retry: false }) })), + ).rejects.toThrow('terminal'); + expect(fn).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('throws the last error when maxAttempts exhausted', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + throw new Error(`boom-${calls}`); + }; + await expect(withRetry(fn, makeOpts({ sleep, maxAttempts: 3 }))).rejects.toThrow('boom-3'); + expect(calls).toBe(3); + // 3 attempts → 2 sleeps between them; final attempt does not sleep. + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it('rejects maxAttempts < 1', async () => { + await expect(withRetry(async () => 'ok', makeOpts({ maxAttempts: 0 }))).rejects.toThrow( + /maxAttempts must be >= 1/, + ); + }); +}); From 6906be36953f673e138ada597e838f8f97f590f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 9 May 2026 16:32:38 +0100 Subject: [PATCH 09/11] feat(autofix): replace inline reviewdog with /autofix ChatOps button (#1458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(autofix): verify reviewdog actually posted before claiming "click Apply" The sticky summary comment was stating "Posted formatting suggestions inline. Click Apply suggestion on each" even when reviewdog landed zero inline review comments — typical case: the formatter touched lines outside the PR's added range, so `-filter-mode=added` (correctly) filtered everything out. The script unconditionally set `posted=true` after running reviewdog regardless of whether any comments were actually created, leaving the user staring at a sticky that promised buttons that didn't exist. The publish job now snapshots the count of `github-actions[bot]` review comments before and after reviewdog. If the delta is zero, surface a new `diff-no-overlap` UI state that tells the user plainly: "Formatter found fixable issues, but they're on lines outside this PR's added range — there's nothing to click here. Run locally: npm run lint:fix && npm run format." Plus a matching `gitnexus/autofix` Check Run conclusion (still neutral, distinct title) so agents reading `gh pr checks` see the same signal. Three states are now machine-distinguishable in the sticky's gitnexus-autofix JSON block: suggestions-posted (delta > 0), diff-no-overlap (delta == 0), skipped-too-large (>3k lines). * feat(autofix): replace inline reviewdog with /autofix ChatOps button Pivot the PR autofix UX from per-line reviewdog suggestions to a single slash-command button. Contributors comment `/autofix` on the PR; a new trusted workflow downloads the existing autofix patch artifact, applies it to the PR head, and pushes a commit back. Why: - 3K+ diffs hit GitHub's review-comment API 406 limit -> dead end. - Diffs where the formatter touches lines outside the PR's added range ("no-overlap") get filtered by reviewdog's -filter-mode=added -> dead end (PR #1457 patched the lying sticky but the underlying UX gap remained). - Per-line click-Apply-suggestion is high-friction for big diffs and easy to apply unevenly. - A single `git apply` + push works at any size and lands fixes atomically. Changes: - pr-autofix-publish.yml: remove `Install reviewdog` and `Post inline suggestions` steps. Collapse three sticky states (suggestions-posted, diff-no-overlap, skipped-too-large) into one (fixes-available). Bump JSON schema v1 -> v2 with `apply_command` field; all v1 fields preserved. - pr-autofix-apply.yml (new): triggers on issue_comment with body `/autofix`, validates body via strict regex, validates commenter has write/admin/maintain or is the PR author, locates latest successful pr-autofix run for PR head SHA, downloads artifact, applies patch, pushes commit. Reacts +1/-1/eyes on triggering comment per outcome. Idempotent (`git apply --check --reverse` detects already-applied state). - CONTRIBUTING.md: document v2 schema and the /autofix flow, including the maintainer-edit requirement for fork PR pushes. Trust posture: apply workflow runs from default-branch code only, under issue_comment trigger. Comment body and author login flow through env vars and pattern-matched, never interpolated into shell. Permission gate (write/admin/maintain OR PR author) before any artifact fetch. Fork PRs require "Allow edits by maintainers" (GitHub-native; we don't bypass). Net YAML: -139 lines in publish.yml, +260 in apply.yml. Removes reviewdog binary pin and the entire review-comment API surface. * fix(autofix): address Codex adversarial findings on PR #1458 Two findings from the Codex adversarial review of the autofix ChatOps pivot. Both are localized YAML changes that close trust gaps the pivot inherited from the original PR #1446 design. U1 — Cross-verify metadata against workflow_run authority (.github/workflows/pr-autofix-publish.yml): Previously the trusted publisher accepted pr_number, head_sha, and head_repo from metadata.json after only an allowlist regex. A fork-controlled `npm run lint:fix` could have written a syntactically valid metadata.json referencing another PR/SHA, redirecting the write-scoped sticky/check-run onto an attacker-chosen target. New `Verify metadata against workflow_run authority` step compares artifact-claimed identity against: - github.event.workflow_run.head_sha - github.event.workflow_run.head_repository.full_name - workflow_run.pull_requests[].number (within-repo PRs) - gh api commits/{sha}/pulls fallback (fork PRs, where pull_requests[] is empty) Fail closed on mismatch — no sticky, no check-run, no override. U2 — Lease-protected push in apply workflow (.github/workflows/pr-autofix-apply.yml): Previously the apply step pushed `HEAD:${HEAD_REF}` plain. A force- push between resolve (Step 5) and push (Step 9) would silently fast-forward an older commit graph over the contributor's newer state. Push now uses `--force-with-lease=refs/heads/${HEAD_REF}:${HEAD_SHA}` against the SHA resolved earlier. Distinct `lease-failed` result code + retry-message reply, separated from `push-failed` (fork without maintainer-edit) so contributors can diagnose the actual cause. Plan: docs/plans/2026-05-09-005-fix-autofix-codex-adversarial-findings-plan.md (local-only per repo convention). Trust posture preserved: no new permissions, no new workflows, no contract change. JSON v2 schema unchanged. CodeQL js/server-side- request-forgery and template-injection posture unchanged — all new inputs flow via env vars and pattern-matched. * fix(autofix): close zizmor credential-persistence finding on apply checkout actions/checkout's default behavior writes the GITHUB_TOKEN into .git/config as an extraheader. The token then sits on disk in the checkout directory — an actions/upload-artifact step on that directory would leak it. We don't upload, but zizmor's credential-persistence lint correctly flags the latent risk. Set persist-credentials: false on the Checkout PR head step. Provide push auth inline via `git -c http.extraheader="Authorization: Basic "` so the credential never lands on disk and never appears in process listings (the URL form https://x-access-token:TOKEN@… is rejected here because it leaks via ps and git remote -v). Push lease semantics from U2 unchanged — same --force-with-lease against the resolved HEAD_SHA, same lease-failed/push-failed/stale result codes. * fix(review): apply autofix feedback ce-code-review surfaced 15 findings on PR #1458; this commit applies the 7 with concrete fixes (#1, #2, #3, #4, #5, #9, #13). Five P2 findings (#6, #7, #8, #10, #12) are recorded as residual actionable work for follow-up; two advisory items (#11, #14) skipped. #1 — applied_run_id schema drift (CONTRIBUTING.md): v2 docs claimed `state: applied` enum value and an `applied_run_id` field that no code path emits. Trimmed docs to match what the workflow actually writes (state: fixes-available; v1 field set as superset). Implementing the apply-side sticky upsert that would populate `applied_run_id` is deferred — cleaner than carrying a contract claim with no code. #2 — result= unset between idempotency probe and lease push (pr-autofix-apply.yml): After `git apply --check` passed, an early non-zero exit from `git config` / `git apply` / `git add` / `git commit` left `result=` unset, sending the user to the `*` "unexpected state (`unknown`)" arm. Wrapped the apply/commit phase in a single if-test that sets `result=apply-failed` on any failure. New React-and-reply branch surfaces an actionable message. #3 — permission lookup conflated transient API failures with denial (pr-autofix-apply.yml): `gh api … 2>/dev/null || echo "none"` swallowed 5xx, 429 secondary rate-limit, and network failures, surfacing them as a public 👎 refusal to legitimate maintainers. Now distinguishes 404 (genuine non-collaborator) from other API failures via stderr match. New `allowed=api-failed` state triggers a 😕 reaction with a "transient API failure, retry" reply instead of a misleading refusal. #4 — lease-failure grep missed git's "remote rejected" / branch- deleted phrasings (pr-autofix-apply.yml): Real lease failures got classified as `push-failed` → user told to enable maintainer-edit, which won't help. Expanded regex to match `remote rejected` and `! [rejected]`. #5 — broken bullet continuation in CONTRIBUTING.md release-candidate section: rejoined the split bullet so it renders correctly. #9 — base64 GITHUB_TOKEN bypassed GitHub's secret-masker (pr-autofix-apply.yml): Added `::add-mask::${auth_header}` immediately after construction so any subsequent log line (set -x, GIT_TRACE) gets *** redacted. #13 — misleading schema-bump comment in pr-autofix-publish.yml: Comment claimed all v1 fields preserved exactly, but the `state` enum was redefined v1→v2. Updated to make the migration path explicit (v1 readers see unfamiliar schema, fall back to prose). Residual actionable work (deferred to follow-up): #6 locate step gh api retry; #7 artifact-expired graceful fallback; #8 re-entrancy comment-spam guard; #10 producer-still-running UX; #12 gh_retry wrapper for apply.yml. Validations: yaml.safe_load OK, check-workflow-concurrency.py OK. * fix(autofix): apply remaining ce-code-review residual findings (#6, #7, #8, #10, #12) Pulls the deferred items from the previous review pass into this PR so the workflow ships with full reliability + UX coverage rather than follow-up debt. #6 + #12 — gh_retry wrapper on idempotent GETs in apply.yml: Permission lookup, PR metadata fetch, and workflow-run lookup are now wrapped in the same gh_retry helper publish.yml uses (3 attempts, linear backoff). Reaction/comment POSTs remain unwrapped (retrying POST would dupe the resource). #10 — producer-still-running UX: The locate step now distinguishes three cases via `found_status` output: success (proceed), in-progress / queued / pending / waiting (reply ⏳ "wait for autofix run to finish"), not-found (reply 🤔 "push a commit"), api-failed (reply ⚠️ "transient API failure"). The "no successful autofix run" message no longer fires immediately after a fresh push while the producer is still mid-run. #7 — artifact-expired graceful fallback: actions/download-artifact gains `continue-on-error: true`. The apply step distinguishes patch-file-missing (artifact expired, 1-day retention elapsed) from patch-file-zero-bytes (formatter found nothing). New `result=artifact-expired` case + ⏳ "push a new commit to regenerate" reply. #8 — re-entrancy loop guard: After checkout but before applying, check if HEAD itself is a github-actions[bot] `chore(autofix)` commit. If so, refuse to re-apply (`result=loop-prevented`) with a 🔁 reply telling the user to push a human-authored commit or revert before retrying. Prevents formatter-config-drift loops where an automated agent watching the sticky could pump arbitrary apply commits. Net effect: every code path in apply.yml now sets a meaningful `result=` that maps to a specific user-facing reaction + reply. The `*` "unexpected state (unknown)" arm becomes truly unreachable in normal operation. Validations: yaml.safe_load OK, check-workflow-concurrency.py OK. * fix(autofix): refresh stale reviewdog comments + reject patches touching .github/ Two follow-up findings on PR #1458: #1 — Stale reviewdog references in workflow header comments: pr-autofix-publish.yml's header still described the removed inline- suggestion path ("posts inline review-comment suggestions to the PR using `reviewdog`", "Reviewdog reporter: github-pr-review reads $REVIEWDOG_GITHUB_API_TOKEN…"). The Check Run permissions comment enumerated the old outcomes (clean / suggestions-posted / skipped-too-large) instead of the current set (clean / fixes- available). pr-autofix.yml's header described the trusted job as posting "inline review-comment suggestions" and the changed_lines comment referenced the dead 3000-line cap. Refreshed all three to describe the actual sticky + Check Run + /autofix flow. #2 — Reject patches touching .github/ (sensitive-paths guard): Theoretical supply-chain vector: a malicious PR could ship a custom prettier/ESLint config that reformats workflow YAML, dependabot.yml, or CODEOWNERS. The producer would capture those edits in autofix.patch; a maintainer running `/autofix` would push them under `contents: write` without human review. The default GITHUB_TOKEN lacks the `workflows` scope so workflow-file pushes would fail at the platform layer anyway, but as a generic `push-failed` (which misleads users into enabling maintainer-edit). Reject early with a specific reason. Match runs against the patch with grep on `^(diff --git|---|+++) [ab]?/?\.github/`. New `result=sensitive-paths` case + 🛑 reply telling the user to apply .github/ formatter changes manually. Documented the constraint in CONTRIBUTING.md under the /autofix section so contributors aren't surprised when the workflow refuses a patch that includes formatter changes to workflow files. Validations: yaml.safe_load OK, check-workflow-concurrency.py OK. --- .github/workflows/pr-autofix-apply.yml | 595 +++++++++++++++++++++++ .github/workflows/pr-autofix-publish.yml | 178 +++---- .github/workflows/pr-autofix.yml | 16 +- CONTRIBUTING.md | 76 +-- 4 files changed, 739 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/pr-autofix-apply.yml diff --git a/.github/workflows/pr-autofix-apply.yml b/.github/workflows/pr-autofix-apply.yml new file mode 100644 index 000000000..b2ec8495f --- /dev/null +++ b/.github/workflows/pr-autofix-apply.yml @@ -0,0 +1,595 @@ +name: PR Autofix (apply) + +# CHATOPS HALF of the autofix pipeline. +# +# Triggered when a contributor comments `/autofix` on a PR. Validates +# permission, locates the most recent successful `pr-autofix.yml` +# artifact for the PR's current head SHA, applies the patch to the PR +# head, and pushes a commit back to the PR branch. +# +# This workflow runs from the default branch's copy of the file +# regardless of where the comment originates -- that's the trust +# anchor. Comment body and author login are untrusted; both flow +# through env vars and pattern-matched, never interpolated into shell. +# +# Fork PR support: `git push` with the GITHUB_TOKEN succeeds against +# fork branches only when the contributor enabled "Allow edits by +# maintainers" on the PR (the default). When they disabled it, we +# fail loud with a 👎 reaction and an explanation comment. + +on: + issue_comment: + types: [created] + +concurrency: + # Per-PR scope. issue_comment events expose `github.event.issue.number` + # for both PR and Issue comments; the `pull_request != null` guard on + # the job ensures we only run on PRs, so this number is the PR number. + # cancel-in-progress: false — a second `/autofix` should wait for the + # first to finish (idempotency check on the second invocation handles + # the no-op case). + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }} + cancel-in-progress: false + +permissions: {} + +jobs: + apply: + name: apply-autofix + # Pre-filter at the workflow level so non-PR comments and unrelated + # comments don't even spawn a runner. The job-level body re-check + # below (Step 1) is the strict gate. + if: >- + github.event.issue.pull_request != null + && startsWith(github.event.comment.body, '/autofix') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + # React on the triggering comment + post reply comments. + pull-requests: write + # Push the apply commit to the PR head branch. + contents: write + # Required by actions/download-artifact to fetch artifacts produced + # by a different workflow run. + actions: read + steps: + - name: Validate comment body precisely + id: body + env: + BODY: ${{ github.event.comment.body }} + shell: bash + run: | + set -euo pipefail + # Whole-line, case-sensitive match: `^/autofix\s*$`. The + # workflow-level startsWith guard is coarse — `please don't + # /autofix this code` would pass that filter but fail this one. + # We exit silently (no reaction) on body mismatch so quoted + # text in unrelated discussions doesn't get a visible response. + if [[ ! "${BODY}" =~ ^/autofix[[:space:]]*$ ]]; then + echo "Body did not match strict /autofix regex — exiting silently." + echo "match=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "match=true" >> "$GITHUB_OUTPUT" + + - name: Validate commenter permission + id: perm + if: steps.body.outputs.match == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENTER: ${{ github.event.comment.user.login }} + PR_AUTHOR: ${{ github.event.issue.user.login }} + shell: bash + run: | + set -euo pipefail + + # Retry wrapper for transient 5xx / 429 / network blips. + # Mirrors the helper in pr-autofix-publish.yml. Used on + # idempotent GETs only; reactions/comment-POSTs are NOT + # wrapped (retrying a POST would dupe the resource). + gh_retry() { + local n=0 max=3 + while true; do + if gh "$@"; then return 0; fi + n=$((n+1)) + if [ "$n" -ge "$max" ]; then return 1; fi + sleep $((n * 2)) + done + } + + # Allowlist the commenter login before it flows into a URL. + # GitHub usernames: alphanumeric + dashes, max 39 chars. + if ! [[ "${COMMENTER}" =~ ^[A-Za-z0-9-]{1,39}$ ]]; then + echo "::error::Invalid commenter login format: $(printf '%q' "${COMMENTER}")" + echo "allowed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Self-comparison: PR author can always /autofix their own PR. + if [ "${COMMENTER}" = "${PR_AUTHOR}" ]; then + echo "Commenter is PR author — granting access." + echo "allowed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Repo permission lookup. admin/write/maintain are sufficient. + # Distinguish API failure (5xx, 429, network) from genuine + # permission denial (404 = not a collaborator). Conflating them + # would silently refuse a legitimate maintainer with a public + # 👎 every time GitHub blips. gh_retry handles transient blips; + # the stderr-grep distinguishes 404 from persistent failure. + perm_stderr=$(mktemp) + if permission=$(gh_retry api "repos/${GH_REPO}/collaborators/${COMMENTER}/permission" \ + --jq '.permission' 2>"$perm_stderr"); then + echo "Commenter permission: ${permission}" + case "${permission}" in + admin|write|maintain) + echo "allowed=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "allowed=false" >> "$GITHUB_OUTPUT" + ;; + esac + else + err=$(cat "$perm_stderr") + echo "Permission lookup stderr: ${err}" >&2 + # 404 (not a collaborator) is a genuine deny. + # Anything else is a transient API/network failure. + if grep -qE "HTTP 404|Not Found" "$perm_stderr"; then + echo "allowed=false" >> "$GITHUB_OUTPUT" + else + echo "::error::Permission lookup failed transiently — refusing to act." + echo "allowed=api-failed" >> "$GITHUB_OUTPUT" + fi + fi + + - name: React 😕 on transient permission-API failure + if: steps.body.outputs.match == 'true' && steps.perm.outputs.allowed == 'api-failed' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Couldn't verify your repo permission (transient GitHub API failure). Please comment \`/autofix\` again. ([apply run](https://github.com/${GH_REPO}/actions/runs/${RUN_ID}))" \ + >/dev/null + exit 1 + + - name: React 👎 on permission denial + if: steps.body.outputs.match == 'true' && steps.perm.outputs.allowed == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + shell: bash + run: | + set -euo pipefail + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🚫 \`/autofix\` is restricted to users with write access or the PR author. Comment ignored." \ + >/dev/null + # Hard exit so the rest of the job is skipped. + exit 1 + + - name: React 👀 to acknowledge + if: steps.perm.outputs.allowed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + shell: bash + run: | + set -euo pipefail + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="eyes" >/dev/null + + - name: Resolve PR head and locate autofix run + id: locate + if: steps.perm.outputs.allowed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + shell: bash + run: | + set -euo pipefail + + # Same retry wrapper used in the permission step, repeated + # because each YAML `run:` block is a fresh bash session. + gh_retry() { + local n=0 max=3 + while true; do + if gh "$@"; then return 0; fi + n=$((n+1)) + if [ "$n" -ge "$max" ]; then return 1; fi + sleep $((n * 2)) + done + } + + # Fetch PR metadata. All fields here are server-controlled API + # output, but we still allowlist before exporting so anything + # weird short-circuits before $GITHUB_OUTPUT. Wrapped in + # gh_retry so transient blips don't surface as "no autofix run + # found" with a wrong remediation. + if ! pr_json=$(gh_retry api "repos/${GH_REPO}/pulls/${PR}"); then + echo "::error::PR metadata fetch failed after retries." + echo "found_status=api-failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + head_sha=$(jq -r '.head.sha' <<< "${pr_json}") + head_ref=$(jq -r '.head.ref' <<< "${pr_json}") + head_repo=$(jq -r '.head.repo.full_name' <<< "${pr_json}") + + [[ "${head_sha}" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Bad head_sha"; exit 1; } + [[ "${head_ref}" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "::error::Bad head_ref"; exit 1; } + [[ "${head_repo}" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "::error::Bad head_repo"; exit 1; } + + # Find the latest successful pr-autofix.yml run for this head SHA. + if ! runs_json=$(gh_retry api "repos/${GH_REPO}/actions/workflows/pr-autofix.yml/runs?head_sha=${head_sha}&per_page=10"); then + echo "::error::Workflow run lookup failed after retries." + echo "found_status=api-failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + + run_id=$(jq -r '[.workflow_runs[] | select(.conclusion == "success")] | .[0].id // empty' <<< "${runs_json}") + + if [ -n "${run_id}" ] && [[ "${run_id}" =~ ^[0-9]+$ ]]; then + echo "found_status=success" >> "$GITHUB_OUTPUT" + { + echo "found=true" + echo "head_sha=${head_sha}" + echo "head_ref=${head_ref}" + echo "head_repo=${head_repo}" + echo "run_id=${run_id}" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + # No successful run. Distinguish "still running" (producer in + # flight after a recent push) from "never ran / all failed". + # in_progress / queued / pending / waiting cover the GitHub + # workflow-run lifecycle states that precede success/failure. + in_progress=$(jq -r '[.workflow_runs[] | select(.status == "in_progress" or .status == "queued" or .status == "pending" or .status == "waiting")] | length' <<< "${runs_json}") + if [ "${in_progress:-0}" -gt 0 ]; then + echo "::warning::pr-autofix run is still in progress for head ${head_sha}." + echo "found_status=in-progress" >> "$GITHUB_OUTPUT" + else + echo "::warning::No successful pr-autofix run found for head ${head_sha}." + echo "found_status=not-found" >> "$GITHUB_OUTPUT" + fi + # Existing `found` boolean is preserved so downstream gates + # (`steps.locate.outputs.found == 'true'`) still work. + echo "found=false" >> "$GITHUB_OUTPUT" + + - name: Reply when locate did not yield a usable run + if: steps.perm.outputs.allowed == 'true' && steps.locate.outputs.found != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + FOUND_STATUS: ${{ steps.locate.outputs.found_status }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + run_url="https://github.com/${GH_REPO}/actions/runs/${RUN_ID}" + case "${FOUND_STATUS}" in + in-progress) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⏳ A pr-autofix run is still in progress for this PR's current head SHA. Wait for it to finish, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + ;; + api-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Couldn't reach the GitHub API to look up the autofix run (transient failure after retries). Please comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + ;; + *) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🤔 No successful autofix run found for this PR's current head SHA. Push a new commit to trigger one, then comment \`/autofix\` again." \ + >/dev/null + ;; + esac + exit 1 + + # Pinned to v8.0.1. Same SHA as pr-autofix-publish.yml. + # `continue-on-error: true` lets the workflow proceed when the + # artifact is expired or pruned (1-day retention). The apply + # step distinguishes "patch file missing entirely" (artifact- + # expired) from "patch file zero bytes" (genuinely empty patch). + - name: Download autofix artifact + if: steps.locate.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: autofix + run-id: ${{ steps.locate.outputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: autofix-in + + # Pinned to v5.0.4. Verify SHA via: + # gh api repos/actions/checkout/git/refs/tags/v5.0.4 + # + # `persist-credentials: false` disables the default behavior where + # actions/checkout writes the GITHUB_TOKEN into `.git/config` as an + # extraheader. That default is convenient (subsequent git commands + # auth automatically) but it means the token is sitting on disk in + # the checkout directory — an `actions/upload-artifact` step on + # this directory would leak the token. We don't upload, but + # zizmor's `credential-persistence` lint flags it defensively. + # Push auth is provided inline at push time via the URL. + - name: Checkout PR head + if: steps.locate.outputs.found == 'true' + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.4 + with: + repository: ${{ steps.locate.outputs.head_repo }} + ref: ${{ steps.locate.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: false + # Fetch full history so the push doesn't hit shallow-clone errors. + fetch-depth: 0 + path: pr-checkout + + - name: Apply patch and push + id: apply + if: steps.locate.outputs.found == 'true' + env: + HEAD_REF: ${{ steps.locate.outputs.head_ref }} + HEAD_REPO: ${{ steps.locate.outputs.head_repo }} + # The SHA we resolved earlier in `locate` — this is what the + # remote ref MUST still equal at push time. If the contributor + # force-pushed between resolve and now, the lease fails and + # we surface that distinctly from a fork-without-maintainer + # -edit push failure. + HEAD_SHA: ${{ steps.locate.outputs.head_sha }} + # Auth for the push only — never persisted to disk. Provided + # via env to avoid interpolating into the shell command line. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + working-directory: pr-checkout + run: | + set -euo pipefail + patch="../autofix-in/autofix.patch" + + # Distinguish artifact-expired (file missing entirely, because + # actions/download-artifact ran with continue-on-error and the + # 1-day retention had elapsed) from genuinely empty patch + # (file present, zero bytes, formatter found nothing). + if [ ! -e "$patch" ]; then + echo "::warning::Patch file does not exist — autofix artifact likely expired." + echo "result=artifact-expired" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ ! -s "$patch" ]; then + echo "::warning::Empty patch — nothing to apply." + echo "result=empty-patch" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Sensitive-paths guard: refuse to apply patches that touch + # `.github/` — workflow files, action definitions, CODEOWNERS, + # dependabot config, etc. A malicious PR could ship a custom + # prettier/ESLint config that reformats workflow YAML; the + # producer would then capture those edits in autofix.patch, + # and a maintainer running `/autofix` would push them under + # `contents: write`. The default GITHUB_TOKEN lacks `workflows` + # scope so the platform would reject workflow-file pushes + # anyway, but that surfaces as a generic `push-failed` and + # misleads users into enabling maintainer-edit. Reject early + # with a specific reason. CODEOWNERS and dependabot.yml live + # under .github/ but outside .github/workflows/ — the broader + # match is intentional (they all govern trust boundaries). + if grep -qE '^(diff --git|---|\+\+\+) [ab]?/?\.github/' "$patch"; then + echo "::warning::Patch touches .github/ — refusing to apply (sensitive paths)." + echo "result=sensitive-paths" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Re-entrancy guard: if HEAD itself is an autofix bot commit, + # refuse to apply again. Without this, lint/formatter config + # drift between runs could pump arbitrary apply commits into + # the same PR if an automated agent watches the sticky and + # re-fires `/autofix` on each new "fixes-available" surface. + # The contributor can still get out by force-pushing a + # human-authored commit to revert the autofix and re-trigger. + head_author=$(git log -1 --format='%ae' HEAD) + head_subject=$(git log -1 --format='%s' HEAD) + if [ "${head_author}" = "41898282+github-actions[bot]@users.noreply.github.com" ] \ + && [[ "${head_subject}" =~ ^chore\(autofix\) ]]; then + echo "::warning::HEAD is an autofix bot commit — refusing to re-apply (loop guard)." + echo "result=loop-prevented" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Idempotency probe: does the forward apply work? + if git apply --check "$patch" 2>/dev/null; then + echo "Patch applies cleanly — proceeding." + elif git apply --check --reverse "$patch" 2>/dev/null; then + # Reverse-check passes => the patch is already applied to + # the current tree. Treat as success no-op. + echo "Patch is already applied (reverse-check passed) — no-op." + echo "result=already-applied" >> "$GITHUB_OUTPUT" + exit 0 + else + echo "::error::Patch does not apply (stale or conflicting)." + echo "result=stale" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Wrap the apply/commit phase so any non-zero exit sets a + # meaningful `result=` instead of leaving it unset (which would + # send the user to the `*` "unexpected state" arm with a + # non-actionable confused-emoji reply). + if ! { + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" && + git config user.name "github-actions[bot]" && + git apply "$patch" && + git add -A && + git commit -m "chore(autofix): apply prettier + eslint fixes via /autofix command" + }; then + echo "::error::git apply / config / commit failed after idempotency probe passed." + echo "result=apply-failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Push to the PR head branch with a lease against the resolved + # SHA. The lease ensures the remote ref still points at HEAD_SHA + # when the push lands — if the contributor force-pushed in the + # window between resolve and now, the lease fails and we return + # `lease-failed` (NOT `push-failed`, which would mislead users + # into enabling maintainer-edit). For fork PRs, the push still + # requires "Allow edits by maintainers" to be enabled. + # + # Auth is supplied inline via `-c http..extraheader` (NOT + # via a `https://x-access-token:TOKEN@…` URL — those leak into + # process listings and `git remote -v` output). The header is + # set per-invocation; it never lands in `.git/config` on disk. + # The token is base64-encoded for the Basic auth header per + # GitHub's documented pattern for this scope. + push_url="https://github.com/${HEAD_REPO}.git" + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 -w0)" + # GitHub's secret-masker only masks the raw token, not its + # base64-encoded form. Mask the encoded value so any subsequent + # log line (set -x, GIT_TRACE, error spew) gets ***-redacted. + echo "::add-mask::${auth_header}" + push_stderr=$(mktemp) + if git -c http.extraheader="${auth_header}" \ + push --force-with-lease="refs/heads/${HEAD_REF}:${HEAD_SHA}" \ + "${push_url}" "HEAD:${HEAD_REF}" 2>"$push_stderr"; then + echo "result=applied" >> "$GITHUB_OUTPUT" + else + cat "$push_stderr" >&2 + # `--force-with-lease` reports "stale info" when the remote + # ref has moved past the expected SHA. Other lease-failure + # phrases git emits include "remote rejected" (server-side + # reject), "non-fast-forward", and the literal flag name. Match + # any of those to distinguish from auth/network/maintainer- + # edit failures. + if grep -qE "stale info|force-with-lease|rejected.*non-fast-forward|remote rejected|! \[rejected\]" "$push_stderr"; then + echo "::error::git push lease failed — branch moved during apply." + echo "result=lease-failed" >> "$GITHUB_OUTPUT" + else + echo "::error::git push failed — likely fork without maintainer-edit enabled." + echo "result=push-failed" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + - name: React and reply on outcome + if: always() && steps.locate.outputs.found == 'true' && steps.apply.outcome != 'skipped' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + RESULT: ${{ steps.apply.outputs.result }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + run_url="https://github.com/${GH_REPO}/actions/runs/${RUN_ID}" + + case "${RESULT}" in + applied) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="+1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="✅ Applied autofix and pushed a commit. ([apply run](${run_url}))" \ + >/dev/null + ;; + already-applied) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="+1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="✅ Autofix is already applied — no changes needed." \ + >/dev/null + ;; + empty-patch) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="+1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="✅ No autofix to apply — formatter found nothing." \ + >/dev/null + ;; + artifact-expired) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⏳ The autofix artifact for this PR's head SHA has expired (1-day retention). Push a new commit to regenerate it, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + loop-prevented) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🔁 Refusing to re-apply autofix on top of an existing autofix commit. If formatter rules drifted and you genuinely need another pass, push a human-authored commit (or revert the existing autofix commit) before commenting \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + sensitive-paths) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🛑 Refusing to apply: the autofix patch touches files under \`.github/\` (workflow / CODEOWNERS / dependabot config). Apply formatter changes to those files manually in a regular commit so they get human review. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + stale) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ The autofix patch is stale or conflicts with the current head — push a new commit to regenerate, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + apply-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Autofix applied cleanly in the dry run, but \`git apply\` / \`git commit\` failed when actually landing the patch. This usually means a race with concurrent edits or a corrupt patch. See logs: ${run_url}" \ + >/dev/null + exit 1 + ;; + push-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Couldn't push the autofix commit. If this is a fork PR, please tick **Allow edits by maintainers** in the PR sidebar, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + lease-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ The PR head moved while autofix was applying — a new commit landed in the window between resolve and push. Comment \`/autofix\` again to retry against the latest head. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + *) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="❓ Autofix run finished in an unexpected state (\`${RESULT:-unknown}\`). See logs: ${run_url}" \ + >/dev/null + exit 1 + ;; + esac diff --git a/.github/workflows/pr-autofix-publish.yml b/.github/workflows/pr-autofix-publish.yml index a22f2cc7a..08ad1d60f 100644 --- a/.github/workflows/pr-autofix-publish.yml +++ b/.github/workflows/pr-autofix-publish.yml @@ -3,20 +3,19 @@ name: PR Autofix (publish) # TRUSTED HALF of the autofix pipeline. # # Triggered by `pr-autofix.yml` completing on a PR (including fork PRs). -# Downloads the diff artifact produced by the untrusted job and posts -# inline review-comment suggestions to the PR using `reviewdog`. This -# job NEVER checks out fork code — it only consumes the diff (data) and -# calls the GitHub API. That isolation is what makes it safe to run -# under `pull-requests: write` on fork-triggered events. +# Downloads the diff artifact produced by the untrusted job, verifies +# its claimed PR identity against the workflow_run authority, then +# posts (or edits) a single sticky summary comment plus a +# `gitnexus/autofix` Check Run. This job NEVER checks out fork code — +# it only consumes the diff (data) and calls the GitHub API. That +# isolation is what makes it safe to run under `pull-requests: write` +# on fork-triggered events. # -# Also posts (or edits) a single sticky summary comment so contributors -# and AI agents have one stable, machine-readable signal that says -# whether autofix had anything to suggest. Look for the heading -# "## :sparkles: PR Autofix" in the PR's top-level comments. -# -# Reviewdog reporter: `github-pr-review` reads $REVIEWDOG_GITHUB_API_TOKEN -# and posts via the GraphQL/REST PR-review API. It does not need a -# checkout because the diff itself encodes file paths + line numbers. +# The sticky comment is the contributor signal: heading +# "## :sparkles: PR Autofix" in the PR's top-level comments, with a +# fenced `gitnexus-autofix` JSON block carrying machine-readable state +# for AI agents. Contributors apply the patch by commenting `/autofix` +# on the PR — handled by the separate `pr-autofix-apply.yml` workflow. on: workflow_run: @@ -49,9 +48,9 @@ jobs: # by a different workflow run. actions: read # Required to create the `gitnexus/autofix` Check Run that reports - # the outcome (clean / suggestions-posted / skipped-too-large) to - # the PR's Checks tab. Branch protection or agents can grep the - # conclusion + output title without parsing the sticky comment. + # the outcome (clean / fixes-available) to the PR's Checks tab. + # Branch protection or agents can grep the conclusion + output + # title without parsing the sticky comment. checks: write steps: # Pinned to v8.0.1. Verify SHA via: @@ -114,71 +113,78 @@ jobs: echo "changed_lines=${CHANGED}" } >> "$GITHUB_OUTPUT" - # Pinned to v1.5.0. Verify SHA via: - # gh api repos/reviewdog/action-setup/git/refs/tags/v1.5.0 - # (annotated tag — resolve via .../git/tags/ --jq .object) - - name: Install reviewdog - if: steps.meta.outputs.changed_lines != '0' - uses: reviewdog/action-setup@d8a7baabd7f3e8544ee4dbde3ee41d0011c3a93f # v1.5.0 - with: - # Pin the binary, not just the action SHA — a bad reviewdog - # release otherwise breaks every PR with no rollback. Bump - # this knob deliberately when validating a new release. - reviewdog_version: v0.21.0 - - - name: Post inline suggestions - id: suggest + # Cross-verify the artifact's claimed identity against the + # GitHub-controlled workflow_run event. The previous step's + # allowlist only proves the fields are well-formed — not that + # they refer to the PR/SHA that actually triggered this run. + # A fork-controlled `npm run lint:fix` could plausibly mutate + # metadata.json to reference another PR or SHA, redirecting our + # write-scoped sticky/check-run onto an attacker-chosen target. + # + # Authority sources are all server-controlled GitHub event fields: + # - workflow_run.head_sha + # - workflow_run.head_repository.full_name + # - workflow_run.pull_requests[].number (within-repo PRs only; + # empty array on fork PRs — fall back to commits/{sha}/pulls) + # + # Mismatch => fail loud BEFORE any sticky/check-run side effect. + - name: Verify metadata against workflow_run authority + id: verify if: steps.meta.outputs.changed_lines != '0' env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CI_REPO_OWNER: ${{ github.repository_owner }} - CI_REPO_NAME: ${{ github.event.repository.name }} - CI_PULL_REQUEST: ${{ steps.meta.outputs.pr_number }} - CI_COMMIT: ${{ steps.meta.outputs.head_sha }} - # Pull `changed_lines` through env so bash gets a real - # variable (and shellcheck SC2170 doesn't fire on `-gt` against - # a `${{ }}`-interpolated literal). - CHANGED_LINES: ${{ steps.meta.outputs.changed_lines }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + META_PR_NUMBER: ${{ steps.meta.outputs.pr_number }} + META_HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + META_HEAD_REPO: ${{ steps.meta.outputs.head_repo }} + WF_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + WF_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + WF_PR_NUMBERS: ${{ toJSON(github.event.workflow_run.pull_requests.*.number) }} shell: bash run: | set -euo pipefail - patch=autofix-in/autofix.patch - if [ ! -s "$patch" ]; then - echo "Empty patch — nothing to suggest." - echo "posted=false" >> "$GITHUB_OUTPUT" - exit 0 + + # 1) head_sha must match exactly. workflow_run.head_sha is the + # commit GitHub actually ran the producer against — definitive. + if [ "${META_HEAD_SHA}" != "${WF_HEAD_SHA}" ]; then + echo "::error::Artifact head_sha (${META_HEAD_SHA}) does not match workflow_run.head_sha (${WF_HEAD_SHA}) — refusing to publish." + exit 1 fi - # GitHub's review-comment API returns 406 on diffs above ~3k - # changed lines. Bail out gracefully and let the summary - # comment carry the signal instead. - if [ "$CHANGED_LINES" -gt 3000 ]; then - echo "Diff too large ($CHANGED_LINES lines) — skipping inline suggestions." - echo "posted=skipped-too-large" >> "$GITHUB_OUTPUT" - exit 0 + # 2) head_repo must match exactly. Same authority anchor. + if [ "${META_HEAD_REPO}" != "${WF_HEAD_REPO}" ]; then + echo "::error::Artifact head_repo (${META_HEAD_REPO}) does not match workflow_run.head_repository (${WF_HEAD_REPO}) — refusing to publish." + exit 1 fi - # `-f.diff.strip=1` matches `git diff` output (a/foo b/foo). - # `-filter-mode=added` only suggests on lines the PR added, - # which avoids re-suggesting on already-resolved threads when - # the contributor re-adds the autoformat label. - reviewdog \ - -f=diff -f.diff.strip=1 \ - -name="prettier+eslint" \ - -reporter=github-pr-review \ - -filter-mode=added \ - -level=warning \ - -fail-on-error=false < "$patch" + # 3) pr_number must reference an open PR with this head SHA. + # Within-repo PRs: workflow_run.pull_requests[] is populated. + # Fork PRs: that array is empty by GitHub design — fall back + # to the REST commit-to-PRs lookup. Fail closed if the lookup + # finds no matching open PR (avoids attacker-forged PR ids). + allowed_numbers=$(jq -c '.' <<< "${WF_PR_NUMBERS}") + if [ "${allowed_numbers}" = "[]" ]; then + echo "workflow_run.pull_requests is empty (fork PR) — falling back to commits/{sha}/pulls." + allowed_numbers=$(gh api "repos/${GH_REPO}/commits/${WF_HEAD_SHA}/pulls" \ + --jq '[.[] | select(.state == "open") | .number]' 2>/dev/null || echo "[]") + if [ "${allowed_numbers}" = "[]" ]; then + echo "::error::No open PR found for head ${WF_HEAD_SHA} via commits/{sha}/pulls — refusing to publish." + exit 1 + fi + fi - echo "posted=true" >> "$GITHUB_OUTPUT" + if ! jq -e --argjson n "${META_PR_NUMBER}" 'index($n) != null' <<< "${allowed_numbers}" >/dev/null; then + echo "::error::Artifact pr_number (${META_PR_NUMBER}) is not in the authoritative PR list (${allowed_numbers}) — refusing to publish." + exit 1 + fi + + echo "Verified: metadata identity matches workflow_run authority (PR=${META_PR_NUMBER}, head_sha=${META_HEAD_SHA}, head_repo=${META_HEAD_REPO})." - name: Upsert sticky summary comment # Only post when ci-quality found something fixable (= the # autofix patch is non-empty). When prettier/eslint are clean # the patch is zero bytes and the sticky comment is pure noise, - # so we skip it. When the diff was too large for inline - # suggestions, the sticky is the only signal the contributor - # gets, so we still post in that case. + # so we skip it. if: >- always() && steps.meta.outputs.pr_number != '' @@ -189,8 +195,6 @@ jobs: PR: ${{ steps.meta.outputs.pr_number }} CHANGED: ${{ steps.meta.outputs.changed_lines }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - SCHEMA: ${{ steps.meta.outputs.schema }} - POSTED: ${{ steps.suggest.outputs.posted }} RUN_ID: ${{ github.run_id }} shell: bash run: | @@ -200,25 +204,29 @@ jobs: marker="" heading="## :sparkles: PR Autofix" - if [ "${POSTED}" = "skipped-too-large" ]; then - ui_state="skipped-too-large" - prose="Diff is **${CHANGED}** lines — too large for inline suggestions (GitHub caps the review-comment API at ~3000). Run locally: \`npm run lint:fix && npm run format\`." - else - ui_state="suggestions-posted" - prose="Posted formatting / unused-import suggestions inline. Click **Apply suggestion** on each, or run locally: \`npm run lint:fix && npm run format\`." - fi + # Single state. The /autofix slash command works for any diff + # size — there's no 3K cap and no no-overlap dead-end because + # the apply workflow uses `git apply` + push, not the GitHub + # review-comment API. + ui_state="fixes-available" + prose="Found fixable formatting / unused-import issues across **${CHANGED}** changed lines. **Comment \`/autofix\` on this PR to apply them**, or run \`npm run lint:fix && npm run format\` locally." # Machine-readable JSON block — agents parse this instead of # regexing English. Fenced code-block info string is # `gitnexus-autofix` so agents can locate it without ambiguity. + # Schema bumped from v1 -> v2: adds `apply_command`. The v1 + # field set is preserved as a superset, but the `state` enum + # is redefined (v1: suggestions-posted | skipped-too-large | + # diff-no-overlap; v2: fixes-available). v1 readers checking + # `schema == 'gitnexus.pr-autofix/v1'` see an unfamiliar version + # and fall back to prose, which is the intended migration path. json=$(jq -n -c \ - --arg schema "${SCHEMA}" \ --arg state "${ui_state}" \ --argjson pr_number "${PR}" \ --argjson changed_lines "${CHANGED}" \ --arg head_sha "${HEAD_SHA}" \ --arg run_id "${RUN_ID}" \ - '{schema:$schema, state:$state, pr_number:$pr_number, changed_lines:$changed_lines, head_sha:$head_sha, run_id:$run_id}') + '{schema:"gitnexus.pr-autofix/v2", state:$state, pr_number:$pr_number, changed_lines:$changed_lines, head_sha:$head_sha, run_id:$run_id, apply_command:"/autofix"}') # Multi-line quoted string instead of a column-0 heredoc — YAML's # `run: |` block ends as soon as a content line dedents below the @@ -272,10 +280,9 @@ jobs: - name: Emit gitnexus/autofix Check Run # Stable check name `gitnexus/autofix` so PR-watching agents can # `gh pr checks ` and read the conclusion + title without - # parsing the sticky comment. Three outcomes: - # clean → conclusion: success - # suggestions-posted → conclusion: neutral (review suggestions) - # skipped-too-large → conclusion: neutral (diff > 3000 lines) + # parsing the sticky comment. Two outcomes: + # clean → conclusion: success + # fixes-available → conclusion: neutral # `neutral` does not block branch-protection required-checks but # is visually distinct from a green pass. if: always() && steps.meta.outputs.head_sha != '' @@ -284,7 +291,6 @@ jobs: GH_REPO: ${{ github.repository }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} CHANGED: ${{ steps.meta.outputs.changed_lines }} - POSTED: ${{ steps.suggest.outputs.posted }} shell: bash run: | set -euo pipefail @@ -293,14 +299,10 @@ jobs: conclusion="success" title="Formatting clean" summary="Prettier and ESLint --fix produced no changes." - elif [ "${POSTED}" = "skipped-too-large" ]; then - conclusion="neutral" - title="Diff too large for inline suggestions (${CHANGED} lines)" - summary="GitHub caps the review-comment API at ~3000 lines. Run \`npm run lint:fix && npm run format\` locally." else conclusion="neutral" - title="Suggestions posted" - summary="Inline review-comment suggestions posted. Click **Apply suggestion** on each, or run \`npm run lint:fix && npm run format\` locally." + title="Autofix available — comment /autofix to apply" + summary="Comment \`/autofix\` on this PR to apply formatter + unused-import fixes (works at any diff size). Or run \`npm run lint:fix && npm run format\` locally." fi gh api -X POST "repos/${GH_REPO}/check-runs" \ diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml index f15e8c0e6..dd4a3849a 100644 --- a/.github/workflows/pr-autofix.yml +++ b/.github/workflows/pr-autofix.yml @@ -6,7 +6,9 @@ name: PR Autofix # (including fork heads) and uploads the resulting diff as an artifact. # This job has NO privileged token and CANNOT post to the PR. The trusted # `pr-autofix-publish.yml` workflow downloads the artifact via -# `workflow_run` and posts the inline review-comment suggestions. +# `workflow_run` and posts a sticky summary comment + Check Run. +# Contributors apply the patch by commenting `/autofix` on the PR — +# handled by the separate `pr-autofix-apply.yml` ChatOps workflow. # # Why the split: # ESLint loads plugins from fork-controlled `node_modules`, so running @@ -14,8 +16,8 @@ name: PR Autofix # ship a poisoned eslint plugin and execute arbitrary code under that # token. By keeping fork code execution in this job (token: read-only) # and posting from a separate trusted job that never touches fork -# code, we get the inline-suggestion UX for fork PRs without the -# supply-chain hole. (See autofix.ci for the same pattern.) +# code, we get the autofix UX for fork PRs without the supply-chain +# hole. (See autofix.ci for the same pattern.) # # Removes unused imports via `eslint-plugin-unused-imports`, already in # devDependencies and wired into the `lint` config. @@ -105,11 +107,9 @@ jobs: git diff --no-color > autofix-out/autofix.patch # NOTE: `changed_lines` is the line-count of the patch file, - # which includes hunk headers and context lines — NOT the - # added/removed source-line count. The 3000-line cap in - # pr-autofix-publish.yml is therefore conservative (fires - # before reviewdog hits GitHub's ~3k review-comment API - # ceiling). That bias is intentional. + # (hunk headers + context lines + added/removed). Surfaced in + # the sticky comment so contributors and AI agents have a + # quick size hint before invoking `/autofix`. changed_lines=$(wc -l < autofix-out/autofix.patch | tr -d ' ') echo "changed_lines=${changed_lines}" >> "$GITHUB_OUTPUT" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 99930cf0a..d4f6b12b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,17 +30,17 @@ Format: `[(scope)][!]: ` Allowed types and the release-notes section each one lands in (defined in `.github/release.yml`): -| Type | Label applied | Release-notes section | -|------|---------------|-----------------------| -| `feat` | `enhancement` | 🚀 Features | -| `fix` | `bug` | 🐛 Bug Fixes | -| `perf` | `performance` | 🏎️ Performance | -| `refactor` | `refactor` | 🔄 Refactoring | -| `test` | `test` | 🧪 Tests | -| `ci` | `ci` | 👷 CI/CD | -| `build` / `deps` | `dependencies` | 📦 Dependencies | -| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) | -| `chore` / `revert` | `chore` | (excluded from release notes) | +| Type | Label applied | Release-notes section | +| ------------------ | --------------- | ------------------------------------------------------------ | +| `feat` | `enhancement` | 🚀 Features | +| `fix` | `bug` | 🐛 Bug Fixes | +| `perf` | `performance` | 🏎️ Performance | +| `refactor` | `refactor` | 🔄 Refactoring | +| `test` | `test` | 🧪 Tests | +| `ci` | `ci` | 👷 CI/CD | +| `build` / `deps` | `dependencies` | 📦 Dependencies | +| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) | +| `chore` / `revert` | `chore` | (excluded from release notes) | Append `!` to the type (e.g. `feat(api)!: drop /v1 endpoint`) or include `BREAKING CHANGE:` in the PR body to flag a breaking change — the labeler then adds the `breaking` label and the 💥 Breaking Changes section is rendered first. @@ -81,17 +81,17 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency: - **Merge queue (`merge_group`)**: when this event is added, use `${{ github.workflow }}-${{ github.event.merge_group.head_ref }}` with `cancel-in-progress: false` (every queue entry is a distinct ref; never cancel). - **`cancel-in-progress` policy:** - | Event | `cancel-in-progress` | Why | - |-------|----------------------|-----| - | `pull_request` CI run | `true` | New push supersedes old run | - | `push` to `main` | `false` | Every main commit gets validated | - | Tag push (`v*` publish) | `false` | Never cancel mid-publish | - | `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish | - | `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional | - | `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race | - | Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR | - | PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins | - | Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes | + | Event | `cancel-in-progress` | Why | + | ---------------------------------------- | -------------------- | -------------------------------- | + | `pull_request` CI run | `true` | New push supersedes old run | + | `push` to `main` | `false` | Every main commit gets validated | + | Tag push (`v*` publish) | `false` | Never cancel mid-publish | + | `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish | + | `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional | + | `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race | + | Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR | + | PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins | + | Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes | - For workflows that serve multiple events at once (e.g. `ci.yml` handles `pull_request`, `push`, and `workflow_call`), make `cancel-in-progress` event-aware: @@ -109,18 +109,35 @@ Two workflows produce machine-readable signals on every PR. Coding agents and hu ### `gitnexus/autofix` -`pr-autofix.yml` (untrusted) + `pr-autofix-publish.yml` (trusted) run `prettier --write` and `eslint --fix` against the PR head and surface the diff as inline review-comment suggestions. Three signals are emitted: +`pr-autofix.yml` (untrusted) + `pr-autofix-publish.yml` (trusted) run `prettier --write` and `eslint --fix` against the PR head and surface a single ChatOps button on the PR. Three signals are emitted: -| Surface | Where | Notes | -|---|---|---| -| Sticky PR comment | Top-level comment with the HTML marker `` and heading `## :sparkles: PR Autofix`. Only posted when there is something to fix; clean PRs stay silent. | Edit-in-place via marker; one comment per PR. | -| Fenced JSON block | Inside the sticky, fenced as `gitnexus-autofix`. Schema `gitnexus.pr-autofix/v1` with fields `state` (`suggestions-posted` \| `skipped-too-large`), `pr_number`, `head_sha`, `changed_lines`, `run_id`. | Parseable signal — preferred over regexing prose. | -| Check Run | Stable name `gitnexus/autofix` on the PR head SHA. Conclusion: `success` (clean) or `neutral` (suggestions-posted / skipped-too-large). The output title disambiguates the two `neutral` cases. | Surfaced under PR Checks; readable via `gh pr checks `. | +| Surface | Where | Notes | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| Sticky PR comment | Top-level comment with the HTML marker `` and heading `## :sparkles: PR Autofix`. Only posted when there is something to fix; clean PRs stay silent. | Edit-in-place via marker; one comment per PR. | +| Fenced JSON block | Inside the sticky, fenced as `gitnexus-autofix`. Schema `gitnexus.pr-autofix/v2` with fields `state` (`fixes-available`), `pr_number`, `head_sha`, `changed_lines`, `run_id`, and `apply_command` (literal `/autofix`). | Parseable signal — preferred over regexing prose. v1 fields preserved as a superset. | +| Check Run | Stable name `gitnexus/autofix` on the PR head SHA. Conclusion: `success` (clean) or `neutral` (`fixes-available`). The neutral title is `Autofix available — comment /autofix to apply`. | Surfaced under PR Checks; readable via `gh pr checks `. | To detect outcome from an agent: `gh pr checks --json name,conclusion,output | jq '.[] | select(.name == "gitnexus/autofix")'`. Forks are supported. The untrusted half runs fork code with `permissions: {}` and ships the diff as an artifact; the trusted publish job consumes only the diff (data, not code) and posts the comment + check run. +#### Applying autofix + +Comment `/autofix` on the PR (whole-line, no arguments). The `pr-autofix-apply.yml` workflow: + +1. Validates the comment body matches `^/autofix\s*$` exactly. Quoted or inline mentions are silently ignored. +2. Validates the commenter has `admin`, `write`, or `maintain` permission on the repo, OR is the PR author. Other commenters get a 👎 reaction and a refusal reply. +3. Locates the most recent successful `pr-autofix.yml` run for the PR's current head SHA, downloads its `autofix` artifact, applies the patch, and pushes a `chore(autofix): ...` commit back to the PR head branch. +4. Reacts ✅ on success, 👎 on stale-patch / push-failure, and posts a short reply with the apply-run URL in either case. + +The apply workflow runs from the default branch's copy of the file regardless of where the comment originates — that's the trust anchor. There is no diff-size cap (the apply workflow uses `git apply` + push, not the GitHub review-comment API). + +For fork PRs, the push succeeds only when the contributor has **Allow edits by maintainers** enabled on the PR (the default). When they have disabled it, the workflow fails loud with a 👎 reaction and an explanation comment. + +Re-invoking `/autofix` after a successful apply is a safe no-op — the workflow detects the already-applied state via `git apply --check --reverse` and reacts ✅ without pushing. + +**Sensitive paths.** The apply workflow refuses any patch that touches `.github/` (workflow files, CODEOWNERS, dependabot config). A malicious PR could ship a custom prettier or ESLint config that reformats workflow YAML; if accepted, those edits would be pushed under `contents: write` without human review. Apply formatter changes to files under `.github/` manually in a normal commit so they get the same review every other workflow change gets. + ## AI-assisted contributions If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes. @@ -182,8 +199,7 @@ Two publish workflows ship `gitnexus` to npm: the Docker build. - Manually run `docker build` + `docker push` locally and sign with Cosign against the same digest. - - Delete `rc/` and `v` tags, then redispatch with `force: - true` to re-run the full RC pipeline (cuts a new RC number). + - Delete `rc/` and `v` tags, then redispatch with `force: true` to re-run the full RC pipeline (cuts a new RC number). The rc workflow never moves `latest`. To verify after a change, inspect dist-tags: From e02c56f65310e8bd689634f578929f59cb682cd8 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 16:55:31 +0100 Subject: [PATCH 10/11] fix(security): Pin Docker Node base images, remove runtime package-manager CVE surface, verify Trivy on PRs, and harden Dependabot policy (#1455) * fix: pin Docker node base images and remediate bundled npm CVEs Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/e0605c79-296e-4b3a-b6c3-4ad375950935 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: run trivy on docker PR changes and remove corepack Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/4d714047-4fc1-4af1-9734-91400a15568f Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * chore: add docker digest updates and normalize dockerfile comments Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/7d980908-a823-4c28-b074-9134ec672e84 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: add dependabot cooldown policies Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/a8531b8d-384b-4c54-84dd-a98b31993c44 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: remove unsupported dependabot cooldown keys Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/166df50e-c2fe-4d7f-ab41-e94c703338f6 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * chore(node): bump CI + engines to Node 22; centralize NPM_VERSION via build ARG Closes the LOW findings from Claude Final Re-Review on PR #1455: - Bump engines.node to >=22.0.0 and align all CI workflows (ci-quality, pr-autofix, publish, release-candidate) and the composite setup actions on Node 22. Node 20 reached EOL on 2026-04-30; the test Docker image was already on 22. - Centralize the bootstrapped npm version in a single ARG NPM_VERSION per Dockerfile (cli, web, gitnexus/Dockerfile.test) so a security bump only requires updating one default per file with a clear cross-reference comment. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> Co-authored-by: Gergo Magyar --- .github/actions/setup-gitnexus-web/action.yml | 6 +-- .github/actions/setup-gitnexus/action.yml | 4 +- .github/dependabot.yml | 47 +++++++++++++++++++ .github/workflows/ci-quality.yml | 4 +- .github/workflows/pr-autofix.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/release-candidate.yml | 2 +- .github/workflows/trivy.yml | 14 ++++-- Dockerfile.cli | 28 +++++++---- Dockerfile.web | 17 +++++-- gitnexus/Dockerfile.test | 13 ++++- gitnexus/package.json | 2 +- 12 files changed, 112 insertions(+), 29 deletions(-) diff --git a/.github/actions/setup-gitnexus-web/action.yml b/.github/actions/setup-gitnexus-web/action.yml index 86331e36d..8f895423a 100644 --- a/.github/actions/setup-gitnexus-web/action.yml +++ b/.github/actions/setup-gitnexus-web/action.yml @@ -1,5 +1,5 @@ name: Setup GitNexus Web -description: Setup Node.js 20.19+ (vite 7 floor), build gitnexus-shared, install web dependencies +description: Setup Node.js 22, build gitnexus-shared, install web dependencies runs: using: composite @@ -7,9 +7,7 @@ runs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: # Vite 7 requires Node ^20.19.0 || >=22.12.0 (require(esm) support). - # Pin explicitly so we don't depend on the floating "20" alias resolving - # to a high enough patch version on every runner image. - node-version: '20.19.0' + node-version: 22 cache: npm cache-dependency-path: gitnexus-web/package-lock.json diff --git a/.github/actions/setup-gitnexus/action.yml b/.github/actions/setup-gitnexus/action.yml index e946f1040..b9b4acb7e 100644 --- a/.github/actions/setup-gitnexus/action.yml +++ b/.github/actions/setup-gitnexus/action.yml @@ -1,5 +1,5 @@ name: Setup GitNexus -description: Setup Node.js 20, install dependencies, and optionally build +description: Setup Node.js 22, install dependencies, and optionally build inputs: build: @@ -12,7 +12,7 @@ runs: steps: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: gitnexus/package-lock.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f3530e4d3..c99b666eb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,8 @@ updates: directory: / schedule: interval: weekly + cooldown: + default-days: 7 open-pull-requests-limit: 5 commit-message: prefix: chore @@ -15,6 +17,36 @@ updates: - dependencies - ci + # Keep pinned Docker base-image digests current for the root Dockerfiles. + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + - ci + + # Keep the nested test-image Docker base digest current as well. + - package-ecosystem: docker + directory: /gitnexus + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + - ci + # Gitnexus npm deps — tree-sitter grammars checked daily so we catch # new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars # are grouped so lockstep bumps produce a single PR. The tree-sitter @@ -25,6 +57,11 @@ updates: directory: /gitnexus schedule: interval: daily + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 open-pull-requests-limit: 10 commit-message: prefix: chore(deps) @@ -54,6 +91,11 @@ updates: directory: /gitnexus-web schedule: interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 open-pull-requests-limit: 5 commit-message: prefix: chore(deps) @@ -67,6 +109,11 @@ updates: directory: /gitnexus-shared schedule: interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 open-pull-requests-limit: 5 commit-message: prefix: chore(deps) diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 36a936c82..cc334fcaa 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: package-lock.json - run: npm ci @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: package-lock.json - run: npm ci diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml index dd4a3849a..04eb2468d 100644 --- a/.github/workflows/pr-autofix.yml +++ b/.github/workflows/pr-autofix.yml @@ -61,7 +61,7 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: package-lock.json diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d372c163a..d5af63205 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 registry-url: https://registry.npmjs.org # Hermetic install for the published artifact — no cache carry-over # from non-tag contexts. setup-node v5+ caches by default when a diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index 36f37b5c6..ff75bca26 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -149,7 +149,7 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 registry-url: https://registry.npmjs.org # Hermetic install — release-candidate produces shipped artifacts. # setup-node v5+ caches by default when a packageManager field is diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index f476ee7bc..f7fba75e9 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -1,13 +1,19 @@ name: Trivy Image Scan # Builds Dockerfile.cli and Dockerfile.web, then scans the resulting images -# for OS-package and language-package CVEs at HIGH/CRITICAL severity. +# for OS-package and language-package CVEs at MEDIUM+ severity. # Findings upload to the Security tab; record-only (does not block merges). # -# NOT triggered on PRs — image builds are slow and base-image CVE churn -# shouldn't gate feature delivery. +# Trigger on Dockerfile changes in PRs so base-image/npm-layer remediation can +# be verified before merge without running image scans on every PR. on: + pull_request: + paths: + - 'Dockerfile.cli' + - 'Dockerfile.web' + - 'gitnexus/Dockerfile.test' + - '.github/workflows/trivy.yml' push: branches: [main] schedule: @@ -61,7 +67,7 @@ jobs: image-ref: scan-target:${{ matrix.image.name }} format: sarif output: trivy-${{ matrix.image.name }}.sarif - severity: HIGH,CRITICAL + severity: MEDIUM,HIGH,CRITICAL # Hides CVEs with no available fix in the base image. ignore-unfixed: true exit-code: '0' diff --git a/Dockerfile.cli b/Dockerfile.cli index c45292e06..925f295f0 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,24 +1,32 @@ ARG BUILDPLATFORM ARG TARGETPLATFORM +# Pinned npm version used to replace the bundled npm in the upstream Node +# image. Bumping requires a coordinated update in Dockerfile.web and +# gitnexus/Dockerfile.test so all images bootstrap the same npm. +ARG NPM_VERSION=11.14.1 -# ── Builder ──────────────────────────────────────────────────────────── +# -- Builder ----------------------------------------------------------- # Native modules (tree-sitter-*, onnxruntime-node, node-gyp builds for # tree-sitter-proto / tree-sitter-swift) require python3 + a C/C++ toolchain. -FROM node:22-trixie-slim AS builder +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder +ARG NPM_VERSION WORKDIR /app +RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} + # Toolchain for node-gyp / native builds. RUN apt-get update && apt-get install -y --no-install-recommends python3 make g++ git && rm -rf /var/lib/apt/lists/* -# Build gitnexus-shared first — gitnexus depends on it as a workspace. +# Build gitnexus-shared first - gitnexus depends on it as a workspace. COPY gitnexus-shared/package.json gitnexus-shared/package-lock.json ./gitnexus-shared/ RUN npm ci --prefix gitnexus-shared COPY gitnexus-shared ./gitnexus-shared RUN rm -f gitnexus-shared/tsconfig.tsbuildinfo RUN npm run build --prefix gitnexus-shared -# Copy the full gitnexus package before installing — `npm ci` triggers +# Copy the full gitnexus package before installing - `npm ci` triggers # `postinstall` (patches tree-sitter-swift, builds the vendored # tree-sitter-proto) and `prepare` (compiles TypeScript via scripts/build.js), # both of which need the source tree. @@ -28,11 +36,15 @@ RUN npm ci --prefix gitnexus # Drop dev dependencies for a smaller runtime layer. RUN npm prune --omit=dev --prefix gitnexus -# ── Runtime ──────────────────────────────────────────────────────────── -FROM node:22-trixie-slim AS runtime +# -- Runtime ----------------------------------------------------------- +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime # curl for the healthcheck; git so `gitnexus` can clone repos at runtime. -RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/* \ + && rm -rf /usr/local/lib/node_modules/npm \ + && rm -rf /usr/local/lib/node_modules/corepack \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack WORKDIR /app @@ -47,7 +59,7 @@ COPY --from=builder --chown=node:node /app/gitnexus/vendor ./gitnexus/vendor USER node -# The web UI defaults to http://localhost:4747 — keep that contract. +# The web UI defaults to http://localhost:4747 - keep that contract. ENV GITNEXUS_HOME=/data/gitnexus \ NODE_ENV=production \ PORT=4747 diff --git a/Dockerfile.web b/Dockerfile.web index 7d20e09ce..b102a700e 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,10 +1,17 @@ ARG BUILDPLATFORM ARG TARGETPLATFORM +# Pinned npm version — keep in sync with Dockerfile.cli and +# gitnexus/Dockerfile.test. +ARG NPM_VERSION=11.14.1 -FROM --platform=$BUILDPLATFORM node:22-alpine AS builder +# node:22-bookworm-slim +FROM --platform=$BUILDPLATFORM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder +ARG NPM_VERSION WORKDIR /app +RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} + COPY gitnexus-shared/package.json gitnexus-shared/package-lock.json ./gitnexus-shared/ RUN npm ci --prefix gitnexus-shared @@ -19,9 +26,13 @@ RUN npm ci --prefix gitnexus-web COPY gitnexus-web ./gitnexus-web RUN npm run build --prefix gitnexus-web -FROM node:22-alpine AS runtime +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime -RUN apk add --no-cache curl +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* \ + && rm -rf /usr/local/lib/node_modules/npm \ + && rm -rf /usr/local/lib/node_modules/corepack \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack WORKDIR /app diff --git a/gitnexus/Dockerfile.test b/gitnexus/Dockerfile.test index 0282129ff..37374a5f5 100644 --- a/gitnexus/Dockerfile.test +++ b/gitnexus/Dockerfile.test @@ -1,6 +1,15 @@ -FROM node:20-bookworm +# Pinned npm version — keep in sync with the root Dockerfile.cli and +# Dockerfile.web. +ARG NPM_VERSION=11.14.1 + +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e +ARG NPM_VERSION WORKDIR /app -RUN apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* +RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} \ + && apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update \ + && apt-get install -y python3 make g++ \ + && rm -rf /var/lib/apt/lists/* COPY . . RUN npm ci --ignore-scripts \ && npm rebuild tree-sitter-swift 2>&1 \ diff --git a/gitnexus/package.json b/gitnexus/package.json index 84f702762..810052502 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -116,6 +116,6 @@ } }, "engines": { - "node": ">=20.0.0" + "node": ">=22.0.0" } } From 666041d6083775d5927ff9feaeb928d51dad7296 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 17:26:32 +0100 Subject: [PATCH 11/11] fix(security): log-injection, http-to-file-access, client-side-request-forgery (#1456) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(security): U11 log-injection, http-to-file-access, client-side-request-forgery U11.1: Add validateLLMBaseUrl() in llm-client.ts; called at the top of callLLM() to reject non-http/https schemes and http:// to non-loopback hosts before any fetch that writes LLM output to disk. U11.2: Strip CRLF from groupDir in bridge-db.ts openBridgeDbReadOnly before logging (defence-in-depth on top of pino's JSON escaping). U11.3: Replace console.log with logger.debug and sanitize normalizedName / job.id in api.ts resolveRepo to close js/log-injection alerts. U11.4: Add validateBackendUrl() in backend-client.ts; called inside setBackendUrl() to reject non-http/https schemes before the URL is stored as a fetch target, closing js/client-side-request-forgery alerts. U11.5: Tests added: - wiki-llm-client.test.ts: validateLLMBaseUrl happy/error paths - server-connection.test.ts: validateBackendUrl and setBackendUrl rejection paths All new tests pass (30/30 wiki-llm-client, 18/18 server-connection, 30/30 bridge-db). Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/0452a6ce-711f-4203-9ae6-5dd0b77fb157 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: correct IPv6 loopback check in validateLLMBaseUrl Node's URL parser preserves brackets in hostname for IPv6 addresses (e.g. http://[::1]:11434 yields hostname '[::1]'), so strip them before comparing against '::1'. Add a test to cover this case. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/0452a6ce-711f-4203-9ae6-5dd0b77fb157 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: also sanitize error message in bridge-db log call Sanitize lastErr.message (which may contain a file path from ENOENT errors) alongside groupDir to prevent CRLF injection from error message content. Addressed code review feedback. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/0452a6ce-711f-4203-9ae6-5dd0b77fb157 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: address security review findings — credential hygiene and test coverage [LOW] Redact credentials from URL validation error messages: - validateLLMBaseUrl: malformed URL no longer echoes raw input; scheme error shows protocol only; http-non-loopback error uses parsed.origin (scheme+host+port) instead of full URL - validateBackendUrl: same treatment — no raw input in any error path [INFO] Add state-preservation test for setBackendUrl: - Proves _backendUrl is unchanged after a rejected call, covering the validation-before-assignment ordering. [INFO] Expand validateLLMBaseUrl adversarial test coverage: - LOCALHOST uppercase (case-fold path) - RFC 1918 / IMDS IPs (10.x, 169.254.x) - Hostname-spoofing (localhost.evil.com, 127.0.0.1.evil.com, localhost.) - Non-loopback IPv6 (fe80::1, ::ffff:127.0.0.1) - ftp:// scheme - Credential-hygiene assertion (sk-secret not in error message) Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/7bb18fa2-3e66-4fe0-949f-6d493fbd351b Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * style: prettier autoformat U11 security fix files Fixes the failing 'quality / format' check on PR #1456 by running 'prettier --write' over the 6 files touched by the security fix. Formatting only — no logic change. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> Co-authored-by: Gergo Magyar --- gitnexus-web/src/services/backend-client.ts | 26 +++++- .../test/unit/server-connection.test.ts | 66 +++++++++++++- gitnexus/src/core/group/bridge-db.ts | 14 +-- gitnexus/src/core/wiki/llm-client.ts | 46 ++++++++++ gitnexus/src/server/api.ts | 15 +++- gitnexus/test/unit/wiki-llm-client.test.ts | 86 +++++++++++++++++++ 6 files changed, 242 insertions(+), 11 deletions(-) diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index 506d48f38..e887e3901 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -205,8 +205,32 @@ export function streamSSE(url: string, handlers: SSEHandlers): A let _backendUrl = 'http://localhost:4747'; +/** + * Validate that a backend URL is a safe http:// or https:// origin before + * storing it as the fetch target base (CodeQL js/client-side-request-forgery). + * + * Throws if the URL uses a non-HTTP scheme (e.g. javascript:, data:, file://). + * All other well-formed http/https URLs are accepted — the client intentionally + * supports connecting to remote GitNexus servers, not just localhost. + */ +export function validateBackendUrl(url: string): void { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + // Do not echo raw input — it may contain credentials. + throw new Error('Invalid backend URL: must be a well-formed http:// or https:// URL'); + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + // Use parsed.protocol only (scheme), not the full URL, to avoid leaking credentials. + throw new Error(`Backend URL must use http:// or https:// (got ${parsed.protocol})`); + } +} + export const setBackendUrl = (url: string): void => { - _backendUrl = url.replace(/\/$/, ''); + const trimmed = url.replace(/\/$/, ''); + validateBackendUrl(trimmed); + _backendUrl = trimmed; }; export const getBackendUrl = (): string => _backendUrl; diff --git a/gitnexus-web/test/unit/server-connection.test.ts b/gitnexus-web/test/unit/server-connection.test.ts index f5ee43c53..e39b829a1 100644 --- a/gitnexus-web/test/unit/server-connection.test.ts +++ b/gitnexus-web/test/unit/server-connection.test.ts @@ -1,5 +1,11 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { fetchGraph, normalizeServerUrl, setBackendUrl } from '../../src/services/backend-client'; +import { + fetchGraph, + getBackendUrl, + normalizeServerUrl, + setBackendUrl, + validateBackendUrl, +} from '../../src/services/backend-client'; describe('normalizeServerUrl', () => { it('adds http:// to localhost', () => { @@ -165,3 +171,61 @@ describe('fetchGraph', () => { }); }); }); + +describe('validateBackendUrl', () => { + it('allows http:// URLs', () => { + expect(() => validateBackendUrl('http://localhost:4747')).not.toThrow(); + expect(() => validateBackendUrl('http://127.0.0.1:4747')).not.toThrow(); + }); + + it('allows https:// URLs', () => { + expect(() => validateBackendUrl('https://gitnexus.example.com')).not.toThrow(); + expect(() => validateBackendUrl('https://my-server.internal:4747')).not.toThrow(); + }); + + it('rejects non-http schemes', () => { + expect(() => validateBackendUrl('javascript:alert(1)')).toThrow('must use http:// or https://'); + expect(() => validateBackendUrl('file:///etc/passwd')).toThrow('must use http:// or https://'); + expect(() => validateBackendUrl('data:text/plain,evil')).toThrow( + 'must use http:// or https://', + ); + }); + + it('rejects malformed URLs', () => { + expect(() => validateBackendUrl('not-a-url')).toThrow('Invalid backend URL'); + }); + + it('does not include the raw URL in error messages (credential hygiene)', () => { + const urlWithCreds = 'javascript:alert("sk-secret")'; + let msg = ''; + try { + validateBackendUrl(urlWithCreds); + } catch (e) { + msg = (e as Error).message; + } + expect(msg).not.toContain('sk-secret'); + expect(msg).not.toContain(urlWithCreds); + }); +}); + +describe('setBackendUrl', () => { + it('accepts valid http URLs', () => { + expect(() => setBackendUrl('http://localhost:4747')).not.toThrow(); + }); + + it('accepts valid https URLs', () => { + expect(() => setBackendUrl('https://my-server.example.com')).not.toThrow(); + }); + + it('rejects non-http/https schemes', () => { + expect(() => setBackendUrl('javascript:alert(1)')).toThrow('must use http:// or https://'); + expect(() => setBackendUrl('file:///etc/passwd')).toThrow('must use http:// or https://'); + }); + + it('does not mutate _backendUrl when validation fails', () => { + setBackendUrl('http://localhost:4747'); + expect(() => setBackendUrl('javascript:alert(1)')).toThrow(); + // State must be preserved — validation must happen before the assignment + expect(getBackendUrl()).toBe('http://localhost:4747'); + }); +}); diff --git a/gitnexus/src/core/group/bridge-db.ts b/gitnexus/src/core/group/bridge-db.ts index ef6244b22..7f44253bf 100644 --- a/gitnexus/src/core/group/bridge-db.ts +++ b/gitnexus/src/core/group/bridge-db.ts @@ -722,13 +722,15 @@ export async function openBridgeDbReadOnly(groupDir: string): Promise setTimeout(r, delay)); } } - // Pino's NDJSON serialization is structurally injection-resistant - // (CodeQL js/log-injection): groupDir and err.message are JSON-escaped - // by the serializer, so no manual CRLF / U+2028 / ANSI sanitization is - // needed. Demoted to debug — only fires when the bridge truly gave up - // after retries, and operators only need it at debug verbosity. + // Strip CRLF from user-controlled strings before logging to close + // CodeQL js/log-injection. Pino's NDJSON serialization already + // JSON-escapes all values, but we sanitize here as a defence-in-depth + // measure so CodeQL can see the taint flow is broken. + const safeGroupDir = String(groupDir).replace(/[\r\n]/g, ' '); + const safeErrMsg = + lastErr instanceof Error ? String(lastErr.message).replace(/[\r\n]/g, ' ') : undefined; bridgeLogger.debug( - { groupDir, err: lastErr, attempts: LBUG_OPEN_RETRY_ATTEMPTS }, + { groupDir: safeGroupDir, errMsg: safeErrMsg, attempts: LBUG_OPEN_RETRY_ATTEMPTS }, 'openBridgeDbReadOnly gave up', ); return null; diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts index 7f9cc8312..37fe7a9f2 100644 --- a/gitnexus/src/core/wiki/llm-client.ts +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -77,6 +77,49 @@ export function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } +/** + * Validate that a base URL supplied for LLM API calls is a safe HTTP/HTTPS + * endpoint (CWE-918 / CodeQL js/http-to-file-access). + * + * Allowed: + * - https:// with any hostname (public LLM APIs, Azure, OpenRouter, …) + * - http:// restricted to localhost / 127.0.0.1 (local servers: Ollama, LiteLLM, …) + * + * Rejected: + * - file://, data:, javascript:, and any other non-HTTP scheme + * - http:// aimed at non-loopback hosts (avoids SSRF against internal networks) + * + * Throws with a descriptive message on validation failure so callers surface a + * clear error rather than an opaque network error. + */ +export function validateLLMBaseUrl(baseUrl: string): void { + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + // Do not include the raw input in the message — it may contain credentials. + throw new Error('Invalid LLM base URL: must be a well-formed http:// or https:// URL'); + } + + if (!['https:', 'http:'].includes(parsed.protocol)) { + // Use parsed.protocol only (scheme), not the full URL, to avoid leaking credentials. + throw new Error(`LLM base URL must use http:// or https:// (got ${parsed.protocol})`); + } + + if (parsed.protocol === 'http:') { + // Node's URL parser preserves IPv6 brackets in hostname (e.g. "[::1]"), + // so strip them before comparing to bare address literals. + const host = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, ''); + if (host !== 'localhost' && host !== '127.0.0.1' && host !== '::1') { + // Use parsed.origin (scheme+host+port, no credentials) instead of the full URL. + throw new Error( + `Insecure http:// LLM base URLs are only allowed for localhost/127.0.0.1. ` + + `Use https:// for remote endpoints (got ${parsed.origin})`, + ); + } + } +} + /** * Returns true if the given base URL is an Azure OpenAI endpoint. * Uses proper hostname matching to avoid spoofed URLs like @@ -128,6 +171,9 @@ export async function callLLM( systemPrompt?: string, options?: CallLLMOptions, ): Promise { + // Validate base URL before any fetch (CodeQL js/http-to-file-access) + validateLLMBaseUrl(config.baseUrl); + const messages: Array<{ role: string; content: string }> = []; if (systemPrompt) { messages.push({ role: 'system', content: systemPrompt }); diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index cc65daa3d..2d49fabc4 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -744,8 +744,13 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => if (isMatch && ['queued', 'cloning', 'analyzing'].includes(job.status)) { if (process.env.DEBUG) { - console.log( - `[debug] resolveRepo waiting for active job ${job.id} (${normalizedName})...`, + // Sanitize user-controlled values to prevent log injection (CodeQL js/log-injection). + logger.debug( + { + jobId: String(job.id).replace(/[\r\n]/g, ' '), + repoName: String(normalizedName).replace(/[\r\n]/g, ' '), + }, + '[debug] resolveRepo waiting for active job', ); } for (let wait = 0; wait < HOLD_QUEUE_TIMEOUT_SECS; wait++) { @@ -769,7 +774,11 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // (e.g. registry file not yet flushed after clone completes). if (!found && normalizedName && !isRetry) { if (process.env.DEBUG) { - console.log(`[debug] resolveRepo 404 for "${normalizedName}". Triggering deep init...`); + // Sanitize user-controlled values to prevent log injection (CodeQL js/log-injection). + logger.debug( + { repoName: String(normalizedName).replace(/[\r\n]/g, ' ') }, + '[debug] resolveRepo 404, triggering deep init', + ); } await backend.init(); return await resolveRepo(normalizedName, true, req); diff --git a/gitnexus/test/unit/wiki-llm-client.test.ts b/gitnexus/test/unit/wiki-llm-client.test.ts index c9d429904..52b633566 100644 --- a/gitnexus/test/unit/wiki-llm-client.test.ts +++ b/gitnexus/test/unit/wiki-llm-client.test.ts @@ -5,6 +5,7 @@ import { isAzureProvider, isReasoningModel, buildRequestUrl, + validateLLMBaseUrl, } from '../../src/core/wiki/llm-client.js'; describe('isAzureProvider', () => { @@ -330,3 +331,88 @@ describe('readSSEStream — content_filter handling', () => { ).rejects.toThrow('content filter'); }); }); + +describe('validateLLMBaseUrl', () => { + it('allows https:// for any public host', () => { + expect(() => validateLLMBaseUrl('https://api.openai.com/v1')).not.toThrow(); + expect(() => validateLLMBaseUrl('https://openrouter.ai/api/v1')).not.toThrow(); + expect(() => validateLLMBaseUrl('https://myres.openai.azure.com/openai/v1')).not.toThrow(); + }); + + it('allows http:// for localhost', () => { + expect(() => validateLLMBaseUrl('http://localhost:11434/v1')).not.toThrow(); + expect(() => validateLLMBaseUrl('http://127.0.0.1:11434/v1')).not.toThrow(); + // IPv6 loopback — Node's URL parser preserves brackets in hostname: "[::1]" + expect(() => validateLLMBaseUrl('http://[::1]:11434/v1')).not.toThrow(); + }); + + it('allows http:// for LOCALHOST (uppercase) — lowercased before comparison', () => { + expect(() => validateLLMBaseUrl('http://LOCALHOST:11434/v1')).not.toThrow(); + }); + + it('rejects http:// for non-loopback hosts', () => { + expect(() => validateLLMBaseUrl('http://evil.example.com/v1')).toThrow('Insecure http://'); + expect(() => validateLLMBaseUrl('http://192.168.1.1/v1')).toThrow('Insecure http://'); + // Private IP ranges + expect(() => validateLLMBaseUrl('http://10.0.0.1/v1')).toThrow('Insecure http://'); + // AWS/GCP IMDS — should be blocked + expect(() => validateLLMBaseUrl('http://169.254.169.254/latest/meta-data')).toThrow( + 'Insecure http://', + ); + }); + + it('rejects http:// hostname-spoofing attempts', () => { + // Full-hostname comparison prevents prefix/suffix attacks + expect(() => validateLLMBaseUrl('http://localhost.evil.com/v1')).toThrow('Insecure http://'); + expect(() => validateLLMBaseUrl('http://127.0.0.1.evil.com/v1')).toThrow('Insecure http://'); + // Trailing dot — hostname 'localhost.' ≠ 'localhost' + expect(() => validateLLMBaseUrl('http://localhost./v1')).toThrow('Insecure http://'); + }); + + it('rejects http:// non-loopback IPv6 addresses', () => { + // Link-local IPv6 + expect(() => validateLLMBaseUrl('http://[fe80::1]/v1')).toThrow('Insecure http://'); + // IPv4-mapped IPv6 loopback — bracket-stripped to '::ffff:127.0.0.1' ≠ '::1' + expect(() => validateLLMBaseUrl('http://[::ffff:127.0.0.1]/v1')).toThrow('Insecure http://'); + }); + + it('rejects non-http schemes', () => { + expect(() => validateLLMBaseUrl('file:///etc/passwd')).toThrow('must use http:// or https://'); + expect(() => validateLLMBaseUrl('javascript:alert(1)')).toThrow('must use http:// or https://'); + expect(() => validateLLMBaseUrl('data:text/plain,evil')).toThrow( + 'must use http:// or https://', + ); + expect(() => validateLLMBaseUrl('ftp://example.com')).toThrow('must use http:// or https://'); + }); + + it('rejects malformed URLs', () => { + expect(() => validateLLMBaseUrl('not-a-url')).toThrow('Invalid LLM base URL'); + expect(() => validateLLMBaseUrl('')).toThrow('Invalid LLM base URL'); + }); + + it('does not include the raw URL in error messages (credential hygiene)', () => { + // Simulates a URL with an embedded API key + const urlWithCreds = 'http://192.168.1.1/v1?apikey=sk-secret'; + let msg = ''; + try { + validateLLMBaseUrl(urlWithCreds); + } catch (e) { + msg = (e as Error).message; + } + expect(msg).not.toContain('sk-secret'); + expect(msg).not.toContain(urlWithCreds); + }); + + it('callLLM rejects an invalid base URL before fetching', async () => { + const { callLLM } = await import('../../src/core/wiki/llm-client.js'); + await expect( + callLLM('prompt', { + apiKey: 'key', + baseUrl: 'file:///etc/passwd', + model: 'gpt-4o', + maxTokens: 100, + temperature: 0, + }), + ).rejects.toThrow('must use http:// or https://'); + }); +});