From 48106d3c00f3807413d7761f37cd4535d3cfff56 Mon Sep 17 00:00:00 2001 From: DuduPhudu <34869259+ReidenXerx@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:35:36 +0300 Subject: [PATCH 01/28] fix(ingestion): index NestJS decorator routes so api_impact and route_map stop reporting live endpoints as non-existent (#3017) --- ARCHITECTURE.md | 2 +- .../group/extractors/http-patterns/node.ts | 200 +---- .../core/ingestion/languages/typescript.ts | 2 + .../core/ingestion/pipeline-phases/routes.ts | 76 +- .../route-extractors/data-route-table.ts | 16 +- .../core/ingestion/route-extractors/nest.ts | 548 +++++++++++++ gitnexus/src/storage/parse-cache.ts | 65 +- .../api/widgetsController.ts | 26 + .../src/health/health.controller.ts | 14 + .../src/legacy/legacy.controller.ts | 16 + .../src/venues/venues.controller.ts | 35 + .../src/venues/venues.service.ts | 34 + .../multi-verb-route-identity.test.ts | 52 ++ .../integration/nest-route-pipeline.test.ts | 216 +++++ .../test/unit/group/nest-route-parity.test.ts | 289 +++++++ .../test/unit/incremental-parse-cache.test.ts | 32 +- .../test/unit/nest-decorator-routes.test.ts | 738 ++++++++++++++++++ 17 files changed, 2160 insertions(+), 201 deletions(-) create mode 100644 gitnexus/src/core/ingestion/route-extractors/nest.ts create mode 100644 gitnexus/test/fixtures/multi-verb-route-app/api/widgetsController.ts create mode 100644 gitnexus/test/fixtures/nest-route-app/src/health/health.controller.ts create mode 100644 gitnexus/test/fixtures/nest-route-app/src/legacy/legacy.controller.ts create mode 100644 gitnexus/test/fixtures/nest-route-app/src/venues/venues.controller.ts create mode 100644 gitnexus/test/fixtures/nest-route-app/src/venues/venues.service.ts create mode 100644 gitnexus/test/integration/nest-route-pipeline.test.ts create mode 100644 gitnexus/test/unit/group/nest-route-parity.test.ts create mode 100644 gitnexus/test/unit/nest-decorator-routes.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7ae30a56b..6fe547b70 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -174,7 +174,7 @@ converging on the routes phase's `(method, url)` registry: | Filesystem convention | path → URL, no parsing | Next.js `app/`, Expo, PHP | | Single-file framework route | `isRouteFile` + worker extraction | Laravel `routes/*.php` | | Cross-file framework route | `discoverRootRouteFiles` + `extractRoutes` | Django `urlpatterns` | -| AST-level route in a normal file | `extractDecoratorRoutes` | Spring, FastAPI, NestJS, **JS/TS dispatch guards and static data route tables** | +| AST-level route in a normal file | `extractDecoratorRoutes` | Spring, FastAPI, NestJS (`@Controller` + `@Get`/`@Post`/…; URLs are controller-relative — `setGlobalPrefix` and URI versioning live in the bootstrap file and are not applied), **JS/TS dispatch guards and static data route tables** | The last row is the one whose name undersells it. A route is DECLARED by a decorator, but it can also be **inferred** from a raw `node:http` server's own diff --git a/gitnexus/src/core/group/extractors/http-patterns/node.ts b/gitnexus/src/core/group/extractors/http-patterns/node.ts index fa7c453df..7a198f595 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/node.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/node.ts @@ -15,6 +15,8 @@ import { DATA_ROUTE_TABLE_SOURCE, scanDataRouteTables, } from '../../../ingestion/route-extractors/data-route-table.js'; +import { extractNestRoutes } from '../../../ingestion/route-extractors/nest.js'; +import { normalizeExtractedRoutePath } from '../../../ingestion/route-extractors/route-path.js'; import { buildJsRepoFacts, extractJsModuleFacts, @@ -27,7 +29,8 @@ import { /** * Node.js / TypeScript HTTP plugin family. Handles: - * - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods + * - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods, + * delegated wholesale to the indexer's `extractNestRoutes` * - Express `router.get(...)` / `app.post(...)` providers * - `fetch(url)` / `fetch(url, { method: 'POST' })` consumers * - `axios.get(url)` / `axios.delete(url)` consumers @@ -42,34 +45,8 @@ import { * same `scan` function but bind to different grammars. */ -// ─── Provider: NestJS — class-level @Controller('prefix') ──────────── -// In tree-sitter-typescript decorators are NOT children of -// class_declaration / method_definition — they're siblings in the -// surrounding class_body / program node. We therefore match the -// decorator standalone and walk to its related class/method in JS. -const NEST_CONTROLLER_SPEC: PatternSpec> = { - meta: {}, - query: ` - (decorator - (call_expression - function: (identifier) @dec (#eq? @dec "Controller") - arguments: (arguments . [(string) (template_string)] @prefix))) @ctrl_decorator - `, -}; - -// ─── Provider: NestJS — method-level @Get/@Post/... decorators ─────── -// Matches either `@Get('path')` or `@Get()`. The `@path` capture is -// optional — when the first argument isn't a string, the plugin falls -// back to '/' for the method-level path. -const NEST_METHOD_SPEC: PatternSpec> = { - meta: {}, - query: ` - (decorator - (call_expression - function: (identifier) @dec (#match? @dec "^(Get|Post|Put|Delete|Patch)$") - arguments: (arguments) @args)) @method_decorator - `, -}; +// NestJS providers are not queried here at all — see the `extractNestRoutes` +// call in `scanBundle`. // ─── Provider: Express — router.get/app.post/... ───────────────────── const EXPRESS_SPEC: PatternSpec> = { @@ -176,8 +153,6 @@ const AXIOS_OBJECT_SPEC: PatternSpec> = { }; interface NodePatternBundle { - controller: CompiledPatterns>; - methodDecorator: CompiledPatterns>; express: CompiledPatterns>; fetchNoOptions: CompiledPatterns>; fetchWithOptions: CompiledPatterns>; @@ -195,8 +170,6 @@ function compileBundle(language: unknown, name: string): NodePatternBundle { patterns: [spec], } satisfies LanguagePatterns>); return { - controller: mk(NEST_CONTROLLER_SPEC, 'nest-controller'), - methodDecorator: mk(NEST_METHOD_SPEC, 'nest-method-decorator'), express: mk(EXPRESS_SPEC, 'express'), fetchNoOptions: mk(FETCH_NO_OPTIONS_SPEC, 'fetch-no-options'), fetchWithOptions: mk(FETCH_WITH_OPTIONS_SPEC, 'fetch-with-options'), @@ -211,33 +184,6 @@ const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-http'); const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-http'); const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-http'); -const NEST_DECORATOR_TO_HTTP: Record = { - Get: 'GET', - Post: 'POST', - Put: 'PUT', - Delete: 'DELETE', - Patch: 'PATCH', -}; - -/** - * Find the nearest enclosing class_declaration for a node, or null. - */ -function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let cur: Parser.SyntaxNode | null = node.parent; - while (cur) { - if (cur.type === 'class_declaration') return cur; - cur = cur.parent; - } - return null; -} - -function joinPath(prefix: string, sub: string): string { - const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, ''); - const cleanSub = sub.replace(/^\/+/, ''); - if (!cleanPrefix) return `/${cleanSub}`; - return `/${cleanPrefix}/${cleanSub}`; -} - /** * Walk `pair` children of an `object` literal and return the unquoted * string/template_string value for the first pair whose key matches one @@ -260,68 +206,6 @@ function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string return null; } -/** - * For a standalone `decorator` node (child of class_body / program), - * find the related `class_declaration` node that it decorates. In - * tree-sitter-typescript the decorator is placed before the class - * declaration as a sibling (when decorating a class) or inside the - * class_body before a method_definition (when decorating a method); - * we walk the parent chain until we find the enclosing class. - */ -function findDecoratedClass(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null { - const parent = decoratorNode.parent; - if (!parent) return null; - // Case 1: decorator is a sibling of the class_declaration at program / - // export_statement level. Walk forward through siblings until we find - // the class_declaration this decorator belongs to. - for (let i = 0; i < parent.namedChildCount; i++) { - const child = parent.namedChild(i); - if (child && child.id === decoratorNode.id) { - for (let j = i + 1; j < parent.namedChildCount; j++) { - const next = parent.namedChild(j); - if (!next) continue; - if (next.type === 'decorator') continue; // adjacent decorators stack - if (next.type === 'class_declaration') return next; - if (next.type === 'export_statement') { - // `export class Foo { ... }` wraps the declaration. - for (let k = 0; k < next.namedChildCount; k++) { - const inner = next.namedChild(k); - if (inner?.type === 'class_declaration') return inner; - } - } - break; - } - break; - } - } - // Case 2: decorator is inside a class_body (decorating a method) — - // walk up to the enclosing class_declaration. - return findEnclosingClass(decoratorNode); -} - -/** - * For a method-level decorator node (child of class_body before a - * method_definition), find the method_definition it decorates. - */ -function findDecoratedMethod(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null { - const parent = decoratorNode.parent; - if (!parent || parent.type !== 'class_body') return null; - for (let i = 0; i < parent.namedChildCount; i++) { - const child = parent.namedChild(i); - if (child && child.id === decoratorNode.id) { - for (let j = i + 1; j < parent.namedChildCount; j++) { - const next = parent.namedChild(j); - if (!next) continue; - if (next.type === 'decorator') continue; - if (next.type === 'method_definition') return next; - return null; - } - return null; - } - } - return null; -} - /** * Map each named import's LOCAL binding to its DECLARED export name and source * module, by walking the file's `import { x as y } from 'm'` statements. Lets @@ -589,57 +473,33 @@ function scanBundle( // symbol resolves to the real definition rather than its local alias text. const importMap = buildImportMap(tree); - // NestJS: collect `@Controller('prefix')` class decorators, keyed by - // the `class_declaration` they decorate. - const prefixByClassId = new Map(); - for (const match of runCompiledPatterns(bundle.controller, tree)) { - const prefixNode = match.captures.prefix; - const decoratorNode = match.captures.ctrl_decorator; - if (!prefixNode || !decoratorNode) continue; - const prefix = unquoteLiteral(prefixNode.text); - if (prefix === null) continue; - const classNode = findDecoratedClass(decoratorNode); - if (!classNode) continue; - prefixByClassId.set(classNode.id, prefix); - } - - // NestJS: method-level @Get/@Post/... decorators. The decorator's - // arguments list may be empty (`@Get()`), a string (`@Get('path')`), - // or something else (which we skip). - for (const match of runCompiledPatterns(bundle.methodDecorator, tree)) { - const decNode = match.captures.dec; - const argsNode = match.captures.args; - const decoratorNode = match.captures.method_decorator; - if (!decNode || !argsNode || !decoratorNode) continue; - const httpMethod = NEST_DECORATOR_TO_HTTP[decNode.text]; - if (!httpMethod) continue; - const methodNode = findDecoratedMethod(decoratorNode); - if (!methodNode) continue; - const enclosingClass = findEnclosingClass(methodNode); - // Only emit NestJS detections when the class actually has a - // @Controller decorator — without it, the match is almost certainly - // something else (e.g. an unrelated library using similar names). - if (!enclosingClass || !prefixByClassId.has(enclosingClass.id)) continue; - const prefix = prefixByClassId.get(enclosingClass.id) ?? ''; - - let rawPath = '/'; - const firstArg = argsNode.namedChild(0); - if (firstArg && (firstArg.type === 'string' || firstArg.type === 'template_string')) { - const unquoted = unquoteLiteral(firstArg.text); - if (unquoted !== null) rawPath = unquoted; - } - - // Get the method name from the decorated method_definition. - const methodNameNode = methodNode.childForFieldName('name'); - const name = methodNameNode?.text ?? null; - + // NestJS: delegated to the indexer's extractor rather than re-queried here. + // Two independent readings of the same decorators is how the layers drift: + // the local scan saw only `class_declaration` (never `abstract class`), only + // five of the nine verbs, only a positional string `@Controller('x')`, and + // — worst — INVENTED `/` for a method path it could not read, so + // `@Get(ROUTES.SEARCH)` became a `GET /venues` contract that the graph, which + // correctly drops it, has no Route node for. "A missing route is a coverage + // limit; an invented one is a lie" (ARCHITECTURE.md). Calling the extractor + // makes that divergence structurally impossible, exactly as the + // `scanDataRouteTables` call below already does for static route tables. + // + // `filePath` rides only on the returned struct and never reaches the + // `HttpDetection`, so a bare `scan(tree)` with no `fileRel` passes '' rather + // than losing the routes. `lineOffset` is 0: the group scanner parses whole + // files, so `lineNumber` is already the absolute 1-based line this + // `HttpDetection.line` wants. + for (const route of extractNestRoutes(tree, fileRel ?? '', 0)) { out.push({ role: 'provider', framework: 'nest', - method: httpMethod, - path: joinPath(prefix, rawPath), - name, - line: methodNode.startPosition.row + 1, + method: route.httpMethod, + // The prefix travels separately at the ingestion layer, so the join is + // ours to do — with ingestion's own joiner, so the two layers cannot + // disagree about the URL either. + path: normalizeExtractedRoutePath(route.routePath, route.prefix ?? null), + name: route.handlerName ?? null, + line: route.lineNumber, confidence: 0.8, }); } diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index e6106df69..4cf95f9c9 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -126,11 +126,13 @@ import { } from './javascript/index.js'; import { extractDispatchGuardRoutes } from '../route-extractors/dispatch-guard.js'; import { extractDataRouteTableRoutes } from '../route-extractors/data-route-table.js'; +import { extractNestRoutes } from '../route-extractors/nest.js'; import { extractConvexEndpointProperties } from './typescript/convex-endpoint-metadata.js'; const extractJsTsRoutes = (...args: Parameters) => [ ...extractDispatchGuardRoutes(...args), ...extractDataRouteTableRoutes(...args), + ...extractNestRoutes(...args), ]; /** diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index 8115f5b40..e93fa033c 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -195,6 +195,45 @@ export const routesPhase: PipelinePhase = { const allFetchCalls = [...parseFetchCalls]; const routeRegistry = new Map(); + /** + * Registry keys written straight from the file list below, never through + * `addRoute`. `resolveRouteHandlerSymbols` walks only `extractedRoutes` and + * `decoratorRoutes`, so it never sees these URLs and its `claimed` set never + * contains them — which means a handler stamped on one of these keys was + * resolved for a DIFFERENT route. + * + * That is reachable, and it fabricates rather than omits (#3049). A + * method-agnostic route (`@All`, a Django function view, a verb-less + * dispatch guard) keys by URL alone via `routeNodeKey`, so it collides with + * a file-convention route at the same URL. It claims the key unopposed in + * `claim()`, then loses first-writer-wins here in `addRoute` and is dropped + * as a duplicate — and without this guard the surviving file-convention node + * would read that handler and present another application's controller + * method as its own. `api_impact` is documented to be run BEFORE editing a + * route handler, so it would answer with a handler from the wrong app. + * + * Dropping the losing route is a separate and deliberate consequence of + * URL-only identity; this only stops the false attribution. + * + * Membership is recorded AT the pre-seeding `set`, mirroring `claim()` in + * call-processor.ts, which writes `claimed` and its result map together + * rather than re-deriving either by rescanning. Identifying pre-seeded + * entries by matching `entry.source` against a list of source strings would + * spell them a second time, away from the sites that produce them — and a + * fourth pre-seeded source added later would then reopen #3049 in silence. + * `addRoute` deliberately does NOT record here: its routes ARE + * handler-resolved, so suppressing them would widen the guard into a bug of + * its own. + * + * Each `add` below sits inside its own `!routeRegistry.has(key)` gate, as + * every `routeRegistry.set` in this phase does: the map is write-once per + * key, so a losing candidate cannot record a key it did not claim and no + * later writer can take a recorded key away. Key-membership is therefore + * equivalent to source-matching by construction — pre-seeded routes carry no + * verb and `routeNodeKey(undefined, url) === url` — which is why the + * two-candidates-one-URL case needs no fixture to settle it. + */ + const preSeededKeys = new Set(); // Detect Expo Router app/ roots vs Next.js app/ roots (monorepo-safe) const expoAppRoots = new Set(); @@ -217,32 +256,33 @@ export const routesPhase: PipelinePhase = { } } + // One writer for every pre-seeded route, so recording membership cannot be + // forgotten. Inlining `has` / `set` / `add` at each site made the invariant + // a convention three call sites had to remember — and a fourth source that + // forgot the `add` would reopen #3049 exactly as silently as the source-set + // it replaced. This is the shape `claim()` in call-processor.ts uses for the + // same reason: one helper writes the collection and its key set together. + const preSeed = (url: string, entry: Omit): boolean => { + if (routeRegistry.has(url)) return false; + routeRegistry.set(url, { ...entry, url }); + preSeededKeys.add(url); + return true; + }; + for (const p of allPaths) { if (expoAppPaths.has(p)) { const expoURL = expoFileToRouteURL(p); - if (expoURL && !routeRegistry.has(expoURL)) { - routeRegistry.set(expoURL, { - filePath: p, - source: 'expo-filesystem-route', - url: expoURL, - }); + if (expoURL && preSeed(expoURL, { filePath: p, source: 'expo-filesystem-route' })) { continue; } } const nextjsURL = nextjsFileToRouteURL(p); - if (nextjsURL && !routeRegistry.has(nextjsURL)) { - routeRegistry.set(nextjsURL, { - filePath: p, - source: 'nextjs-filesystem-route', - url: nextjsURL, - }); + if (nextjsURL && preSeed(nextjsURL, { filePath: p, source: 'nextjs-filesystem-route' })) { continue; } if (p.endsWith('.php')) { const phpURL = phpFileToRouteURL(p); - if (phpURL && !routeRegistry.has(phpURL)) { - routeRegistry.set(phpURL, { filePath: p, source: 'php-file-route', url: phpURL }); - } + if (phpURL) preSeed(phpURL, { filePath: p, source: 'php-file-route' }); } } @@ -311,7 +351,11 @@ export const routesPhase: PipelinePhase = { const { source: routeSource, method: routeMethod, url } = entry; const handlerPath = handlerPathFor(routeKey, entry); const content = handlerContents.get(handlerPath); - const handlerSymbolId = routeHandlerSymbols.get(routeKey); + // A pre-seeded route can never legitimately appear in + // `routeHandlerSymbols`, so a key that does is a route that LOST (#3049). + const handlerSymbolId = preSeededKeys.has(routeKey) + ? undefined + : routeHandlerSymbols.get(routeKey); const analysisContent = entry.source === DATA_ROUTE_TABLE_SOURCE && content ? handlerSymbolContent( diff --git a/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts b/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts index 053d207fe..bcc879e3c 100644 --- a/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts +++ b/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts @@ -116,7 +116,14 @@ function decodeJavaScriptStringLiteral(raw: string): string | null { return decoded; } -function plainString(node: SyntaxNode): string | null { +/** + * A `string`/`template_string` node's decoded value, or `null` when it is not a + * readable literal (an interpolated template, an unterminated escape). Shared + * with the NestJS extractor so both agree on what a readable literal is — + * notably that escapes must be DECODED, not dropped, because tree-sitter splits + * a literal around every `escape_sequence`. + */ +export function plainString(node: SyntaxNode): string | null { if (node.type === 'string') return decodeJavaScriptStringLiteral(node.text); if ( node.type === 'template_string' && @@ -129,7 +136,12 @@ function plainString(node: SyntaxNode): string | null { return null; } -function propertyName(node: SyntaxNode): string | null { +/** + * A property key's name, for the spellings that carry one — `{ path: … }` and + * `{ 'path': … }`. A computed key (`{ [KEY]: … }`) has none. Shared with the + * NestJS extractor, which reads `@Controller({ path: … })` the same way. + */ +export function propertyName(node: SyntaxNode): string | null { if (node.type === 'identifier' || node.type === 'property_identifier') return node.text; if (node.type === 'string') return plainString(node); return null; diff --git a/gitnexus/src/core/ingestion/route-extractors/nest.ts b/gitnexus/src/core/ingestion/route-extractors/nest.ts new file mode 100644 index 000000000..136c6354a --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/nest.ts @@ -0,0 +1,548 @@ +/** + * NestJS decorator routes for the indexer. + * + * A NestJS endpoint is declared across two decorators: `@Controller('venues')` + * on the class supplies the prefix, and `@Get('search')` on a method supplies + * the verb and the remainder. Neither half is a route on its own, which is why + * a pattern that only looks at one of them finds nothing. + * + * Until this existed, TypeScript's `extractDecoratorRoutes` hook was dispatch + * guards plus static data route tables only, so a NestJS repo produced + * essentially no `Route` nodes. That is not a quiet gap: `route_map`, + * `api_impact` and `shape_check` all read `Route` nodes and answer "no routes + * matching …" when there are none — so `api_impact`, whose documented job is to + * be run BEFORE modifying a route handler, reported every live endpoint as + * non-existent, and a not-found reads as a safe change (#3009). + * + * The extraction mirrors `spring.ts`, which solves the identical shape for + * `@RequestMapping` + `@GetMapping`: collect class-level prefixes keyed by class + * node id, then walk method decorators and attach the prefix of their enclosing + * class. As there, the prefix travels on `ExtractedDecoratorRoute.prefix` and + * the routes phase performs the join via `normalizeExtractedRoutePath`, so + * NestJS routes are keyed identically to every other framework's. + * + * The multi-path form `@Get(['a', 'b'])` mounts the handler at BOTH paths, so + * it emits both routes: N paths is N elements of the returned + * `ExtractedDecoratorRoute[]`, which is already how this layer spells N routes + * — the same representation `spring.ts` reaches for `@GetMapping({"/a","/b"})`, + * and the reason neither needs a special case downstream. The CLASS-level array + * (`@Controller(['a', 'b'])`) is DECLINED rather than cross-multiplied over the + * class's methods, again matching `spring.ts`: there an array-form class prefix + * only ever suppresses the class, with the cross-product tracked in #2280. + * + * Known limitation: the URLs produced here are CONTROLLER-RELATIVE. A global + * prefix (`app.setGlobalPrefix('api')`) and URI versioning are applied by the + * bootstrap file, not by any decorator this file can see, so neither is + * reflected — a route served at `/api/v1/venues/search` is stored as + * `/venues/search`. The module's "drop rather than guess" floor is unavailable + * for it: the evidence lives in a different file, so honouring it would mean + * dropping every Nest route in every repo. `spring.ts` has the same hole for + * `server.servlet.context-path`; `ExtractedDecoratorRoute.prefix` is the + * channel a cross-file follow-up would use, the way FastAPI resolves its mount. + */ + +import type Parser from 'tree-sitter'; +import type { ExtractedDecoratorRoute } from '../workers/parse-worker.js'; +import { plainString, propertyName } from './data-route-table.js'; +import { isDev } from '../utils/env.js'; +import { logger } from '../../logger.js'; + +/** + * NestJS method decorators → HTTP verb. A Map rather than an object literal + * because the lookup key is an arbitrary decorator name read out of source: a + * plain object answers `@toString()` with `Object.prototype.toString`, which is + * truthy and would be emitted verbatim as the route's httpMethod. + */ +const NEST_METHOD_DECORATORS: ReadonlyMap = new Map([ + ['Get', 'GET'], + ['Post', 'POST'], + ['Put', 'PUT'], + ['Patch', 'PATCH'], + ['Delete', 'DELETE'], + ['Head', 'HEAD'], + ['Options', 'OPTIONS'], + ['All', '*'], + // `@Sse` mounts a real GET endpoint that streams; it is as much a route as + // `@Get`. `@Search` is deliberately absent — `normalizeRouteMethod` rejects + // SEARCH as non-standard and would key the route by URL alone, colliding + // with every other verb on that path. + ['Sse', 'GET'], +]); + +/** + * Class node types that can carry a `@Controller`. `export abstract class C` + * parses as `abstract_class_declaration`, a DIFFERENT node type — and a + * decorated abstract base sharing CRUD routes with its subclasses is ordinary + * Nest, so matching `class_declaration` alone silently drops the whole + * controller rather than one route. + */ +const CLASS_DECLARATION_TYPES: ReadonlySet = new Set([ + 'class_declaration', + 'abstract_class_declaration', +]); + +/** + * Cheap parse-free gate. Every JS/TS file in every repo reaches this hook, so + * skip the walk unless the file could plausibly declare a controller. A file + * without the substring cannot produce a route here, because a `@Controller` + * decorator is REQUIRED before any method decorator is believed (see below). + */ +const CONTROLLER_HINT = '@Controller'; + +/** The decorator's name — `Controller` for `@Controller('x')`, `Get` for `@Get()`. */ +function decoratorName(decorator: Parser.SyntaxNode): string | null { + const inner = decorator.namedChild(0); + if (!inner) return null; + // `@Get()` is a call_expression; a bare `@Injectable` is a plain identifier. + if (inner.type === 'identifier') return inner.text; + if (inner.type === 'call_expression') { + const fn = inner.childForFieldName('function'); + return fn?.type === 'identifier' ? fn.text : null; + } + return null; +} + +/** + * The literal path(s) a decorator call mounts, one entry per path — or `['']` + * when the decorator takes no argument (`@Controller()` / `@Get()` — both legal + * and both meaning "no path segment of my own"). + * + * A list rather than a single string because `@Get(['a', 'b'])` mounts the + * handler at two URLs, and two routes is what the caller's output contract + * already says that in: `ExtractedDecoratorRoute[]`. No new field, and no + * special case at the emit site — the same shape `spring.ts` gets for free from + * a query that matches one element at a time. + * + * Returns `null` when an argument IS present but is not a readable literal. + * That is deliberately distinct from `['']`: a computed prefix + * (`@Controller(ROUTES.VENUES)`) whose value we cannot read must drop the route + * rather than silently mount it at the wrong URL. `route_map` presents its + * output as fact, and a wrong path is worse than a missing one. `[]` is a third + * answer and means neither of those: `@Get([])` is legal, knowably mounts + * nothing, and so emits nothing — it must never be read as the unknowable case, + * which is the one that suppresses a whole controller. + * + * Reading one literal is delegated to `plainString`, the same judge the + * data-route-table extractor uses, so both agree on what is readable. Filtering + * `string_fragment` children and joining them looks equivalent and is not: + * tree-sitter SPLITS a literal around each `escape_sequence`, and the join then + * DELETES the escape rather than decoding it. `@Get(':id(\\d+)')` — the ordinary + * spelling of a Nest regex param, whose value is `:id(\d+)` — came out as + * `:id(d+)`, and `@Get('/v\u0069ews')` came out as `/vews`. Both are paths the + * app never serves, i.e. the wrong-URL outcome the paragraph above forbids. + */ +function decoratorLiteralPaths(decorator: Parser.SyntaxNode): readonly string[] | null { + const call = decorator.namedChild(0); + // A Nest route decorator is a FACTORY: `@Get()` invokes it and returns the + // decorator that registers the route. A BARE `@Get` is the factory itself, + // never applied, so Nest registers nothing — emitting a route for it would + // publish a URL the app does not serve. The same holds one level up for a + // bare `@Controller`, which registers no controller. + // + // `@Get()` with no ARGUMENT is different and still a real pathless route: + // what distinguishes them is the call, not the argument list. That case falls + // through to the `!first` branch below. + if (call?.type !== 'call_expression') return null; + const first = call.childForFieldName('arguments')?.namedChild(0); + if (!first) return ['']; + // The object form belongs to `@Controller` alone — a verb decorator takes + // `string | string[]`, so Nest mounts nothing for `@Get({ path: 'a' })`. + // Reading it as a route would mint a URL the app never serves, which is the + // invented fact this module refuses; an unreadable shape drops instead. + if (first.type === 'object' && decoratorName(decorator) !== 'Controller') return null; + return literalPaths(first); +} + +/** + * The paths carried by one decorator ARGUMENT node, split out from + * {@link decoratorLiteralPaths} only so the object form can re-enter it: Nest + * accepts an array inside `{ path: … }` as well, and reusing the same judge is + * what keeps `@Controller({ path: ['a', 'b'] })` from being read by a second, + * laxer set of rules that has drifted from this one. + */ +function literalPaths(node: Parser.SyntaxNode): readonly string[] | null { + // `@Controller({ path: 'cats', version: '1' })` is the documented form for + // URI/header versioning, and its path is a plain literal sitting right there. + // Worth reading rather than dropping, because the asymmetry is severe: an + // unreadable METHOD path costs one route, an unreadable PREFIX costs every + // route on the class. + if (node.type === 'object') { + // But a `path` pair only PROVES the mount when nothing else in the object + // can replace it, and the first match proves nothing on its own: + // `{ path: 'cats', ...options }` mounts wherever `options.path` says, and + // `{ path: 'cats', path: 'dogs' }` mounts at `dogs` — last write wins in + // both. Either one publishes `/cats`, a URL the app never serves, and it + // looks exactly like a correct one, which is the wrong-answer-dressed-as- + // fact this module refuses. So the object is read only when EVERY member is + // a named, non-repeated pair. That whole-entry fail-closed walk is the + // shape `routeFromObject` uses in `data-route-table.ts`. + const values = new Map(); + for (const child of node.namedChildren) { + // Skipped FIRST. A comment between two pairs is ordinary formatting; run + // through the not-a-pair test below it would refuse the object and cost + // the class every route it has, over a comment. + if (child.type === 'comment') continue; + // `spread_element` (`{ ...options }`), `shorthand_property_identifier` + // (`{ path }`) and `method_definition` (`{ getFoo() {} }`) all land here + // — probed and identical across the three grammars this extractor runs + // under. None offers a key/value this file can read, and the first can + // introduce or overwrite `path` from a value declared elsewhere. + if (child.type !== 'pair') return null; + const key = child.childForFieldName('key'); + const value = child.childForFieldName('value'); + if (key === null || value === null) return null; + // Compared through `propertyName`, the same judge used to READ the key — + // so `{ path: … }` and `{ 'path': … }` are one key and collide as + // duplicates. Comparing raw key text instead makes them two distinct + // keys, and `{ path: 'cats', 'path': 'dogs' }` silently mounts the loser. + const name = propertyName(key); + // No readable name means a computed key (`{ [dynamicKey]: 'b' }`), which + // could evaluate to `path` and take the mount with it — refused, not + // ignored. A repeated key is refused wherever it appears, not only on + // `path`: a duplicate anywhere is evidence the object is not the fixed + // literal it reads as, and cost is one controller against a wrong URL. + if (name === null || values.has(name)) return null; + values.set(name, value); + } + + // Deliberately NOT `containsExecutingExpression` (data-route-table.ts): that + // guards whole-entry declarativeness for a static route table, a different + // invariant. Here only `path` has to be provable, so a non-literal value on + // an unrelated key — `{ path: 'a', scope: Scope.REQUEST }`, ordinary Nest — + // stays benign and keeps its controller. + const path = values.get('path'); + // A missing `path` keeps the existing drop and must never read as `''`: + // `@Controller({ version: '1' })` mounts at a prefix this decorator does + // not state, and `''` would publish every one of its methods at the root. + return path === undefined ? null : literalPaths(path); + } + + // `array` is the node type in all three grammars this extractor runs under — + // tree-sitter-typescript's `typescript` and `tsx`, and tree-sitter-javascript + // — probed rather than assumed, because a name that differs in one of them + // would silently restore the old drop for that grammar alone. + if (node.type === 'array') { + const paths: string[] = []; + for (const element of node.namedChildren) { + const value = plainString(element); + // One unreadable element poisons the whole array. Emitting the readable + // ones would present a partial mapping as a complete one — the endpoint + // behind `ROUTES.ADMIN` would be missing from a controller that otherwise + // looks fully covered, which is the same wrong-answer-dressed-as-fact this + // module refuses above, only harder to notice. + if (value === null) return null; + paths.push(value); + } + return paths; + } + + const value = plainString(node); + return value === null ? null : [value]; +} + +/** + * Decorators that immediately precede `node` among its parent's named children. + * In tree-sitter-typescript a decorator is a SIBLING placed before the thing it + * decorates — at `export_statement`/`program` level for a class — and + * decorators stack. + * + * Walks the sibling chain rather than indexing into `parent.namedChildren`, + * which is the same uncached-getter trap {@link collectClassRoutes} documents: + * a class's parent is usually `program`, so reading the list marshals every + * top-level statement in the file, once per class. That is quadratic in + * top-level statements — measured 200ms for a file of 800 classes, against + * 0.9ms for this form. + */ +function precedingDecorators(node: Parser.SyntaxNode): Parser.SyntaxNode[] { + const out: Parser.SyntaxNode[] = []; + for (let sibling = node.previousNamedSibling; sibling; sibling = sibling.previousNamedSibling) { + // A comment between the decorators and the thing they decorate is ordinary + // (`@Post('x')` then a JSDoc block then the method) and must not terminate + // the stack — doing so makes the whole decorated route invisible. + if (sibling.type === 'comment') continue; + if (sibling.type !== 'decorator') break; + out.push(sibling); + } + return out; +} + +/** + * Leading `decorator` children of a node, stopping at the first child that is + * neither a decorator nor a comment. Comments are skipped for the same reason + * as in {@link precedingDecorators}: a doc block sitting between `@Controller` + * and the class must not hide the decorator. + */ +function leadingDecorators(node: Parser.SyntaxNode): Parser.SyntaxNode[] { + const out: Parser.SyntaxNode[] = []; + for (const child of node.namedChildren) { + if (child.type === 'comment') continue; + if (child.type !== 'decorator') break; + out.push(child); + } + return out; +} + +/** + * Cap on the decorator text quoted in the dropped-controller log. Long enough + * to identify the shape, short enough not to dump a wrapped multi-line + * decorator into the operator's terminal. + */ +const DROPPED_CONTROLLER_LOG_LIMIT = 160; + +/** + * Every decorator attached to a class, across the two shapes the grammar + * produces — which differ by whether the class is exported: + * + * `@Controller('a') class A {}` → decorator is a CHILD of class_declaration + * `@Controller('a') export class A {}` → decorator is a child of export_statement, + * i.e. a SIBLING of the class_declaration + * + * Checking only one of them silently drops half of all controllers, so collect + * from both, plus the sibling position for the class itself. There is no fourth + * source: both grammars fold a class's decorators INTO the `export_statement` + * production, so an `export_statement` never has one as a preceding sibling. + */ +function classDecorators(classNode: Parser.SyntaxNode): Parser.SyntaxNode[] { + const out = [...leadingDecorators(classNode), ...precedingDecorators(classNode)]; + const wrapper = classNode.parent; + if (wrapper?.type === 'export_statement') out.push(...leadingDecorators(wrapper)); + return out; +} + +/** + * The `@Controller(...)` prefix for a class, or undefined when it has none. + * One string, not a list: a class-level array (`@Controller(['a', 'b'])`) is + * DECLINED here, exactly as `spring.ts` declines an array-form + * `@RequestMapping` — it detects the shape only to suppress the class, leaving + * the prefix × method cross-product to #2280. Collapsing to `null` is that + * suppression, and this parity is deliberate, not an oversight: the two + * extractors solve the same shape and should not disagree about which half of + * it is supported. + */ +function controllerPrefix( + classNode: Parser.SyntaxNode, + filePath: string, +): string | null | undefined { + for (const decorator of classDecorators(classNode)) { + if (decoratorName(decorator) !== 'Controller') continue; + const paths = decoratorLiteralPaths(decorator); + // `@Controller([])` lands here too and needs no answer of its own: a + // controller mounted at no path serves no route, so "emit nothing for this + // class" is what both readings of it come to. + if (paths === null || paths.length !== 1) { + // The single funnel for EVERY whole-controller drop — an unreadable + // constant (`@Controller(ROUTES.VENUES)`), a multi-path array, an + // unreadable array element, and an options object whose `path` another + // member could override all return null here. Reporting at the refusal + // sites instead would make the rarest cause the loudest, and leave the + // motivating one from this module's own header silent. + // + // `isDev` at `info`, not `debug`: the logger's base level IS `info`, so + // an isDev-gated `debug` is gated twice and stays silent in exactly the + // dev run it exists for. Same shape the routes phase uses. + if (isDev) { + const shape = decorator.text.replace(/\s+/g, ' '); + logger.info( + `🗺️ NestJS: dropped @Controller in ${filePath} — its prefix is not provable: ${ + shape.length > DROPPED_CONTROLLER_LOG_LIMIT + ? `${shape.slice(0, DROPPED_CONTROLLER_LOG_LIMIT)}…` + : shape + }`, + ); + } + return null; + } + return paths[0]; + } + return undefined; +} + +/** + * Extract NestJS routes from one parsed TypeScript/JavaScript file. + * + * A method decorator is only believed when its enclosing class carries a + * `@Controller`. `@Get`/`@Post`/`@Delete` are common identifiers, and without + * that requirement any unrelated library using the same decorator names would + * mint phantom endpoints. + */ +export function extractNestRoutes( + tree: Parser.Tree, + filePath: string, + lineOffset = 0, +): ExtractedDecoratorRoute[] { + if (!tree.rootNode.text.includes(CONTROLLER_HINT)) return []; + + const out: ExtractedDecoratorRoute[] = []; + + const visit = (node: Parser.SyntaxNode): void => { + if (CLASS_DECLARATION_TYPES.has(node.type)) { + const prefix = controllerPrefix(node, filePath); + // `undefined` — not a controller at all. `null` — a controller whose + // prefix could not be read, so its routes' URLs are unknowable. + if (prefix !== undefined) { + if (prefix !== null) collectClassRoutes(node, prefix, filePath, lineOffset, out); + return; // a controller's methods are handled here; don't re-walk them + } + } + for (const child of node.namedChildren) visit(child); + }; + + visit(tree.rootNode); + return out; +} + +/** + * Modifiers that take a `method_definition` out of Nest's handler set. + * + * Nest's `RequestMapping` writes the handler onto the class PROTOTYPE's + * `descriptor.value`, and `RouterExplorer` scans prototype instance methods for + * that metadata. A `static` method lives on the constructor and is never + * scanned; an accessor's descriptor carries `get`/`set` and no `value` to + * register. A verb decorator on any of the three therefore mounts NOTHING, so a + * route minted from one is a URL the app does not serve — the invented fact + * this module refuses everywhere else. + */ +const NON_HANDLER_MODIFIERS: ReadonlySet = new Set(['static', 'get', 'set']); + +/** Longest entry above — the cheap gate that keeps `.trim()` off a method body. */ +const LONGEST_NON_HANDLER_MODIFIER = 6; + +/** + * Whether Nest could register this `method_definition` as a request handler. + * + * Reads `children`, NOT `namedChildren`, and that is the whole difficulty: + * `static`, `get` and `set` are ANONYMOUS tokens in all three grammars this + * extractor runs under, so they never appear among named children. A static + * method, a getter, a setter and a plain method expose the IDENTICAL + * `namedChildren` (`property_identifier`, `formal_parameters`, + * `statement_block`) — probed, not assumed — so the module's usual + * `namedChildren` idiom cannot see the modifier at all and every one of the + * three reads as an ordinary handler. + * + * Matches on child TEXT, not node type, and skips the `name` field — the same + * two rules `hasKeyword` in `field-extractors/configs/helpers.ts` applies, and + * that the TS/JS captures and method extractor already use for this question. + * The text rule is load-bearing: `static` reaches the tree as an anonymous + * token in some grammar versions and a keyword node in others, so a + * `child.type === 'static'` test silently stops firing on a grammar bump — here + * that would readmit exactly the phantom routes this function removes, with the + * suite still green. Skipping `name` is what keeps a method literally called + * `get()` or `static()` from reading as a modifier. + * + * Open-coded rather than calling `hasKeyword` three times, which was measured + * at 13.96us per method against 4.30us here: that helper takes ONE keyword, so + * three keywords is three full passes, and it calls `.text.trim()` on every + * child including `statement_block` — the whole method body. `.some()` does not + * rescue it, since a real handler matches nothing and pays all three. The + * length guard keeps `.trim()` off a multi-KB body; no modifier exceeds it. + * + * The scan is bounded by one method's own children (a handful), so it is not + * the uncached-getter trap {@link collectClassRoutes} documents — that one bites + * when a PARENT's child list is re-marshalled once per member. + */ +function isRequestHandler(member: Parser.SyntaxNode): boolean { + const nameNode = member.childForFieldName('name'); + for (const child of member.children) { + if (child === nameNode) continue; + const text = child.text; + if (text.length <= LONGEST_NON_HANDLER_MODIFIER && NON_HANDLER_MODIFIERS.has(text.trim())) { + return false; + } + } + return true; +} + +function collectClassRoutes( + classNode: Parser.SyntaxNode, + prefix: string, + filePath: string, + lineOffset: number, + out: ExtractedDecoratorRoute[], +): void { + const body = classNode.childForFieldName('body'); + if (!body) return; + + // ONE forward pass over the body, accumulating the decorator run and flushing + // it at each method. Calling `precedingDecorators` per method instead is + // quadratic in methods-per-controller for a reason that is invisible in the + // source: `namedChildren` is an UNCACHED getter in node-tree-sitter, so every + // call re-marshals the entire class body into fresh JS objects before the + // `findIndex`. Measured here, 800 methods cost 362ms (450us/method, up from + // 42us/method at 50); a single pass is flat. `spring.ts` never had this + // because a Java annotation is a child of the declaration it annotates. + const pending: Parser.SyntaxNode[] = []; + + for (const member of body.namedChildren) { + if (member.type === 'decorator') { + pending.push(member); + continue; + } + // Same reason as in `precedingDecorators`: a JSDoc block between a + // decorator stack and its method must not hide the route (a real + // controller shape, pinned by the suite). Known limitation of that skip: a + // decorator ORPHANED by a commented-out handler is then absorbed onto the + // NEXT method, minting a phantom route with the wrong handler. There is no + // AST fix — an orphan followed by a comment is indistinguishable from a + // stack whose method happens to be documented — and losing every + // documented route is the worse trade, so it is made deliberately. + if (member.type === 'comment') continue; + + // tree-sitter-javascript makes a method decorator a CHILD of the + // `method_definition`, not a preceding sibling as in tree-sitter-typescript + // — and this extractor is registered on the JavaScript provider too, which + // already advertises `framework: 'nestjs'`. Reading only siblings meant + // every `.js` Nest controller emitted nothing. On TypeScript the first + // named child is the method name, so `leadingDecorators` contributes + // nothing there and no route is collected twice. + // + // A static member, a getter and a setter are decorated exactly like a + // handler and registered as none, so they contribute no decorators (see + // `isRequestHandler`). They still fall THROUGH to the `pending.length = 0` + // below rather than `continue` past it: skipping the clear would hand their + // decorator run to the next method, trading a phantom route for a + // misattributed one — the strictly worse of the two, since it corrupts a + // route that is otherwise correct. + const decorators = + member.type === 'method_definition' && isRequestHandler(member) + ? [...pending, ...leadingDecorators(member)] + : []; + for (const decorator of decorators) { + const name = decoratorName(decorator); + if (name === null) continue; + const httpMethod = NEST_METHOD_DECORATORS.get(name); + if (httpMethod === undefined) continue; + + const routePaths = decoratorLiteralPaths(decorator); + if (routePaths === null) continue; // unreadable → skip + + const handlerName = member.childForFieldName('name')?.text; + + // One route per path. `@Get(['a', 'b'])` mounts the handler at both, and + // everything else about the two is identical — same verb, same handler, + // same line — so the loop is the whole of the multi-path support. An + // empty array falls out as zero iterations without a special case. + for (const routePath of routePaths) { + out.push({ + filePath, + // A pathless `@Get()` is the controller's index route and carries no + // segment of its own. Emit '/' rather than '': `claim()` in + // call-processor short-circuits on a falsy routePath, so an empty + // string would still produce the Route node but silently lose its + // handler symbol — the route would exist with nothing attached to it. + // Both spellings normalize to the same URL against the prefix. + routePath: routePath === '' ? '/' : routePath, + httpMethod, + decoratorName: name, + lineNumber: member.startPosition.row + 1 + lineOffset, + prefix: prefix === '' ? null : prefix, + ...(handlerName === undefined ? {} : { handlerName }), + }); + } + } + + // Anything that is not a decorator or a comment ends the run — including + // the method that just consumed it, so a decorated FIELD's stack is never + // absorbed onto the method after it. + pending.length = 0; + } +} diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 64ddc8264..8cc639900 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -586,8 +586,71 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // detects the changed analyzer build in run-analyze.ts. Both guards are // required; a parse-cache bump alone must never be read as a graph rebuild. // Version 75 is intentionally skipped because concurrent PR #3017 claims it. +// +// 74 -> 75 adds #3009's NestJS decorator routes to the JS/TS decoratorRoutes +// channel. Same reasoning as 69: a warm pre-feature cache replays unchanged +// worker results, which for every already-indexed NestJS repo means replaying +// the empty route set this change exists to fix — the fix would appear to do +// nothing until something else invalidated the cache. +// +// 75, not 71 (this branch's original claim) and not 74: origin/main cascaded +// past both while this PR was open. #2980 took 71, #3046 claims 73, and the +// Convex endpoint-metadata change took 74 (skipping 73 for exactly that +// reason). 75 is the next free value above origin/main AND above every +// in-flight claim — the rule the ledger states, not "one above main". Every +// open PR touching gitnexus/src/storage/parse-cache.ts was scanned at this +// merge: #3046 (73) and #1616 (a stale 2) are the only other claimants. // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING. -const SCHEMA_BUMP = 76; +// +// 75 -> 76 for the NestJS multi-path (array) form: `@Get(['a','b'])` now yields +// one `decoratorRoutes` entry per path where it previously yielded none. That +// changes worker output for the same file, and 75 was already claimed earlier +// on this same branch — so a warm cache written by a dev or CI build at v75, +// before the array form landed, would replay the pre-feature captures under a +// BYTE-IDENTICAL `PARSE_CACHE_VERSION` and the array form would be inert. The +// package version is untouched here, so it cannot rescue that case. Same-branch +// re-bumping is unusual, but the ledger's rule is about what a warm cache can +// replay, not about how the value was reached. +// +// 76 -> 77 because 76 was NOT free. While this branch sat in review, origin/main +// advanced to ac68f5254 and #3046 took 76 — the value this branch already held. +// `gitnexus/package.json` is 1.6.9 on both sides, so `PARSE_CACHE_VERSION` was +// the byte-identical string `76+1.6.9` on two branches that changed +// incompatible worker output. Every warm cache would have been reused across +// both features, making both inert while every test stayed green. +// +// The sharpest part: #3046 skipped 75 BECAUSE this branch held it, then took +// 76 — and this branch had meanwhile moved 75 -> 76 for the array form. Two +// PRs each doing the bookkeeping correctly still collided, because each +// re-checked once and neither re-checked after the other moved. That is what +// the re-check line below is for, and why it says AT MERGE rather than +// when you pick the number. +// +// 77 is free at this merge: origin/main is 76, and the open PRs touching this +// constant are #2840 (a stale 71) and #1616 (a stale 2). Scan with the contents +// API at each PR head, not `gh pr diff` — that exits non-zero on an +// inaccessible fork and prints nothing, so a grep over its output skips the PR +// silently. #2840 was missed exactly that way this round. +// +// WHY THIS IS STILL A HAND-PICKED NUMBER, when `SCHEMA_FINGERPRINT` next door +// is a derived sha256 that cannot collide. The derivation exists and already +// runs: `resolveAnalyzerRunnerIdentity` computes `build.digest` over the +// analyzer build tree on every analyze. It is not used here because it moves on +// ANY build change — a comment-only edit, this paragraph included — so every +// dev and CI rebuild would force a full re-parse of every repo. That is the +// expensive half of the trade, and `PARSE_CACHE_VERSION` already carries the +// package version, so this counter's only job is separating builds that SHARE +// one: dev, CI, unreleased. Exactly the population a whole-build digest would +// punish, and exactly the population that hits the exact-clash failure. +// +// So the counter stays, and the open cost is that a bump nobody makes is +// invisible: a capture change with no bump ships inert and no check can see it. +// #2860 (a base-branch CI comparison) closes the "not greater than main" axis +// only. A digest over just the determinant subset — the language queries plus +// `route-extractors/` and `workers/` module content — would close the missing- +// bump axis without invalidating on unrelated churn, and is the real follow-up. +// RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING. +const SCHEMA_BUMP = 77; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/test/fixtures/multi-verb-route-app/api/widgetsController.ts b/gitnexus/test/fixtures/multi-verb-route-app/api/widgetsController.ts new file mode 100644 index 000000000..4a5c0f676 --- /dev/null +++ b/gitnexus/test/fixtures/multi-verb-route-app/api/widgetsController.ts @@ -0,0 +1,26 @@ +/** + * A NestJS controller mounting a METHOD-AGNOSTIC route at `/api/widgets` — the + * same URL `app/api/widgets/route.ts` produces as a Next.js filesystem route. + * + * `@All` maps to httpMethod '*', which `routeNodeKey` keys by URL alone, so + * this route collides with the filesystem one. It is the shape behind #3049. + * + * `@Get('gadgets')` is the NON-colliding companion, and it is here for a + * reason the `@All` route cannot serve: because the `@All` route loses its key + * to the filesystem node and is dropped as a duplicate, it leaves NO trace in + * the graph at all — so the whole of this file could stop being extracted + * without a single assertion changing. `GET /api/gadgets` is the only witness + * this fixture can offer that NestJS extraction ran (see the test's comment). + */ +@Controller('api') +export class WidgetsController { + @All('widgets') + handleEveryVerb() { + return 'nest handler'; + } + + @Get('gadgets') + listGadgets() { + return '[]'; + } +} diff --git a/gitnexus/test/fixtures/nest-route-app/src/health/health.controller.ts b/gitnexus/test/fixtures/nest-route-app/src/health/health.controller.ts new file mode 100644 index 000000000..462ba7717 --- /dev/null +++ b/gitnexus/test/fixtures/nest-route-app/src/health/health.controller.ts @@ -0,0 +1,14 @@ +import { Controller, Get } from '@nestjs/common'; + +/** + * A controller with no prefix of its own. `@Controller()` is legal NestJS and + * means "mount at the root", so the method path must reach the graph bare — + * this is the fixture half of the extractor's `prefix: '' -> null` mapping. + */ +@Controller() +export class HealthController { + @Get('health') + health(): string { + return 'ok'; + } +} diff --git a/gitnexus/test/fixtures/nest-route-app/src/legacy/legacy.controller.ts b/gitnexus/test/fixtures/nest-route-app/src/legacy/legacy.controller.ts new file mode 100644 index 000000000..d91a8ee82 --- /dev/null +++ b/gitnexus/test/fixtures/nest-route-app/src/legacy/legacy.controller.ts @@ -0,0 +1,16 @@ +import { Controller, Get } from '@nestjs/common'; + +const ROUTE_PREFIXES = { legacy: 'legacy' }; + +/** + * The prefix is not a literal, so the URLs of this controller's methods are + * unknowable here. `route_map` presents its output as fact: a missing route is + * recoverable, a route mounted at the wrong URL is not. + */ +@Controller(ROUTE_PREFIXES.legacy) +export class LegacyController { + @Get('reports') + legacyReports(): string { + return 'legacy'; + } +} diff --git a/gitnexus/test/fixtures/nest-route-app/src/venues/venues.controller.ts b/gitnexus/test/fixtures/nest-route-app/src/venues/venues.controller.ts new file mode 100644 index 000000000..4a519f31c --- /dev/null +++ b/gitnexus/test/fixtures/nest-route-app/src/venues/venues.controller.ts @@ -0,0 +1,35 @@ +import { Body, Controller, Delete, Get, Param, Post, Query } from '@nestjs/common'; +import { VenuesService } from './venues.service'; +import type { Venue } from './venues.service'; + +/** + * The shape #3009 is about: the URL is split across two decorators. The class + * decorator carries the prefix and the method decorator carries the verb plus + * the remainder, so neither half is a route on its own. + */ +@Controller('venues') +export class VenuesController { + constructor(private readonly venues: VenuesService) {} + + // Pathless index route: its URL is the controller prefix and nothing else. + @Get() + findAll(): Venue[] { + return this.venues.listVenues(); + } + + @Get('search') + search(@Query('q') term: string): Venue[] { + return this.venues.searchVenues(term); + } + + // Same URL as findAll(), different verb — two Route nodes, not one. + @Post() + create(@Body() input: Venue): Venue { + return this.venues.insertVenue(input); + } + + @Delete(':id') + remove(@Param('id') id: string): void { + this.venues.deleteVenue(id); + } +} diff --git a/gitnexus/test/fixtures/nest-route-app/src/venues/venues.service.ts b/gitnexus/test/fixtures/nest-route-app/src/venues/venues.service.ts new file mode 100644 index 000000000..b5225f100 --- /dev/null +++ b/gitnexus/test/fixtures/nest-route-app/src/venues/venues.service.ts @@ -0,0 +1,34 @@ +import { Injectable } from '@nestjs/common'; + +export interface Venue { + id: string; + name: string; +} + +/** + * Not a controller. It exists so each handler body calls a real symbol, and so + * the `@Controller` file gate is shown skipping a decorated class that declares + * no routes. + */ +@Injectable() +export class VenuesService { + private readonly rows: Venue[] = []; + + listVenues(): Venue[] { + return this.rows; + } + + searchVenues(term: string): Venue[] { + return this.rows.filter((row) => row.name.includes(term)); + } + + insertVenue(input: Venue): Venue { + this.rows.push(input); + return input; + } + + deleteVenue(id: string): void { + const index = this.rows.findIndex((row) => row.id === id); + if (index !== -1) this.rows.splice(index, 1); + } +} diff --git a/gitnexus/test/integration/multi-verb-route-identity.test.ts b/gitnexus/test/integration/multi-verb-route-identity.test.ts index 938c44305..d488c2522 100644 --- a/gitnexus/test/integration/multi-verb-route-identity.test.ts +++ b/gitnexus/test/integration/multi-verb-route-identity.test.ts @@ -12,6 +12,8 @@ * Fixture: `test/fixtures/multi-verb-route-app/` * - ItemController.java: GET /api/items, POST /api/items, GET /api/widgets * - app/api/widgets/route.ts: Next.js filesystem route → /api/widgets + * - api/widgetsController.ts: NestJS `@All` → method-agnostic /api/widgets, + * plus `@Get('gadgets')` → GET /api/gadgets (the non-colliding witness) * - web/itemsClient.ts: verb-less fetch() consumers of both URLs */ import { describe, it, expect, beforeAll } from 'vitest'; @@ -81,6 +83,56 @@ describe('Multi-verb Route node identity (#2289)', () => { expect(decoratorNode!.properties.method).toBe('GET'); }); + it("never stamps a losing route's handler onto a filesystem node (#3049)", () => { + // The NestJS `@All('widgets')` route keys by URL alone (routeNodeKey drops + // '*'), so it collides with the filesystem route, claims the key unopposed + // in claim(), and is then dropped here by first-writer-wins. Its handler + // must not survive that loss: a Next.js Route node carrying a NestJS + // controller method is a fabricated fact, not a missing one, and + // api_impact is documented to be run BEFORE editing a route handler. + const fsNode = routeNode(generateId('Route', '/api/widgets')); + + expect(fsNode, 'filesystem Route node /api/widgets should exist').toBeTruthy(); + expect(fsNode!.properties.handlerSymbolId).toBeUndefined(); + }); + + it('extracts a NON-colliding NestJS route (the witness that #3049 suppressed a REAL claim)', () => { + // Load-bearing for the assertion above it, which cannot stand alone: + // `handlerSymbolId === undefined` passes identically whether the guard + // correctly dropped a real NestJS `@All` claim or whether NestJS + // extraction produced nothing at all. Disproved by experiment — commenting + // out `...extractNestRoutes(...args)` in `languages/typescript.ts` and + // rebuilding `dist/` left the whole suite green. + // + // Asserting that the `WidgetsController` class and its `handleEveryVerb` + // method exist does NOT repair that, and is the first thing the next + // reader will try: those nodes come from the definitions phase, which runs + // regardless, while `extractNestRoutes` is wired only as the + // `decoratorRoutes` hook and contributes no class or method node. + // `@All('widgets')` also cannot witness its own extraction — it collides + // on `routeNodeKey`, is dropped as a duplicate, and leaves no graph trace. + // A second route at a URL nothing else claims is the only evidence the + // graph can carry, so this node existing IS the proof the `@All` claim the + // guard suppressed was real. + const gadgets = routeNode(routeId('GET', '/api/gadgets')); + + expect(gadgets, 'NestJS GET /api/gadgets Route node should exist').toBeTruthy(); + + const handler = result.graph.getNode(String(gadgets!.properties.handlerSymbolId)); + expect(String(handler?.properties.name)).toBe('listGadgets'); + }); + + it('still resolves a same-URL route that did NOT lose its key', () => { + // The guard above is scoped to pre-seeded sources, so the Java + // `GET /api/widgets` node — a different key, resolved for itself — keeps + // its handler. Without this, the fix reads as "filesystem collisions are + // handler-less" when the real rule is "a route that lost donates nothing". + const decoratorNode = routeNode(routeId('GET', '/api/widgets')); + const handler = result.graph.getNode(String(decoratorNode!.properties.handlerSymbolId)); + + expect(String(handler?.properties.name)).toBe('getWidgets'); + }); + it('connects a verb-less fetch() consumer to every Route node at the URL', () => { const consumerFileId = generateId('File', 'web/itemsClient.ts'); // Collect FETCHES targets without a test-level conditional: pass the diff --git a/gitnexus/test/integration/nest-route-pipeline.test.ts b/gitnexus/test/integration/nest-route-pipeline.test.ts new file mode 100644 index 000000000..7ea13e8ee --- /dev/null +++ b/gitnexus/test/integration/nest-route-pipeline.test.ts @@ -0,0 +1,216 @@ +/** + * End-to-end coverage of NestJS `@Controller` + `@Get`/`@Post`/`@Delete` route + * ingestion (#3009). + * + * The unit suite (`test/unit/nest-decorator-routes.test.ts`) pins what the + * extractor RETURNS. Nothing there proves the return value becomes anything: + * the reported symptom was not a wrong `ExtractedDecoratorRoute`, it was + * `api_impact` — whose documented job is to be run BEFORE modifying a route + * handler — reporting every live endpoint as non-existent, because the graph + * held no `Route` nodes at all. That is the tier this file covers: the routes + * phase performing the prefix/path join, `claim()` in call-processor resolving + * `handlerName` to a real symbol UID, and `prefix: null` surviving both. + * + * The fixture lives at `test/fixtures/nest-route-app/`, mirroring + * `spring-route-app/` — Spring solves the identical two-decorator shape and + * `nest.ts` is modelled on it. + */ + +import { describe, it, expect, beforeAll } from 'vitest'; +import path from 'node:path'; +import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; +import type { PipelineResult } from '../../src/types/pipeline.js'; + +const FIXTURE = path.resolve(__dirname, '..', 'fixtures', 'nest-route-app'); + +// Compared against POSIX-normalized graph paths (see `routes()`), so these stay +// forward-slashed rather than going through `path.join` — the Windows shard +// would otherwise compare backslashes against the graph's forward slashes. +const CONTROLLER_FILE = 'src/venues/venues.controller.ts'; +const HEALTH_FILE = 'src/health/health.controller.ts'; + +interface RouteView { + /** `${method} ${url}` — the `routeNodeKey` identity, as a sortable string. */ + readonly identity: string; + /** The handler the route resolved to, or the literal `'undefined'`. */ + readonly handler: string; + readonly handlerLabel: string; + readonly handlerFile: string; +} + +describe('NestJS decorator route ingestion pipeline', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(FIXTURE, () => {}, {}); + }, 120_000); + + const nodes = (): readonly GraphNode[] => { + const out: GraphNode[] = []; + result.graph.forEachNode((node) => void out.push(node)); + return out; + }; + + const relationships = (): readonly GraphRelationship[] => { + const out: GraphRelationship[] = []; + result.graph.forEachRelationship((rel) => void out.push(rel)); + return out; + }; + + const routeNodes = (): readonly GraphNode[] => nodes().filter((node) => node.label === 'Route'); + + /** + * Unresolved fields are stringified rather than branched on, so a route that + * lost its handler reads as the literal `'undefined'` in the diff instead of + * quietly skipping an assertion. + */ + const routes = (): readonly RouteView[] => + routeNodes() + .map((node) => { + const handler = result.graph.getNode(String(node.properties.handlerSymbolId)); + return { + identity: `${String(node.properties.method)} ${String(node.properties.name)}`, + handler: String(handler?.properties.name), + handlerLabel: String(handler?.label), + handlerFile: String(handler?.properties.filePath).replaceAll('\\', '/'), + }; + }) + .sort((a, b) => a.identity.localeCompare(b.identity)); + + const identities = (): readonly string[] => routes().map((route) => route.identity); + + it('emits Route nodes at all — the reported symptom was zero', () => { + // Its own case because every expectation below is satisfiable by an empty + // graph in the ways that matter least: `not.toContain` passes vacuously on + // an empty array, and a full-table `toEqual` against `[]` reports a missing + // element rather than "the extractor never ran". + expect(routeNodes().length).toBeGreaterThan(0); + }); + + it('mounts a pathless @Get() at the controller prefix WITH its handler attached', () => { + // The reason `nest.ts` emits '/' instead of '' for a pathless decorator: + // `claim()` short-circuits on a falsy `routePath`, so '' would still create + // this Route node and silently drop `handlerSymbolId`. The unit test can + // only see `routePath === '/'` at the extractor boundary; the guard being + // load-bearing is observable here and nowhere else. + expect(routes().find((route) => route.identity === 'GET /venues')).toEqual({ + identity: 'GET /venues', + handler: 'findAll', + handlerLabel: 'Method', + handlerFile: CONTROLLER_FILE, + }); + }); + + it('joins each @Controller prefix with its method paths and resolves every handler', () => { + // Two invariants, one table, because a projection of this assertion can + // only fail where this one already does. The join is what the extractor + // cannot do alone — it ships `prefix` and the routes phase folds it in via + // `normalizeExtractedRoutePath`, so these read `/venues/search`, not + // `search`. And `claim()` is first-writer-wins per `(method, url)`, so a + // mis-keyed route surfaces as a handler donated to the wrong URL rather + // than as an absence. + expect(routes()).toEqual([ + { + identity: 'DELETE /venues/:id', + handler: 'remove', + handlerLabel: 'Method', + handlerFile: CONTROLLER_FILE, + }, + { + identity: 'GET /health', + handler: 'health', + handlerLabel: 'Method', + handlerFile: HEALTH_FILE, + }, + { + identity: 'GET /venues', + handler: 'findAll', + handlerLabel: 'Method', + handlerFile: CONTROLLER_FILE, + }, + { + identity: 'GET /venues/search', + handler: 'search', + handlerLabel: 'Method', + handlerFile: CONTROLLER_FILE, + }, + { + identity: 'POST /venues', + handler: 'create', + handlerLabel: 'Method', + handlerFile: CONTROLLER_FILE, + }, + ]); + }); + + it('splits one URL into one Route node per verb', () => { + // `@Get()` and `@Post()` on the same controller share a URL. `routeNodeKey` + // makes them distinct identities; collapsing them would drop a live + // endpoint and hand its handler to the survivor. + expect( + routes() + .filter((route) => route.identity.endsWith(' /venues')) + .map((route) => `${route.identity} -> ${route.handler}`), + ).toEqual(['GET /venues -> findAll', 'POST /venues -> create']); + }); + + it('carries a prefix-less @Controller() through to a bare URL', () => { + // `@Controller()` maps to `prefix: null`, and null must reach the join as + // "no prefix" rather than as the string 'null' or a leading empty segment. + expect(identities()).toContain('GET /health'); + expect(identities().filter((id) => id.includes('//'))).toEqual([]); + }); + + it('records per-decorator provenance on the HANDLES_ROUTE edge', () => { + // Nest routes are DECLARED by an annotation, so they take the generic + // `decorator-` source rather than a bespoke one — the same channel + // Spring and FastAPI use, which is the point of modelling on `spring.ts`. + const routeIds = new Set(routeNodes().map((node) => node.id)); + expect( + [ + ...new Set( + relationships() + .filter((rel) => rel.type === 'HANDLES_ROUTE' && routeIds.has(rel.targetId)) + .map((rel) => String(rel.reason)), + ), + ].sort(), + ).toEqual(['decorator-Delete', 'decorator-Get', 'decorator-Post']); + }); + + describe('precision — what must NOT become a route', () => { + // Absence assertions are satisfied just as well by a file that was never + // read, so prove ingestion first; otherwise this whole block is decoration. + it('ingested the unreadable-prefix controller', () => { + expect( + nodes() + .filter((node) => String(node.properties.filePath ?? '').endsWith('legacy.controller.ts')) + .map((node) => String(node.properties.name)), + ).toEqual(expect.arrayContaining(['LegacyController', 'legacyReports'])); + }); + + it('drops a controller whose prefix is not a literal', () => { + // `@Controller(ROUTE_PREFIXES.legacy)` — the URL is unknowable, and + // `route_map` presents its output as fact. Emitting `/reports` here would + // be a wrong route, which is worse than a missing one. + expect(identities().filter((id) => id.includes('reports'))).toEqual([]); + expect(identities().filter((id) => id.includes('legacy'))).toEqual([]); + }); + + it('ingested the decorated non-controller service', () => { + expect( + nodes() + .filter((node) => String(node.properties.filePath ?? '').endsWith('venues.service.ts')) + .map((node) => String(node.properties.name)), + ).toEqual(expect.arrayContaining(['VenuesService', 'listVenues', 'searchVenues'])); + }); + + it('does not mint routes from a class with no @Controller', () => { + // Verb decorators are believed only inside a `@Controller` class; without + // that gate any library sharing the names would mint phantom endpoints. + expect(routes().filter((route) => route.handlerFile.endsWith('venues.service.ts'))).toEqual( + [], + ); + }); + }); +}); diff --git a/gitnexus/test/unit/group/nest-route-parity.test.ts b/gitnexus/test/unit/group/nest-route-parity.test.ts new file mode 100644 index 000000000..c0ed482d6 --- /dev/null +++ b/gitnexus/test/unit/group/nest-route-parity.test.ts @@ -0,0 +1,289 @@ +/** + * Parity guard for the two NestJS route layers. + * + * GitNexus reads `@Controller` / `@Get` decorators for two consumers: the + * indexer's `route-extractors/nest.ts` (which mints graph `Route` nodes) and + * the group layer's `http-patterns/node.ts` (which mints cross-repo HTTP + * contracts). They used to be two independent tree-sitter scans, and that is + * the shape #2265 already showed to be a slow leak: there the group query + * matched Spring's array form `@GetMapping({"/a","/b"})` and ingestion's did + * not, so the graph silently under-covered what the contracts claimed. Nest had + * the same divergence pointing the other way and worse — the group scan + * INVENTED `/` for any method path it could not read, so `@Get(ROUTES.SEARCH)` + * became a `GET /venues` contract with no Route node behind it. "A missing + * route is a coverage limit; an invented one is a lie" (ARCHITECTURE.md). + * + * The group layer now CALLS `extractNestRoutes` instead of re-querying the + * decorators, so the two cannot disagree by construction. What this file + * guards is that the call stays wired and keeps its `HttpDetection` shape: the + * assertions below pair each group result with the value computed straight from + * `extractNestRoutes` + `normalizeExtractedRoutePath`, and also pin the literal + * expected URLs, so a mutual regression cannot pass by having both sides go + * quiet together. + */ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import JavaScript from 'tree-sitter-javascript'; +import TypeScript from 'tree-sitter-typescript'; +import { + JAVASCRIPT_HTTP_PLUGIN, + TYPESCRIPT_HTTP_PLUGIN, +} from '../../../src/core/group/extractors/http-patterns/node.js'; +import type { HttpLanguagePlugin } from '../../../src/core/group/extractors/http-patterns/types.js'; +import { extractNestRoutes } from '../../../src/core/ingestion/route-extractors/nest.js'; +import { normalizeExtractedRoutePath } from '../../../src/core/ingestion/route-extractors/route-path.js'; + +// Compiled tree-sitter queries are grammar-bound, so a plugin must be driven +// with a tree parsed by ITS grammar. +interface Lang { + readonly parser: Parser; + readonly plugin: HttpLanguagePlugin; +} + +function lang(grammar: unknown, plugin: HttpLanguagePlugin): Lang { + const parser = new Parser(); + parser.setLanguage(grammar as Parameters[0]); + return { parser, plugin }; +} + +const TS = lang(TypeScript.typescript, TYPESCRIPT_HTTP_PLUGIN); +const JS = lang(JavaScript, JAVASCRIPT_HTTP_PLUGIN); + +/** `METHOD /full/url` pairs the GROUP layer reports as NestJS providers. */ +function groupPairs(src: string, target: Lang = TS): string[] { + return target.plugin + .scan(target.parser.parse(src)) + .filter((d) => d.role === 'provider' && d.framework === 'nest') + .map((d) => `${d.method} ${d.path}`) + .sort(); +} + +/** + * The same pairs computed straight from the indexer's extractor, joining the + * prefix the way the routes phase does. This is the reference the group layer + * must equal — and, since the group layer now calls the same function, the + * assertion is really "the call is still there and still joins the prefix". + */ +function ingestionPairs(src: string, target: Lang = TS): string[] { + return extractNestRoutes(target.parser.parse(src), 'venues.controller.ts') + .map((r) => `${r.httpMethod} ${normalizeExtractedRoutePath(r.routePath, r.prefix ?? null)}`) + .sort(); +} + +/** A minimal `@Controller('venues')` wrapping the given class-body members. */ +function venuesController(members: string): string { + return ` +import { Controller, Get, Post, Put, Patch, Delete, Head, Options, All, Sse } from '@nestjs/common'; + +@Controller('venues') +export class VenuesController { +${members} +} +`; +} + +describe('NestJS route parity — group node.ts delegates to ingestion nest.ts', () => { + const VERB_CASES: ReadonlyArray = [ + ['Get', 'GET'], + ['Post', 'POST'], + ['Put', 'PUT'], + ['Patch', 'PATCH'], + ['Delete', 'DELETE'], + // The four the group layer's own query never listed: its verb set stopped + // at Patch, so every @Head/@Options/@All/@Sse endpoint was invisible to + // contract matching while sitting in the graph as a Route node. + ['Head', 'HEAD'], + ['Options', 'OPTIONS'], + ['All', '*'], + // @Sse mounts a real streaming GET; '*' is the method-agnostic spelling + // `findMatchingKeys` already understands from Spring's @RequestMapping. + ['Sse', 'GET'], + ]; + + it.each(VERB_CASES)('pins the verb @%s → %s', (decorator, method) => { + const src = venuesController(` @${decorator}('slots')\n handler() {}`); + expect(groupPairs(src)).toEqual([`${method} /venues/slots`]); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('pins an abstract controller — a node type the group query never matched', () => { + // `export abstract class C` parses as `abstract_class_declaration`, a + // DIFFERENT node type from `class_declaration`. A decorated abstract base + // sharing CRUD routes with its subclasses is ordinary Nest, and the old + // group query dropped the WHOLE controller for it, not one route. + const src = ` +import { Controller, Get } from '@nestjs/common'; + +@Controller('venues') +export abstract class BaseVenuesController { + @Get('list') + list() {} +} +`; + expect(groupPairs(src)).toEqual(['GET /venues/list']); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('pins the object-form @Controller({ path }) — the documented versioning shape', () => { + // The old group query required a positional `(string)`/`(template_string)` + // argument, so `@Controller({ path: 'cats', version: '1' })` — the form the + // Nest docs give for URI/header versioning — suppressed the whole class. + const src = ` +import { Controller, Get } from '@nestjs/common'; + +@Controller({ path: 'venues', version: '1' }) +export class VenuesController { + @Get('list') + list() {} +} +`; + expect(groupPairs(src)).toEqual(['GET /venues/list']); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('pins the argument-less @Controller() — routes mount at the root', () => { + // Legal Nest, and the old group query's mandatory prefix argument made the + // class invisible rather than rooting its methods at '/'. + const src = ` +import { Controller, Get } from '@nestjs/common'; + +@Controller() +export class HealthController { + @Get('health') + health() {} +} +`; + expect(groupPairs(src)).toEqual(['GET /health']); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('pins a pathless @Get() at the controller prefix with no trailing slash', () => { + // The group layer's local `joinPath('venues', '/')` returned '/venues/'; + // the graph stored '/venues'. Both contract-id generation + // (`normalizeHttpPath`) and match-time canonicalization + // (`normalizeContractId`) strip a trailing slash, so this was a latent + // divergence rather than a live mismatch — but it is one fewer way for the + // two layers to describe the same endpoint differently. + const src = venuesController(' @Get()\n index() {}'); + expect(groupPairs(src)).toEqual(['GET /venues']); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('decodes an escaped literal identically in both layers', () => { + // tree-sitter SPLITS a string around each `escape_sequence`. The group + // layer's `unquoteLiteral` (`raw.slice(1, -1)`) left the backslash in place, + // so the ordinary spelling of a Nest regex param came out as a path the app + // never serves. `plainString` decodes it. + const src = venuesController(String.raw` @Get(':id(\\d+)')` + '\n byId() {}'); + expect(groupPairs(src)).toEqual([String.raw`GET /venues/:id(\d+)`]); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('emits NOTHING for an unreadable method path instead of inventing the prefix', () => { + // The precision case. `@Get(ROUTES.SEARCH)` is not readable from this file, + // and the old group scan answered it with a fabricated `GET /venues` — a + // contract that exact-matches any consumer of the controller root and has + // no Route node behind it. The readable sibling proves the controller is + // still SEEN, so this is a dropped route rather than a dropped class. + const src = ` +import { Controller, Get } from '@nestjs/common'; +import { ROUTES } from './routes.js'; + +@Controller('venues') +export class VenuesController { + @Get('list') + list() {} + + @Get(ROUTES.SEARCH) + search() {} +} +`; + expect(groupPairs(src)).toEqual(['GET /venues/list']); + expect(ingestionPairs(src)).toEqual(groupPairs(src)); + }); + + it('reads a JavaScript Nest controller, whose decorators sit under the method', () => { + // tree-sitter-javascript makes a method decorator a CHILD of the + // `method_definition`; tree-sitter-typescript makes it a preceding SIBLING. + // The group scan only ever walked siblings, so every `.js` Nest controller + // emitted zero contracts. + const src = ` +const { Controller, Get } = require('@nestjs/common'); + +@Controller('venues') +class VenuesController { + @Get('search') + search() {} +} +`; + expect(groupPairs(src, JS)).toEqual(['GET /venues/search']); + expect(ingestionPairs(src, JS)).toEqual(groupPairs(src, JS)); + }); + + it('agrees with the indexer on the full (method, URL) set for one mixed fixture', () => { + // The genuine parity assertion (#2265's lesson): one fixture, both layers, + // set equality — plus the literal expectation, so the two cannot agree by + // both returning nothing. + const src = ` +import { Controller, Get, Post, Delete, All, Sse } from '@nestjs/common'; +import { ROUTES } from './routes.js'; + +@Controller({ path: 'venues' }) +export abstract class VenuesController { + @Get() + index() {} + + @Get(':id') + byId() {} + + @Post('/') + create() {} + + @Delete(ROUTES.PURGE) + purge() {} + + @All('proxy') + proxy() {} + + @Sse('events') + events() {} +} + +@Controller() +export class RootController { + @Get('healthz') + healthz() {} +} +`; + const expected = [ + '* /venues/proxy', + 'GET /healthz', + 'GET /venues', + 'GET /venues/:id', + 'GET /venues/events', + 'POST /venues', + ].sort(); + + expect(groupPairs(src)).toEqual(expected); + expect(ingestionPairs(src)).toEqual(expected); + }); + + it('carries the handler name, 1-based line and provider confidence onto the detection', () => { + // The fields the contract extractor resolves a symbol from. `lineNumber` is + // already 1-based at the ingestion layer, so the delegation must NOT add + // one again — `list()` is on line 7 of this source, the same line the + // replaced code reported via `methodNode.startPosition.row + 1`. + const src = venuesController(" @Get('list')\n list() {}"); + expect(TS.plugin.scan(TS.parser.parse(src)).filter((d) => d.framework === 'nest')).toEqual([ + { + role: 'provider', + framework: 'nest', + method: 'GET', + path: '/venues/list', + name: 'list', + line: 7, + confidence: 0.8, + }, + ]); + }); +}); diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index 89b840feb..d6500c702 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -222,22 +222,32 @@ describe('PARSE_CACHE_VERSION', () => { // adds Spring non-HTTP handler side-channel facts (#2417 / #2891), so it is // the next free value after both cache payload changes. // Moved 70 -> 71 for #2980's Java constant-route capture set (moduleConstants - // + routePathOperands). This branch first argued no bump was needed because - // "the ledger already sits at 70, whose capture set post-dates and includes - // this harvest" — it does not: 70 was cut by fe3d7e56b for #2417/#2891, an - // ancestor of this PR's base. Leaving it made PARSE_CACHE_VERSION byte- - // identical across the merge, so every same-package-version warm cache - // replayed pre-feature captures and the feature was inert. 71 is the next - // free value above every claim at this merge — origin/main is 70 and open - // PR #3017 already claims 71, so 71 would have collided. - it('pins SCHEMA_BUMP to 76 so v74 caches cannot retain pre-#3041 identities', () => { - expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(76); + // + routePathOperands). 72 -> 74 added import-proven Convex endpoint metadata, + // skipping 73 because open PR #3046 claims it. + // Version 75 adds #3009's NestJS decorator routes to the same JS/TS + // decoratorRoutes channel, so a warm pre-feature cache cannot replay the empty + // route set that change fixes. This branch originally claimed 71; origin/main + // cascaded past it (71 to #2980, 74 to Convex) while the PR was open, so 71 + // would now be BELOW main and the reuse gate would never fire. 75 is the next + // free value above origin/main and above every in-flight claim (#3046 at 73, + // #1616 at a stale 2) — the rule, re-applied at merge, not at authoring time. + // Moved 75 -> 76 within this same branch for the NestJS array form, then + // 76 -> 77 because 76 turned out not to be free: origin/main reached 76 via + // #3046 while this branch was in review, and package.json is 1.6.9 on both + // sides, so the cache key was the byte-identical `76+1.6.9` on two branches + // with incompatible worker output. #3046 had skipped 75 precisely because + // this branch held it. Two PRs each doing the bookkeeping correctly still + // collided, because each re-checked once and neither re-checked after the + // other moved — which is why the rule is re-applied AT MERGE, not when the + // number is picked. + it('pins SCHEMA_BUMP to 77 so concurrent bumps cannot silently collide (#2766)', () => { + expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(77); // The PREVIOUS version must fail the reuse gate, not merely differ from the // current one — a hardcoded number outside the conflict hunk rebases cleanly // while being wrong, which is exactly how the 37/38 exact clashes landed. // Every nearby historical or in-flight value is rejected, including 69, // which carried the route-table payload before this merge. - for (const taken of [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75]) { + for (const taken of [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76]) { expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).not.toBe(taken); } }); diff --git a/gitnexus/test/unit/nest-decorator-routes.test.ts b/gitnexus/test/unit/nest-decorator-routes.test.ts new file mode 100644 index 000000000..c201af701 --- /dev/null +++ b/gitnexus/test/unit/nest-decorator-routes.test.ts @@ -0,0 +1,738 @@ +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import JavaScript from 'tree-sitter-javascript'; +import { extractNestRoutes } from '../../src/core/ingestion/route-extractors/nest.js'; +import { normalizeExtractedRoutePath } from '../../src/core/ingestion/route-extractors/route-path.js'; + +const tsParser = new Parser(); +tsParser.setLanguage(TypeScript.typescript); + +// The extractor is registered on the JavaScript provider too, and the two +// grammars place a method decorator differently, so both must be exercised. +const jsParser = new Parser(); +jsParser.setLanguage(JavaScript); + +const extract = (source: string) => + extractNestRoutes(tsParser.parse(source), 'src/x.controller.ts'); + +const extractJs = (source: string) => + extractNestRoutes(jsParser.parse(source), 'src/x.controller.js'); + +/** What the routes phase will key the Route node by: verb + joined path. */ +const format = (routes: ReturnType) => + routes.map( + (r) => `${r.httpMethod} ${normalizeExtractedRoutePath(r.routePath, r.prefix ?? null)}`, + ); + +const urls = (source: string) => format(extract(source)); +const jsUrls = (source: string) => format(extractJs(source)); + +describe('NestJS decorator routes', () => { + it('joins the controller prefix with each method path', () => { + expect( + urls(` + @Controller('venues') + export class VenueController { + @Get() + findAll() {} + + @Get('search') + search() {} + + @Post(':id/follow') + follow(@Param('id') id: string) {} + + @Delete(':id') + remove() {} + } + `), + ).toEqual([ + 'GET /venues', + 'GET /venues/search', + 'POST /venues/:id/follow', + 'DELETE /venues/:id', + ]); + }); + + it("emits '/' rather than '' for a pathless @Get, so the handler still resolves", () => { + // call-processor's claim() short-circuits on a falsy routePath, so '' + // would create the Route node but silently lose its handler symbol. + // Both spellings normalize to the same URL. + const [route] = extract(` + @Controller('venues') + export class VenueController { + @Get() + findAll() {} + } + `); + expect(route.routePath).toBe('/'); + expect(normalizeExtractedRoutePath(route.routePath, route.prefix ?? null)).toBe('/venues'); + }); + + it('handles a prefixless @Controller()', () => { + expect( + urls(` + @Controller() + export class AppController { + @Get('health') + health() {} + } + `), + ).toEqual(['GET /health']); + }); + + it('captures the handler method name for symbol resolution', () => { + const routes = extract(` + @Controller('users') + export class UserController { + @Patch(':id') + updateOne() {} + } + `); + + expect(routes).toHaveLength(1); + expect(routes[0]).toMatchObject({ + httpMethod: 'PATCH', + routePath: ':id', + prefix: 'users', + decoratorName: 'Patch', + handlerName: 'updateOne', + filePath: 'src/x.controller.ts', + }); + }); + + it('supports a non-exported controller and all verbs', () => { + expect( + urls(` + @Controller('a') + class A { + @Put('p') p() {} + @Head('h') h() {} + @Options('o') o() {} + @All('any') any() {} + } + `), + ).toEqual(['PUT /a/p', 'HEAD /a/h', 'OPTIONS /a/o', '* /a/any']); + }); + + it('applies each controller its own prefix when a file declares several', () => { + expect( + urls(` + @Controller('one') + export class One { @Get('x') x() {} } + + @Controller('two') + export class Two { @Get('y') y() {} } + `), + ).toEqual(['GET /one/x', 'GET /two/y']); + }); + + it('carries stacked decorators through to the route', () => { + expect( + urls(` + @Controller('secure') + export class SecureController { + @UseGuards(AuthGuard) + @Get('me') + me() {} + } + `), + ).toEqual(['GET /secure/me']); + }); + + it('sees through a comment between the decorators and the method', () => { + // Found on a real controller: four stacked decorators, then a JSDoc block, + // then the method. Breaking the backward walk at the comment made the + // entire route invisible. + expect( + urls(` + @Controller('dev') + export class DevController { + @Post('simulate-expiry') + @HttpCode(HttpStatus.OK) + @ApiOperation({ summary: 'x' }) + /** + * Simulate an expiry. + */ + simulateExpiry() {} + } + `), + ).toEqual(['POST /dev/simulate-expiry']); + }); + + it('sees through a comment between @Controller and the class', () => { + expect( + urls(` + @Controller('docs') + /** The controller. */ + export class DocsController { + @Get('x') x() {} + } + `), + ).toEqual(['GET /docs/x']); + }); + + // ─── Precision guards ────────────────────────────────────────────── + + it('ignores verb-named decorators on a class that is not a @Controller', () => { + // `Get`/`Post` are ordinary identifiers; without the @Controller + // requirement any library reusing those names mints phantom endpoints. + // The unrelated controller is what makes this test reach the per-class + // check: without a `@Controller` anywhere the file short-circuits at the + // parse-free substring gate and the assertion proves nothing. + expect( + urls(` + @Controller('y') + export class RealController { + @Get('real') + real() {} + } + + @Injectable() + export class NotAController { + @Get('looks-like-a-route') + nope() {} + } + `), + ).toEqual(['GET /y/real']); + }); + + it.each([ + { label: 'a constant it cannot read', argument: 'ROUTES.SEARCH' }, + { label: 'an interpolated template', argument: '`${prefix}/search`' }, + { label: 'an array with one element it cannot read', argument: "['a', ROUTES.ADMIN]" }, + ])('drops a route whose path is $label', ({ argument }) => { + // A wrong URL is worse than a missing one — route_map presents this as fact. + // The array row is why one bad element poisons the whole array rather than + // emitting its readable siblings: a half-mapped controller reads as a fully + // mapped one, which is the same lie with less to notice. + expect( + extract(` + @Controller('x') + export class C { + @Get(${argument}) + search() {} + } + `), + ).toEqual([]); + }); + + it('drops every route of a controller whose prefix cannot be read', () => { + expect( + extract(` + @Controller(BASE_PATH) + export class C { + @Get('search') + search() {} + } + `), + ).toEqual([]); + }); + + it('returns nothing for a file with no @Controller at all', () => { + expect(extract(`export function get() { return 1; }`)).toEqual([]); + }); + + // ─── Literal decoding ────────────────────────────────────────────── + + it('decodes an escape in a path instead of deleting it', () => { + // The source below spells the Nest regex param the way a controller does, + // `@Get(':id(\\d+)')`, whose runtime value is `:id(\d+)`. tree-sitter SPLITS + // that literal around the escape_sequence, so keeping only the + // string_fragment children and joining them yielded `:id(d+)` — a URL the + // app never serves, i.e. the wrong-path outcome this module calls worse + // than a missing one. + const [route] = extract(` + @Controller('users') + export class UserController { + @Get(':id(\\\\d+)') + one() {} + } + `); + expect(route.routePath).toBe(':id(\\d+)'); + }); + + it('decodes a unicode escape rather than dropping its payload', () => { + expect( + urls(` + @Controller('v') + export class C { + @Get('/v\\u0069ews') + views() {} + } + `), + ).toEqual(['GET /v/views']); + }); + + it("treats an empty @Controller('') as carrying no prefix", () => { + const [route] = extract(` + @Controller('') + export class C { + @Get('a') a() {} + } + `); + expect(route.prefix).toBeNull(); + expect(normalizeExtractedRoutePath(route.routePath, route.prefix ?? null)).toBe('/a'); + }); + + // ─── Multi-path (array form) ─────────────────────────────────────── + + it('emits one route per path for the array form', () => { + // `@Get(['a','b'])` mounts the handler at BOTH URLs, so both are routes. + // N paths needs no new field to say so: N routes is what an + // ExtractedDecoratorRoute[] already is, the same representation spring.ts + // uses for `@GetMapping({"/a","/b"})`. + const routes = extract(` + @Controller('x') + export class C { + @Get(['a', 'b']) + search() {} + } + `); + + expect(format(routes)).toEqual(['GET /x/a', 'GET /x/b']); + // Everything other than the path is the same route twice — in particular + // the handler, or only one of the two URLs would resolve to a symbol. + expect(routes.map((r) => r.handlerName)).toEqual(['search', 'search']); + }); + + it('reads a single-element array as that one path', () => { + expect( + urls(` + @Controller(['a']) + export class C { + @Get(['b']) b() {} + } + `), + ).toEqual(['GET /a/b']); + }); + + it('emits nothing for an empty array path, and drops only the route it skips', () => { + // `@Get([])` is legal and mounts no URL. It is neither a pathless `@Get()` + // nor an unreadable path: reading it as the first would mint `GET /x`, a + // URL the app does not serve. + expect( + extract(` + @Controller('x') + export class C { + @Get([]) none() {} + } + `), + ).toEqual([]); + + // Whichever the reason a path yields no route — knowably empty, or + // unreadable — it costs exactly its own route and not the controller's + // others, which is what makes a per-decorator skip safe. + expect( + urls(` + @Controller('x') + export class C { + @Get([]) none() {} + @Get(ROUTES.ADMIN) admin() {} + @Get('a') a() {} + } + `), + ).toEqual(['GET /x/a']); + }); + + it('decodes escapes inside array elements too', () => { + // Each element goes through the same `plainString` a scalar path does, so + // the split-around-escape_sequence trap cannot come back on this arm alone. + expect( + extract(` + @Controller('u') + export class C { + @Get([':id(\\\\d+)', '/v\\u0069ews']) + one() {} + } + `).map((r) => r.routePath), + ).toEqual([':id(\\d+)', '/views']); + }); + + // ─── Controller shapes ───────────────────────────────────────────── + + it('extracts routes from an abstract controller base class', () => { + // `export abstract class` parses as abstract_class_declaration, a separate + // node type — and a decorated abstract base sharing CRUD routes with its + // subclasses is ordinary Nest, so missing it drops the whole controller. + expect( + urls(` + @Controller('base') + export abstract class BaseController { + @Get('a') a() {} + } + `), + ).toEqual(['GET /base/a']); + }); + + it('reads the path out of the object form used for URI versioning', () => { + expect( + urls(` + @Controller({ path: 'cats', version: '1' }) + export class CatsController { + @Get('breeds') breeds() {} + } + `), + ).toEqual(['GET /cats/breeds']); + }); + + it('reads a quoted path key, rather than dropping the class over the quotes', () => { + expect( + urls(` + @Controller({ 'path': 'cats' }) + export class CatsController { + @Get('breeds') breeds() {} + } + `), + ).toEqual(['GET /cats/breeds']); + }); + + it.each([ + { label: 'no path key', argument: "{ version: '1' }" }, + { label: 'a computed path', argument: '{ path: BASE_PATH }' }, + { label: 'a computed path key', argument: "{ [PATH_KEY]: 'cats' }" }, + ])('still drops a controller whose object form has $label', ({ argument }) => { + expect( + extract(` + @Controller(${argument}) + export class C { + @Get('a') a() {} + } + `), + ).toEqual([]); + }); + + // A `path` pair proves the mount point only when nothing ELSE in the object + // can replace it. `{ path: 'cats', ...options }` reads as `cats` under a + // first-match scan and mounts wherever `options.path` says at runtime; + // `{ path: 'cats', path: 'dogs' }` mounts at `dogs`. Both are the wrong-URL + // outcome this module calls worse than a missing one, and both are silent — + // a published `/cats` looks exactly like a correct one. So the object is read + // only when every member is a named, non-repeated pair: the whole-entry + // fail-closed shape `routeFromObject` uses in data-route-table.ts. + it.each([ + // `options.path` overrides the pair above it, so the extracted prefix and + // the served prefix disagree with nothing in the file to say so. + { label: 'a trailing spread', argument: "{ path: 'cats', ...options }" }, + // Deterministically SAFE under JS evaluation order — a later `path` pair + // always wins over an earlier spread — and refused anyway. Reading member + // order as proof makes the verdict turn on which side of the spread the + // author happened to type `path`, and how often each spelling occurs in + // real controllers is unmeasured. Meanwhile the two failure directions are + // not symmetric: reading it wrong publishes a URL the app never serves, and + // `route_map`/`api_impact` present that as fact, while refusing omits a + // route that is still findable in source. + { label: 'a leading spread', argument: "{ ...options, path: 'cats' }" }, + { label: 'nothing but a spread', argument: '{ ...options }' }, + // Last write wins at runtime, so a first-match scan names the loser. + { label: 'a repeated path key', argument: "{ path: 'cats', path: 'dogs' }" }, + // Visible only through `propertyName`: compared as raw key text, `path` and + // `'path'` are two different keys and the duplicate check never fires. + { + label: 'a repeated path key in its quoted spelling', + argument: "{ path: 'cats', 'path': 'dogs' }", + }, + // Refusal is on ANY repeated key, not only `path` — a duplicate anywhere is + // evidence the object is not the fixed literal it reads as. + { + label: 'a repeated key other than path', + argument: "{ path: 'a', version: '1', version: '2' }", + }, + // A computed key could evaluate to `path` and take the mount with it. + { label: 'a computed key beside the path', argument: "{ path: 'a', [dynamicKey]: 'b' }" }, + // `shorthand_property_identifier` and `method_definition`; neither is a + // `pair`, so neither offers a key/value this file can read. + { label: 'a shorthand property', argument: '{ path }' }, + { label: 'a method', argument: "{ path: 'a', getFoo() {} }" }, + ])('refuses an object form whose path another member could override: $label', ({ argument }) => { + expect( + extract(` + @Controller(${argument}) + export class C { + @Get('b') b() {} + } + `), + ).toEqual([]); + }); + + it.each([ + // A comment between two pairs is ordinary formatting. It has to be skipped + // BEFORE the not-a-pair test above, or the refusal fires on it and costs + // the controller every route it has. + { + label: 'a comment between its pairs', + argument: "{ path: 'a', /* URI versioning */ version: '1' }", + }, + // Only `path` has to be provable. A non-literal value on an unrelated key + // is benign: `containsExecutingExpression` in data-route-table.ts refuses + // these, but it guards whole-entry declarativeness for a static route + // table — a different invariant from "can this member move the mount". + { + label: 'non-literal values on keys other than path', + argument: "{ path: 'a', host: 'x', scope: Scope.REQUEST, durable: true }", + }, + ])('still reads the prefix out of an object form with $label', ({ argument }) => { + expect( + urls(` + @Controller(${argument}) + export class C { + @Get('b') b() {} + } + `), + ).toEqual(['GET /a/b']); + }); + + it.each([ + { label: 'a single path', argument: "{ path: 'a' }" }, + { label: 'an array of paths', argument: "{ path: ['a', 'b'] }" }, + // The verb gate short-circuits on the object form before the object is + // walked at all, so the class-form refusal never gets a say here. + { label: 'an object the class form would also refuse', argument: "{ path: 'a', ...options }" }, + ])('mints nothing from the object form on a VERB decorator ($label)', ({ argument }) => { + // `@Controller` takes the object form; `@Get` and friends take + // `string | string[]`. Nest mounts nothing here, so emitting a route would + // invent a URL — the failure this module exists to avoid, and the reason + // the class prefix below is deliberately readable: the route is dropped + // because the METHOD path is unreadable, not because the class was. + expect( + extract(` + @Controller('x') + export class C { + @Get(${argument}) a() {} + } + `), + ).toEqual([]); + }); + + it.each([ + { label: 'the bare array form', argument: "['a', 'b']" }, + { label: 'an array inside the object form', argument: "{ path: ['a', 'b'] }" }, + ])('declines a controller whose prefix is multi-path: $label', ({ argument }) => { + // Deliberate parity with spring.ts, which detects an array-form class + // @RequestMapping only to SUPPRESS that class, leaving the prefix x method + // cross-product to #2280. The method path here is perfectly readable, so + // the alternative is not "drop one route" but "publish it under one of the + // two prefixes, or none" — URLs the application does not serve. + expect( + extract(` + @Controller(${argument}) + export class C { + @Get('a') a() {} + } + `), + ).toEqual([]); + }); + + it('extracts from a .js controller, where a decorator is a CHILD of the method', () => { + // tree-sitter-javascript nests a method decorator inside method_definition + // rather than placing it before as a sibling. The same extractor serves the + // JavaScript provider, so reading siblings only meant every .js Nest + // controller emitted nothing while the wiring claimed nestjs coverage. + expect( + jsUrls(` + @Controller('venues') + export class VenueController { + @UseGuards(AuthGuard) + @Get('search') + search() {} + } + `), + ).toEqual(['GET /venues/search']); + }); + + // ─── Verb coverage ───────────────────────────────────────────────── + + it('treats @Sse as the GET endpoint it mounts', () => { + expect( + urls(` + @Controller('events') + export class EventsController { + @Sse('stream') stream() {} + } + `), + ).toEqual(['GET /events/stream']); + }); + + it('emits nothing for a decorator that mounts no endpoint', () => { + // Paired with the @Sse case above: without it, an unsupported route + // decorator and a non-route decorator are the same silent []. + expect( + extract(` + @Controller('events') + export class EventsController { + @UseGuards(AuthGuard) guarded() {} + } + `), + ).toEqual([]); + }); + + it.each(['toString', 'constructor'])( + 'does not mint a route for a decorator named @%s', + (name) => { + // The verb table is looked up by decorator name, so a plain object would + // answer `Object.prototype.toString` here — truthy, and emitted verbatim + // as the route's httpMethod. + expect( + extract(` + @Controller('x') + export class C { + @${name}() f() {} + } + `), + ).toEqual([]); + }, + ); + + it("does not carry a decorated property's decorators onto the next method", () => { + // The decorator run is accumulated in one forward pass over the class body; + // a non-method member must reset it, the way the backward walk used to stop. + expect( + urls(` + @Controller('di') + export class C { + @Inject(SERVICE) + private readonly svc: Service; + + @Get('a') a() {} + } + `), + ).toEqual(['GET /di/a']); + }); + + // A Nest route decorator is a FACTORY: `@Get()` invokes it and returns the + // decorator that registers the route. A bare `@Get` is the factory itself, + // never applied to anything, so Nest registers nothing — emitting a route for + // it publishes a URL the app does not serve. `@Get()` with no argument IS a + // real pathless route and must keep working; the difference is the call, not + // the argument list. + it.each([ + { label: 'a bare verb decorator', member: '@Get a() {}' }, + { + label: 'a bare verb decorator beside a real one', + member: "@Get a() {}\n @Post('b') b() {}", + }, + ])('mints no route for $label', ({ member }) => { + expect( + urls(` + @Controller('x') + export class C { + ${member} + } + `).filter((route) => route.startsWith('GET')), + ).toEqual([]); + }); + + it('drops a class whose @Controller is bare rather than invoked', () => { + // Same rule one level up: an uninvoked `@Controller` registers no + // controller, so its methods are not routes either. + expect( + extract(` + @Controller + export class C { + @Get('a') a() {} + } + `), + ).toEqual([]); + }); + + it('still emits a pathless route for an INVOKED decorator with no argument', () => { + // The control: `@Get()` differs from `@Get` by the call, and only the call. + expect( + urls(` + @Controller('x') + export class C { + @Get() a() {} + } + `), + ).toEqual(['GET /x']); + }); + + // ─── Members Nest never registers as handlers ────────────────────── + + /** Routes a controller emits when `member` is its only member. */ + const memberUrls = (member: string) => + urls(` + @Controller('v') + export class C { + ${member} + } + `); + + /** A non-handler followed by a real one — the shape both arms below need. */ + const STATIC_THEN_INSTANCE = ` + @Controller('v') + export class C { + @Get('s') + static s() {} + + @Get('i') + i() {} + } + `; + + // Nest's `RequestMapping` writes the handler onto the class PROTOTYPE's + // `descriptor.value`, and `RouterExplorer` scans prototype instance methods + // for that metadata. A `static` method lives on the constructor and is never + // scanned; an accessor's descriptor carries `get`/`set` and no `value` to + // register. A verb decorator on any of the three mounts NOTHING, so a route + // minted from one is a URL the app does not serve — the same + // wrong-answer-dressed-as-fact the object-form refusals above exist for. + it.each([ + { label: 'a static method', member: "@Get('s') static s() {}" }, + { label: 'a getter', member: "@Get('s') get s(): string { return ''; }" }, + { label: 'a setter', member: "@Get('s') set s(v: string) {}" }, + ])('mints nothing for $label, which Nest never registers as a handler', ({ member }) => { + expect(memberUrls(member)).toEqual([]); + }); + + // The other half of the modifier check, and the half a mutation can actually + // reach: these modifiers must NOT reject. `async` matters most — it is the + // dominant shape of a real Nest handler, so widening the exclusion set to + // include it would silently delete most routes in most Nest repos while the + // table above stayed green. `override` and an accessibility modifier are the + // other tokens that sit in the same position on a `method_definition`, and a + // method merely NAMED `get`/`set`/`static` is a property_identifier, not a + // modifier — it must survive too. + it.each([ + { label: 'an async method', member: "@Get('s') async s() {}" }, + { label: 'a public method', member: "@Get('s') public s() {}" }, + { label: 'a protected method', member: "@Get('s') protected s() {}" }, + { + label: 'an async method with an accessibility modifier', + member: "@Get('s') public async s() {}", + }, + { label: 'a method named get', member: "@Get('s') get() {}" }, + { label: 'a method named static', member: "@Get('s') static() {}" }, + // The control for the mints-nothing table above: identical source minus the + // modifier, which is what makes those three empty results evidence of the + // check rather than of a fixture that happens to parse to nothing. + { label: 'a plain instance method', member: "@Get('s') s() {}" }, + ])('still emits the route for $label', ({ member }) => { + expect(memberUrls(member)).toEqual(['GET /v/s']); + }); + + it('drops a decorated static method under the JavaScript grammar too', () => { + // tree-sitter-javascript makes a method decorator a CHILD of + // `method_definition`, so `children` reads `decorator | static | + // property_identifier | …` and the modifier is NOT at a fixed index — the + // check has to test every child's type. The instance method beside it is + // the in-fixture control: its route proves this .js arm still measures + // something rather than passing on a fixture that parses to nothing. + expect(jsUrls(STATIC_THEN_INSTANCE)).toEqual(['GET /v/i']); + }); + + it("does not donate a non-handler member's decorator run to the method after it", () => { + // Pins the UNCONDITIONAL `pending.length = 0` at the end of the member + // loop, NOT the modifier check: a non-handler must fall through to that + // clear rather than `continue` past it. Deliberately green before the + // modifier check existed too — there the static member consumed the run + // into its own (wrong) route and then cleared it — so this goes red only + // if a future edit adds the early `continue`. Asserted over the routes + // attributed to `i`, because the whole-output form would instead be + // measuring the modifier check the table above already covers. + const routes = extract(STATIC_THEN_INSTANCE); + + expect(format(routes.filter((route) => route.handlerName === 'i'))).toEqual(['GET /v/i']); + }); +}); From 35591b22d0d104f3fa4fc1b1a55243be9f5fa49c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:42:29 +0100 Subject: [PATCH 02/28] chore(deps)(deps-dev): bump @testing-library/jest-dom in /gitnexus-web (#3050) Bumps [@testing-library/jest-dom](https://github.com/testing-library/jest-dom) from 6.9.1 to 7.0.0. - [Release notes](https://github.com/testing-library/jest-dom/releases) - [Changelog](https://github.com/testing-library/jest-dom/blob/main/CHANGELOG.md) - [Commits](https://github.com/testing-library/jest-dom/compare/v6.9.1...v7.0.0) --- updated-dependencies: - dependency-name: "@testing-library/jest-dom" dependency-version: 7.0.0 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gitnexus-web/package-lock.json | 13 ++++++++----- gitnexus-web/package.json | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index a0162ee9c..d4fb6e57b 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -49,7 +49,7 @@ "devDependencies": { "@babel/types": "^8.0.4", "@playwright/test": "^1.62.0", - "@testing-library/jest-dom": "^6.9.1", + "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/dompurify": "^3.2.0", @@ -2091,9 +2091,9 @@ } }, "node_modules/@testing-library/jest-dom": { - "version": "6.9.1", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", - "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-7.0.0.tgz", + "integrity": "sha512-HKAH9C6mBo5yBG6yRO5i43L2iisencAo5z+o5P/saHUoY+miC5ivXRxHBJcFyB5ypPNxHJdK3BoF/3O4DIptMg==", "dev": true, "license": "MIT", "dependencies": { @@ -2105,9 +2105,12 @@ "redent": "^3.0.0" }, "engines": { - "node": ">=14", + "node": ">=22", "npm": ">=6", "yarn": ">=1" + }, + "peerDependencies": { + "@testing-library/dom": ">=10 <11" } }, "node_modules/@testing-library/jest-dom/node_modules/dom-accessibility-api": { diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 046bab225..7de002950 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -59,7 +59,7 @@ "devDependencies": { "@babel/types": "^8.0.4", "@playwright/test": "^1.62.0", - "@testing-library/jest-dom": "^6.9.1", + "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/dompurify": "^3.2.0", From c34468f0f270284e8a441ca431a2081af8a9bf6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:43:01 +0100 Subject: [PATCH 03/28] chore(deps)(deps): bump react-i18next in /gitnexus-web (#3051) Bumps [react-i18next](https://github.com/i18next/react-i18next) from 17.0.11 to 17.0.12. - [Changelog](https://github.com/i18next/react-i18next/blob/master/CHANGELOG.md) - [Commits](https://github.com/i18next/react-i18next/compare/v17.0.11...v17.0.12) --- updated-dependencies: - dependency-name: react-i18next dependency-version: 17.0.12 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gitnexus-web/package-lock.json | 16 ++++++++-------- gitnexus-web/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index d4fb6e57b..bd4713fe7 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -36,7 +36,7 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.11", + "react-i18next": "^17.0.12", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -246,9 +246,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -7417,12 +7417,12 @@ } }, "node_modules/react-i18next": { - "version": "17.0.11", - "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.11.tgz", - "integrity": "sha512-cDtkXgxjuFTWUH6V+aQn1Ve5vDiUztCNPWW5GtSHDccsgRXO1nE6QFWCEmc1KAutrb3OUv87wFShJL5RhUwPXg==", + "version": "17.0.12", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.12.tgz", + "integrity": "sha512-lFWPEGkxQ6RhusdUkysFBD58VHfSSzvHBzqMgN0SvfVpdQGfwtNkStTqdy08/sJd7s807qqutgx93fRpD0DJ3Q==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.29.2", + "@babel/runtime": "^7.29.7", "html-parse-stringify": "^4.0.1", "use-sync-external-store": "^1.6.0" }, diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 7de002950..dbb2a23c1 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -46,7 +46,7 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.11", + "react-i18next": "^17.0.12", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", From 15274029aa66f0f0146515d6be90c69b01516b69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:43:17 +0100 Subject: [PATCH 04/28] chore(deps)(deps-dev): bump @vercel/node in /gitnexus-web (#3052) Bumps [@vercel/node](https://github.com/vercel/vercel/tree/HEAD/packages/node) from 5.9.9 to 5.10.1. - [Release notes](https://github.com/vercel/vercel/releases) - [Changelog](https://github.com/vercel/vercel/blob/main/packages/node/CHANGELOG.md) - [Commits](https://github.com/vercel/vercel/commits/@vercel/fs-detectors@5.10.1/packages/node) --- updated-dependencies: - dependency-name: "@vercel/node" dependency-version: 5.10.1 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gitnexus-web/package-lock.json | 16 ++++++++-------- gitnexus-web/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index bd4713fe7..d0c3bf24f 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -57,7 +57,7 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.9.9", + "@vercel/node": "^5.10.1", "@vitejs/plugin-react": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", "jsdom": "^29.1.1", @@ -2641,9 +2641,9 @@ } }, "node_modules/@vercel/build-utils": { - "version": "14.0.5", - "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-14.0.5.tgz", - "integrity": "sha512-ChbTraIvChbcFXMwDPLE8MoWpNGSRhJ2cXsE0V3iJQIVYDRgjFoT6JzWfkuc7w/3ojLLr8eMoae7M1v6OXoC5Q==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-14.1.1.tgz", + "integrity": "sha512-kW9CeW0aokEBvX1rSgNyOKg90VyIQOmT0wBl7KXneM3Qs1+x4Puakqp97BdIgttWEtmN96UvdhQVG2bCA5JsPA==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2694,9 +2694,9 @@ } }, "node_modules/@vercel/node": { - "version": "5.9.9", - "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.9.9.tgz", - "integrity": "sha512-jaMocJLa+rP3WpwYrbx2kUpHObjXK/JZOsbtmodDMAtfXbwl7niPNcEbdYYj/fBPSX8yRUXBF3tQsasocbjD5Q==", + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.10.1.tgz", + "integrity": "sha512-muj+t8sZ2XHQDkWcHxkql2rbvr/HhZOqYdZBG7pw8F5RLasL3o0gjHLXJKwHAEHp2I3fd3AgbMud2oz+hzeV0g==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2704,7 +2704,7 @@ "@edge-runtime/primitives": "4.1.0", "@edge-runtime/vm": "3.2.0", "@types/node": "20.11.0", - "@vercel/build-utils": "14.0.5", + "@vercel/build-utils": "14.1.1", "@vercel/error-utils": "2.2.1", "@vercel/nft": "1.10.0", "@vercel/static-config": "3.4.1", diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index dbb2a23c1..a4642f63d 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -67,7 +67,7 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.9.9", + "@vercel/node": "^5.10.1", "@vitejs/plugin-react": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", "jsdom": "^29.1.1", From 414687ad10677dc331bc438954877ade647cd764 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:43:33 +0100 Subject: [PATCH 05/28] chore(deps): bump docker/setup-buildx-action from 4.2.0 to 4.3.0 (#3057) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/bb05f3f5519dd87d3ba754cc423b652a5edd6d2c...37fe631027851001ddb9b187196cc803df7f5f0e) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- .github/workflows/trivy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 73bb80ba5..dac6a7398 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -141,7 +141,7 @@ jobs: uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 1e2b2d1e3..2bc3701f3 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -50,7 +50,7 @@ jobs: persist-credentials: false - name: Setup Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Build image (load locally for scan) uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 From fb49613a4d423a6a8f98ba3f47a6c59a22e87457 Mon Sep 17 00:00:00 2001 From: DuduPhudu <34869259+ReidenXerx@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:29:05 +0300 Subject: [PATCH 06/28] fix(ingestion): ignore emitted Next.js build output, and delete the inert public/build entry (#3018) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ingestion): ignore emitted Next.js build output, and restore the dead public/build entry `DEFAULT_IGNORE_LIST` contained `.next` — the build CACHE — but not `_next`, the emitted OUTPUT, which are different directories. A Capacitor/Cordova shell copies a built Next.js bundle to `/app/src/main/assets/public/_next/static/`, where no path segment hits the list, so the walker indexed the bundle as source. On a real mobile-wrapped Next.js app that was 256 minified chunk files, and every `Route` node the repo produced pointed at a webpack chunk rather than at source. The filename heuristics did not catch them either: they match `.bundle.`, `.chunk.`, `.generated.` and `.d.ts`, while Next.js emits hashed names like `6862-9d1cdcb99f169a06.js`. Separately, `'public/build'` had been sitting in `DEFAULT_IGNORE_LIST` matching nothing at all. That set is tested one path SEGMENT at a time, and is also read by `isHardcodedIgnoredDirectory(name)`, which receives a bare directory name — so a slash-containing member can never compare equal to anything. Rather than delete the entry and lose its intent, multi-segment paths now live in `DEFAULT_IGNORED_PATH_FRAGMENTS` and are matched against the whole path, so Remix / Laravel Mix asset output is ignored as originally intended. A guard test pins the invariant that made the dead entry possible: no member of the name set may contain a slash. Measured against a production Capacitor-wrapped Next.js app (1558 JS/TS files on disk): 256 newly ignored, none of them under `src/`, and zero files that were previously ignored become indexed. Closes #3007 Co-Authored-By: Claude Opus 5 (1M context) * fix(ingestion): drop the inert public/build machinery, discriminate _next by segment, ignore _next on the web upload path Addresses the review findings on #3018. Remove DEFAULT_IGNORED_PATH_FRAGMENTS, hasIgnoredPathFragment and its shouldIgnorePath branch. The mechanism was correct but unreachable: all four of its match forms put a `/` or end-of-string on both sides of `build`, so a fragment match strictly implies `build` is a whole segment, which the per-segment DEFAULT_IGNORE_LIST loop already catches one branch earlier. Measured over 768,420 generated paths: 65,506 fragment matches, 0 of them decisive, 0 implication violations. `'public/build'` really was an inert member of the name set, but its paths were never unignored — bare `'build'` covered them on both sides — so the entry is deleted rather than relocated, which is the other option #3007 offered. The slash-free guard test stays; it is what stops the next slash-bearing entry from dying the same way. Add negative cases pinning that `_next` matches as a whole path segment. The previous suite could not tell a segment rule from a substring rule: replacing the entry with `normalizedPath.includes('_next')` passed all five tests, while eating `src/_nextgen/index.ts`. Rename the public/build test to what it actually pins — that deleting the inert entry changed no behavior — since it is green on both sides by design. Add `_next` to the web upload filter's EXCLUDED_DIRS. That list is the live browser ingestion path (RepoAnalyzer -> filterRepoFiles -> /api/analyze/upload) and had `.next` but not `_next`, so a Capacitor-wrapped Next.js app uploaded its entire minified tree against the server's 20000-file / 250MB caps for files the analyzer then discards. Co-Authored-By: Claude Opus 5 (1M context) * test(ignore-service): make the single-component set guards able to fail The slash-free guard added for #3007 could not fail. It selected entry lines with startsWith("'") and read only the first quoted token per line, so 'public/build' could return as a backtick string, behind an inline block comment, as a second entry on an existing line, or via .add() and every test stayed green. Prettier and eslint miss the backtick and inline-comment forms too, so CI did not catch them either. U1: remove the duplicate '.serverless' entry so the set can be pinned to one exact number. A Set discarded it, so no ignore behaviour changes. U2/U3: replace the line-based parser with a shared single-pass scanner in test/helpers/ignore-set-source.ts, and extend the guard from DEFAULT_IGNORE_LIST to IGNORED_FILES, ROOT_ARTIFACT_DIRECTORIES and IGNORED_EXTENSIONS, which share the same single-component match contract. The scanner tracks string and comment state together because neither can be removed first: the ignore-list comments quote paths and carry an apostrophe, so matching literals before stripping comments yields phantom slash-bearing entries; and a glob string containing a comment-open sequence makes regex comment-stripping swallow the closing bracket. Only a single pass is correct in both directions. Counts are pinned exactly rather than floored — a floor cannot protect a two-member set and hides a partial parse. Shapes a source parser cannot resolve (spread, interpolation, concatenation, later .add) now throw instead of quietly reporting fewer members, and the parsed names are cross-checked against isHardcodedIgnoredDirectory so parser drift fails without exporting the set. Verified by mutation: all six fail-open spellings now turn the suite red; 187 tests pass, tsc clean. * test(ignore-service): pin that _next prunes the directory, not just its files Every measured benefit of ignoring _next comes from never enumerating the bundle tree, and no file list can observe that: anything under _next is rejected whether the walk pruned the directory or descended and rejected each file. childrenIgnored is the only observation that separates them. The existing build-output tests all call shouldIgnorePath, the leaf predicate, so a refactor moving _next to a shouldIgnorePath-only rule would keep them green while silently restoring the full walk. These assertions close that. Also pins that _next matches as a whole segment (_nextgen and my_next are still walked), and that the `!_next/` negation recovers the directory at any depth — the bare form is the one that works, since `!_next/**` alone never gets tested: childrenIgnored prunes the directory before any descendant pattern is reached. Placed in the .gitnexusignore-negation describe block, which owns mkPath and the tmpdir fixture and is registered in scripts/cross-platform-tests.ts. Verified by mutation: disabling only the pruning branch in childrenIgnored leaves the build-output suite at 26/26 green and turns these assertions red. * test(ignore-service): guard the twin build-output ignore lists against drift _next now lives in two lists in two packages — the analyzer's DEFAULT_IGNORE_LIST and the browser upload filter's EXCLUDED_DIRS — with nothing tying them together. This is the seventh twin-list pair in this repo; the header of receiver-twin-list-drift.test.ts records that the previous ones each shipped a bug when one side moved. Containment runs web -> CLI only, and that is the load-bearing direction: the browser filter decides what the server ever sees, and it reads no .gitnexusignore, so a name it drops that the analyzer would have indexed is silent source loss with no recovery. The reverse is not an error — the analyzer prunes far more aggressively than an upload needs to. .gitnexus is the one exemption and has a mechanism: the walker passes dot: false to glob, so it never enumerates dot-directories. Asserted in both directions so re-adding it to the CLI list or dropping it from the web list both fail. Both sides are source-parsed through the shared helper. DEFAULT_IGNORE_LIST is module-private, and no test in this package imports across the package boundary — every cross-package precedent reads source instead. Also corrects the documentation this PR's comments got wrong: the guard test is cited by path rather than as "below", the unreproducible per-repo percentage is gone, the reason _next is deliberately unanchored is recorded next to the entry (no /_next form matches a root-level _next/static/…), and the upload filter now states that it consults no repository ignore rules — so unlike the CLI, a negation cannot recover what it drops. Verified by mutation: a web-only addition and a CLI removal each turn the guard red. 194 targeted tests pass; tsc clean in both packages. * refactor(test): read the ignore sets with the TypeScript parser, not a hand-rolled scanner The guards read ignore-service.ts as source because the sets are module-private. The first pass hand-rolled a character scanner to do it, and the repo already vendors the right tool: ts.createSourceFile, used this way in literal-collectors, query-determinism-guard, cli-index-help and group/sync-partial-extraction. The scanner had two silent gaps a real parser does not have: - It rejected `${` by substring, but template literals were consumed whole, so that branch could never fire and an interpolated member was accepted as a literal — the exact under-report the file refused to allow. - It took the first `[` after the marker, which on a type-annotated declaration (`readonly string[] = ...`) is the annotation's empty pair. It returned [] with no throw, which would make every assertion in a suite vacuously true. This is the hazard receiver-twin-list-drift.test.ts documents having hit. Reading the declaration node removes both, along with the comment-vs-string ordering problem that motivated the scanner: a parser cannot mistake a comment for a string or a glob's `/*` for a comment-open. Also drops the four pinned exact counts. They were a ratchet — these sets are edited by unrelated PRs, each of which would have failed a count assertion about nothing it touched — and with a real parser the partial-parse hazard they existed to catch cannot happen silently: a member that is not a plain string literal throws. Markers collapse to set names, and the duplicated path-resolution boilerplate moves into the helper the two suites already share. Net 187 deletions against 123 insertions. Verified by mutation: backtick, inline comment, same-line, double-quote, duplicate, interpolation, spread and runtime .add() are all caught; a type-annotated declaration now reads correctly instead of returning empty. 194 tests pass, tsc clean. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gergo Magyar --- gitnexus-web/src/lib/upload-filter.test.ts | 16 +++ gitnexus-web/src/lib/upload-filter.ts | 11 ++ gitnexus/src/config/ignore-service.ts | 23 +++- gitnexus/test/helpers/ignore-set-source.ts | 107 +++++++++++++++ .../test/unit/ignore-build-output.test.ts | 127 ++++++++++++++++++ gitnexus/test/unit/ignore-service.test.ts | 32 +++++ .../unit/upload-filter-ignore-drift.test.ts | 90 +++++++++++++ 7 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 gitnexus/test/helpers/ignore-set-source.ts create mode 100644 gitnexus/test/unit/ignore-build-output.test.ts create mode 100644 gitnexus/test/unit/upload-filter-ignore-drift.test.ts diff --git a/gitnexus-web/src/lib/upload-filter.test.ts b/gitnexus-web/src/lib/upload-filter.test.ts index e97b9ec2c..1943720ba 100644 --- a/gitnexus-web/src/lib/upload-filter.test.ts +++ b/gitnexus-web/src/lib/upload-filter.test.ts @@ -31,6 +31,22 @@ describe('filterRepoFiles', () => { expect(r.droppedCount).toBe(4); }); + it('excludes emitted _next output, including the Capacitor/Cordova copy', () => { + // `.next` was listed but `_next` was not, so a mobile-wrapped Next.js app + // uploaded its whole minified bundle against the server's caps for files + // the analyzer then discards anyway (#3007). + const input = [ + f('repo/android/app/src/main/assets/public/_next/static/chunks/main.js'), + f('repo/ios/App/App/public/_next/static/chunks/framework.js'), + f('repo/_next/static/chunks/x.js'), + f('repo/src/index.ts'), + f('repo/src/_nextgen/index.ts'), + ]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/src/_nextgen/index.ts']); + expect(r.droppedCount).toBe(3); + }); + it('drops files over the per-file size cap', () => { const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)]; const r = filterRepoFiles(input); diff --git a/gitnexus-web/src/lib/upload-filter.ts b/gitnexus-web/src/lib/upload-filter.ts index a24520f74..9b3fb30db 100644 --- a/gitnexus-web/src/lib/upload-filter.ts +++ b/gitnexus-web/src/lib/upload-filter.ts @@ -22,6 +22,17 @@ export const EXCLUDED_DIRS = new Set([ 'build', 'out', '.next', + // `.next` is the build CACHE, `_next` the EMITTED output — different + // directories. A Capacitor/Cordova shell leaves the emitted bundle at + // `/app/src/main/assets/public/_next/`, so without this the whole + // minified tree is uploaded against the server's file/byte caps only to be + // discarded by the analyzer's own ignore list (#3007). + // + // This pre-filter reads no repository ignore rules, so unlike the CLI walker + // a `.gitnexusignore` negation cannot recover anything dropped here. Names + // added below must therefore stay a subset of the analyzer's own list; see + // `gitnexus/test/unit/upload-filter-ignore-drift.test.ts`. + '_next', '.nuxt', '.cache', 'coverage', diff --git a/gitnexus/src/config/ignore-service.ts b/gitnexus/src/config/ignore-service.ts index 645754427..f1b9e1150 100644 --- a/gitnexus/src/config/ignore-service.ts +++ b/gitnexus/src/config/ignore-service.ts @@ -58,13 +58,33 @@ const DEFAULT_IGNORE_LIST = new Set([ 'obj', 'target', // Java/Rust '.next', + // `.next` is Next.js's build CACHE; `_next` is the EMITTED output, and the two + // are different directories. A Capacitor/Cordova shell copies the emitted + // bundle to `/app/src/main/assets/public/_next/static/…`, where none + // of the path segments hit this list — so a mobile-wrapped Next.js app had its + // shipped bundle indexed as source, and every Route node it produced pointed at + // a webpack chunk rather than code anyone wrote (#3007). + // + // The name is deliberately unanchored. No `/_next` form matches a + // root-level `_next/static/…`, which is the shape the reported repo has, so + // anchoring it would miss the case it was added for. The accepted cost is a + // hand-written directory literally named `_next`; recover one with a bare + // `!_next/` line in `.gitnexusignore`. + '_next', '.nuxt', '.output', '.vercel', '.netlify', '.serverless', '_build', - 'public/build', + // `'public/build'` used to sit here. This set is tested one path SEGMENT at a + // time, and `isHardcodedIgnoredDirectory(name)` takes a bare directory name, + // so a slash-containing member could never match either — it was inert. Its + // paths were never unignored though: bare `'build'` above already prunes + // `public/build/**`, so removing the entry changes no behavior (#3007). + // `test/unit/ignore-build-output.test.ts` keeps the next slash-bearing entry + // in this set — or in IGNORED_FILES, ROOT_ARTIFACT_DIRECTORIES or + // IGNORED_EXTENSIONS — from dying the same way. '.parcel-cache', '.turbo', '.svelte-kit', @@ -95,7 +115,6 @@ const DEFAULT_IGNORE_LIST = new Set([ // remains covered by .gitignore/.gitnexusignore and the unambiguous names. 'monaco-workers', // Monaco editor web-worker bundles generated for browser runtime '.terraform', - '.serverless', // Documentation (optional - might want to keep) // 'docs', diff --git a/gitnexus/test/helpers/ignore-set-source.ts b/gitnexus/test/helpers/ignore-set-source.ts new file mode 100644 index 000000000..04db55dd6 --- /dev/null +++ b/gitnexus/test/helpers/ignore-set-source.ts @@ -0,0 +1,107 @@ +/** + * Reads the bare-name sets in `src/config/ignore-service.ts` out of source. + * + * Those sets are module-private, and exporting them purely to be testable would + * widen a production surface to satisfy a test — the call + * `receiver-twin-list-drift.test.ts` documents. So the guards read the source + * instead, through the TypeScript parser the repo already vendors and already + * uses this way (`literal-collectors.ts`, `query-determinism-guard.test.ts`, + * `cli-index-help.test.ts`). + * + * Using the real parser is what makes the guards trustworthy. A text scanner has + * to decide whether a delimiter opens a comment or sits inside a string, and it + * gets that wrong in both directions here: the ignore-list comments quote paths + * and carry an apostrophe (`Next.js's`), while a glob string such as `'** / *'` + * contains a comment-open sequence. It also has to guess which bracket belongs + * to the declaration rather than to a type annotation. Each of those is a way to + * silently read fewer members — and a guard that quietly stops seeing members is + * the exact defect these guards exist to catch. + * + * `setEntries` therefore refuses anything that is not a plain list of string + * literals, rather than skipping the members it cannot resolve. + */ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import ts from 'typescript'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); + +/** The analyzer's ignore rules — the sets every guard in this family reads. */ +export const IGNORE_SERVICE_PATH = path.join( + REPO_ROOT, + 'gitnexus', + 'src', + 'config', + 'ignore-service.ts', +); + +/** The browser upload pre-filter, whose excluded-directory set must not drift from the above. */ +export const UPLOAD_FILTER_PATH = path.join( + REPO_ROOT, + 'gitnexus-web', + 'src', + 'lib', + 'upload-filter.ts', +); + +export const readSource = (file: string): string => readFileSync(file, 'utf8'); + +/** + * The string literals `setName` is constructed from, in declaration order. + * + * Throws — never returns a short list — when the declaration is missing or holds + * anything other than plain string literals (a spread, an interpolation, a + * concatenation, a computed value). + */ +export const setEntries = (source: string, setName: string): string[] => { + const sourceFile = ts.createSourceFile( + 'ignore-set-source.ts', + source, + ts.ScriptTarget.Latest, + true, + ); + + let elements: ts.NodeArray | undefined; + const visit = (node: ts.Node): void => { + if ( + elements === undefined && + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.name.text === setName && + node.initializer !== undefined && + ts.isNewExpression(node.initializer) && + node.initializer.arguments?.length === 1 && + ts.isArrayLiteralExpression(node.initializer.arguments[0]) + ) { + elements = node.initializer.arguments[0].elements; + return; + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + + if (elements === undefined) { + throw new Error(`${setName} is not declared as \`new Set([...])\` — update this test`); + } + + const unresolvable = elements.filter((element) => !ts.isStringLiteral(element)); + if (unresolvable.length > 0) { + throw new Error( + `${setName} holds ${unresolvable.length} member(s) that are not plain string literals ` + + `(first: \`${unresolvable[0].getText(sourceFile)}\`). A source-reading guard cannot resolve ` + + `those, so switch this set to a runtime assertion rather than letting the guard see fewer members.`, + ); + } + + return elements.map((element) => (element as ts.StringLiteral).text); +}; + +/** + * True when `setName` is mutated by `.add(...)` anywhere in `source`. + * + * `setEntries` reads the declaration only, so a member appended afterwards would + * be invisible to it. The guards assert this is false rather than under-reporting. + */ +export const hasRuntimeAdd = (source: string, setName: string): boolean => + new RegExp(`\\b${setName}\\s*\\.\\s*add\\s*\\(`).test(source); diff --git a/gitnexus/test/unit/ignore-build-output.test.ts b/gitnexus/test/unit/ignore-build-output.test.ts new file mode 100644 index 000000000..7e8a87a29 --- /dev/null +++ b/gitnexus/test/unit/ignore-build-output.test.ts @@ -0,0 +1,127 @@ +import { describe, it, expect } from 'vitest'; +import { isHardcodedIgnoredDirectory, shouldIgnorePath } from '../../src/config/ignore-service.js'; +import { + IGNORE_SERVICE_PATH, + hasRuntimeAdd, + readSource, + setEntries, +} from '../helpers/ignore-set-source.js'; + +/** + * Emitted build output must not be indexed as source (#3007). + * + * `.next` (the build cache) was listed but `_next` (the emitted output) was + * not, so a Capacitor/Cordova shell that copies a Next.js bundle into + * `/app/src/main/assets/public/_next/static/` had its shipped bundle + * indexed as source — and every `Route` node the repo produced pointed at a + * webpack bundle instead of code anyone wrote. + */ + +describe('build-output ignores', () => { + it('ignores emitted _next output, including the Capacitor/Cordova copy', () => { + expect(shouldIgnorePath('_next/static/chunks/main.js')).toBe(true); + expect(shouldIgnorePath('.next/server/app/page.js')).toBe(true); + expect( + shouldIgnorePath( + 'android/app/src/main/assets/public/_next/static/chunks/6862-9d1cdcb99f169a06.js', + ), + ).toBe(true); + expect(shouldIgnorePath('ios/App/App/public/_next/static/chunks/framework-abc123.js')).toBe( + true, + ); + }); + + it('does not ignore ordinary source that merely mentions next', () => { + expect(shouldIgnorePath('src/next-steps.ts')).toBe(false); + expect(shouldIgnorePath('src/nextConfig/index.ts')).toBe(false); + expect(shouldIgnorePath('packages/next-auth/src/index.ts')).toBe(false); + }); + + it('matches _next as a whole segment, not as a substring', () => { + // Without these, `normalizedPath.includes('_next')` would satisfy every + // other assertion in this file — the suite could not tell a segment rule + // from a substring rule, and a substring rule would eat real source. + expect(shouldIgnorePath('src/_nextgen/index.ts')).toBe(false); + expect(shouldIgnorePath('packages/my_next/src/index.ts')).toBe(false); + expect(shouldIgnorePath('src/prefix_next.ts')).toBe(false); + }); + + it('keeps public/build ignored after the inert name-set entry was removed', () => { + // NOT a regression test for new behavior — it pins that DELETING the inert + // `'public/build'` entry changed nothing, because bare `'build'` matches + // these as an ordinary segment and always did. Green on both sides of the + // change by design; that is the point. + expect(shouldIgnorePath('public/build/entry.client.js')).toBe(true); + expect(shouldIgnorePath('apps/web/public/build/manifest.js')).toBe(true); + }); + + it('does not ignore public/ or build/-adjacent source outside that pair', () => { + expect(shouldIgnorePath('public/favicon-loader.ts')).toBe(false); + expect(shouldIgnorePath('src/public/api.ts')).toBe(false); + }); + + describe('single-component set guards', () => { + const source = readSource(IGNORE_SERVICE_PATH); + + // Every one of these sets is compared against a single path component, so a + // member containing `/` is dead on arrival — the defect that left + // `'public/build'` inert. + const SET_NAMES = [ + 'DEFAULT_IGNORE_LIST', + 'IGNORED_FILES', + 'ROOT_ARTIFACT_DIRECTORIES', + 'IGNORED_EXTENSIONS', + ] as const; + + it.each(SET_NAMES)('%s holds no slash-bearing member', (setName) => { + expect(setEntries(source, setName).filter((entry) => entry.includes('/'))).toEqual([]); + }); + + it.each(SET_NAMES)('%s holds no duplicate member', (setName) => { + const entries = setEntries(source, setName); + expect(new Set(entries).size).toBe(entries.length); + }); + + it.each(SET_NAMES)('%s is never mutated by .add() after construction', (setName) => { + expect(hasRuntimeAdd(source, setName)).toBe(false); + }); + + it('every IGNORED_EXTENSIONS member starts with a dot', () => { + const entries = setEntries(source, 'IGNORED_EXTENSIONS'); + expect(entries.filter((entry) => !entry.startsWith('.'))).toEqual([]); + }); + + it('reads entries the declaration holds, not text the comments quote', () => { + // The comments in DEFAULT_IGNORE_LIST quote paths and carry an apostrophe + // (`Next.js's`), so a text scanner reads phantom entries out of them — + // several slash-bearing, which would fail the assertion above on correct + // source. Parsing the declaration cannot see comments at all. + const entries = setEntries(source, 'DEFAULT_IGNORE_LIST'); + expect(entries).toContain('_next'); + expect(entries).not.toContain('public/build'); + expect(entries).not.toContain('env/'); + }); + + it('agrees with the runtime set it claims to describe', () => { + // Catches drift between what the guard reads and what the module does, + // without exporting the set. + const entries = setEntries(source, 'DEFAULT_IGNORE_LIST'); + expect(entries.filter((entry) => !isHardcodedIgnoredDirectory(entry))).toEqual([]); + }); + + it('fails loudly when a set is no longer declared as a Set of literals', () => { + expect(() => setEntries(source, 'NOT_A_REAL_SET')).toThrow(/update this test/); + }); + + it('refuses a declaration whose members it cannot resolve', () => { + // A spread, an interpolation, or a concatenation resolves at runtime, not + // in source. Reading the resolvable members and passing is the failure mode + // these guards exist to prevent, so the reader refuses instead. + const poisoned = source.replace( + 'const ROOT_ARTIFACT_DIRECTORIES = new Set([', + 'const ROOT_ARTIFACT_DIRECTORIES = new Set([...OTHER_NAMES,', + ); + expect(() => setEntries(poisoned, 'ROOT_ARTIFACT_DIRECTORIES')).toThrow(/not plain string/); + }); + }); +}); diff --git a/gitnexus/test/unit/ignore-service.test.ts b/gitnexus/test/unit/ignore-service.test.ts index 4fb5536ef..60d6e8f77 100644 --- a/gitnexus/test/unit/ignore-service.test.ts +++ b/gitnexus/test/unit/ignore-service.test.ts @@ -334,6 +334,38 @@ describe('.gitnexusignore negation overrides hardcoded DEFAULT_IGNORE_LIST (#771 expect(filter.childrenIgnored(mkPath('Env'))).toBe(false); }); + // `_next` has to prune the DIRECTORY, not merely reject each file underneath. + // Every measured benefit of ignoring it comes from never enumerating the + // bundle tree, and no file list can show the difference — anything under + // `_next` is rejected either way. `childrenIgnored` is the only observation + // that distinguishes them, so a refactor that moved `_next` to a + // `shouldIgnorePath`-only rule would keep the build-output suite green while + // silently restoring the full walk. + it('prunes emitted _next output as a directory, at any depth', async () => { + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(mkPath('_next'))).toBe(true); + expect(filter.childrenIgnored(mkPath('android/app/src/main/assets/public/_next'))).toBe(true); + }); + + it('matches _next as a whole segment, so _nextgen source is still walked', async () => { + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(mkPath('src/_nextgen'))).toBe(false); + expect(filter.childrenIgnored(mkPath('packages/my_next'))).toBe(false); + }); + + it('`!_next/` negation unlocks the emitted output directory at any depth', async () => { + // The bare form is the one that works. `!_next/**` alone is a silent no-op: + // `childrenIgnored` prunes the directory before any descendant pattern is + // ever tested, so no file underneath reaches `ignored`. + await fs.writeFile(path.join(tmpDir, '.gitnexusignore'), '!_next/\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(mkPath('_next'))).toBe(false); + expect(filter.childrenIgnored(mkPath('android/app/src/main/assets/public/_next'))).toBe(false); + }); + it('prunes a nested env directory only when pyvenv.cfg identifies a virtual environment', async () => { await fs.mkdir(path.join(tmpDir, 'backend', 'env'), { recursive: true }); await fs.writeFile(path.join(tmpDir, 'backend', 'env', 'pyvenv.cfg'), 'home = python\n'); diff --git a/gitnexus/test/unit/upload-filter-ignore-drift.test.ts b/gitnexus/test/unit/upload-filter-ignore-drift.test.ts new file mode 100644 index 000000000..0eea8b06e --- /dev/null +++ b/gitnexus/test/unit/upload-filter-ignore-drift.test.ts @@ -0,0 +1,90 @@ +/** + * The drift guard for the twin build-output ignore lists (#3007 follow-up). + * + * TWO lists spell "do not index this directory", in two packages: + * + * - `DEFAULT_IGNORE_LIST` — gitnexus `src/config/ignore-service.ts`. The + * analyzer's own list, consulted for every path during the repository walk. + * - `EXCLUDED_DIRS` — gitnexus-web `src/lib/upload-filter.ts`. A client-side + * pre-filter that decides what a browser folder upload sends at all. + * + * `_next` was added to both in the same PR, one commit apart. Before it, neither + * carried the name — so #3007 was a shared omission rather than drift between + * them. What this test guards is the divergence that becomes possible now that + * the same name lives in two places with nothing tying them together. + * + * The containment runs web -> CLI only, and that direction is the load-bearing + * one: the browser filter decides what the server ever sees, so a name it drops + * that the analyzer would have indexed is silent source loss with no recovery — + * this pre-filter reads no `.gitnexusignore`, so a negation cannot bring the + * files back. The reverse direction is not an error: roughly sixty CLI-only + * names exist because the analyzer prunes far more aggressively than an upload + * needs to, and the walker's own `dot: false` already hides dot-directories from + * it. That asymmetry is why equality is not the assertion. + * + * `.gitnexus` is the one deliberate exception, and it has a mechanism rather + * than being an oversight: the CLI walker passes `dot: false` to glob + * (`src/core/ingestion/filesystem-walker.ts`), so it never enumerates + * dot-directories and does not need the name in its list. The browser filter has + * no equivalent and must name it. That exemption is asserted explicitly in both + * directions, so re-adding `.gitnexus` to the CLI list or dropping it from the + * web list both fail loudly. + * + * Both sides are read from source rather than imported. `DEFAULT_IGNORE_LIST` is + * module-private. `EXCLUDED_DIRS` is exported, but `upload-filter.ts` types its + * inputs with the DOM `File` interface, which does not resolve under this + * package's `lib: ["ES2022"] / types: ["node"]` — so importing it here would not + * typecheck. + */ +import { describe, it, expect } from 'vitest'; +import { + IGNORE_SERVICE_PATH, + UPLOAD_FILTER_PATH, + readSource, + setEntries, +} from '../helpers/ignore-set-source.js'; + +const cliNames = () => setEntries(readSource(IGNORE_SERVICE_PATH), 'DEFAULT_IGNORE_LIST'); +const webNames = () => setEntries(readSource(UPLOAD_FILTER_PATH), 'EXCLUDED_DIRS'); + +/** + * Names the browser filter may drop that the analyzer does not list. + * + * Only `.gitnexus`, and only because the CLI walker's `dot: false` makes the + * entry unnecessary there. A name added here must cite a comparable structural + * reason in the walker — this list is not a place to park a failing assertion. + */ +const WEB_ONLY_ALLOWLIST = ['.gitnexus']; + +describe('build-output ignore lists stay in agreement across packages', () => { + it('parses both lists — neither is silently empty', () => { + // Guards the guard: a parser that read nothing would make every containment + // assertion below vacuously true. + expect(cliNames().length).toBeGreaterThan(20); + expect(webNames().length).toBeGreaterThan(5); + }); + + it('every name the browser filter drops is one the analyzer also ignores', () => { + const cli = new Set(cliNames()); + const unmatched = webNames().filter( + (name) => !cli.has(name) && !WEB_ONLY_ALLOWLIST.includes(name), + ); + expect(unmatched).toEqual([]); + }); + + it('carries the documented web-only exemption, and only that one', () => { + // Asserted separately from the containment above so the intent survives if + // that assertion is ever relaxed. + expect(webNames()).toContain('.gitnexus'); + expect(cliNames()).not.toContain('.gitnexus'); + }); + + it('shares the build-output names the reported bug was about', () => { + const cli = new Set(cliNames()); + const web = new Set(webNames()); + for (const name of ['_next', '.next', 'dist', 'build', 'out']) { + expect(web.has(name), `${name} missing from the browser upload filter`).toBe(true); + expect(cli.has(name), `${name} missing from the analyzer ignore list`).toBe(true); + } + }); +}); From 3113803c8bcc6895fd0051d9fc3666f0dcbac4d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:29:24 +0100 Subject: [PATCH 07/28] chore(deps)(deps-dev): bump @testing-library/user-event in /gitnexus-web (#3054) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@testing-library/user-event](https://github.com/testing-library/user-event) from 14.6.1 to 14.6.6. - [Release notes](https://github.com/testing-library/user-event/releases) - [Changelog](https://github.com/testing-library/user-event/blob/main/CHANGELOG.md) - [Commits](https://github.com/testing-library/user-event/compare/v14.6.1...v14.6.6) --- updated-dependencies: - dependency-name: "@testing-library/user-event" dependency-version: 14.6.6 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar --- gitnexus-web/package-lock.json | 8 ++++---- gitnexus-web/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index d0c3bf24f..14594e8fd 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -51,7 +51,7 @@ "@playwright/test": "^1.62.0", "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.1", + "@testing-library/user-event": "^14.6.6", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", "@types/react": "^19.2.14", @@ -2149,9 +2149,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.1", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", - "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "version": "14.6.6", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", + "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", "dev": true, "license": "MIT", "engines": { diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index a4642f63d..277d85945 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -61,7 +61,7 @@ "@playwright/test": "^1.62.0", "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.1", + "@testing-library/user-event": "^14.6.6", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", "@types/react": "^19.2.14", From f1b8faec93a73a8e9fdc9fc2b23de000df48472e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:29:37 +0100 Subject: [PATCH 08/28] chore(deps)(deps): bump uuid from 14.0.1 to 14.0.2 in /gitnexus-web (#3055) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [uuid](https://github.com/uuidjs/uuid) from 14.0.1 to 14.0.2. - [Release notes](https://github.com/uuidjs/uuid/releases) - [Changelog](https://github.com/uuidjs/uuid/blob/main/CHANGELOG.md) - [Commits](https://github.com/uuidjs/uuid/compare/v14.0.1...v14.0.2) --- updated-dependencies: - dependency-name: uuid dependency-version: 14.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar --- gitnexus-web/package-lock.json | 8 ++++---- gitnexus-web/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 14594e8fd..82f082d15 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -43,7 +43,7 @@ "remark-gfm": "^4.0.1", "sigma": "^3.0.3", "tailwindcss": "^4.3.3", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "zod": "^4.4.3" }, "devDependencies": { @@ -8316,9 +8316,9 @@ } }, "node_modules/uuid": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", - "integrity": "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 277d85945..440bf4a54 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -53,7 +53,7 @@ "remark-gfm": "^4.0.1", "sigma": "^3.0.3", "tailwindcss": "^4.3.3", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "zod": "^4.4.3" }, "devDependencies": { From 31c9d9223eb5c1f3491c9f8fa94d9fd60f11fa86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:29:52 +0100 Subject: [PATCH 09/28] chore(deps): bump the codeql-action group with 3 updates (#3056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the codeql-action group with 3 updates: [github/codeql-action/init](https://github.com/github/codeql-action), [github/codeql-action/analyze](https://github.com/github/codeql-action) and [github/codeql-action/upload-sarif](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.37.6 to 4.37.7 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/5595ccaf912efad79be6eef63a5619ff05969be3...ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd) Updates `github/codeql-action/analyze` from 4.37.6 to 4.37.7 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/5595ccaf912efad79be6eef63a5619ff05969be3...ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd) Updates `github/codeql-action/upload-sarif` from 4.37.6 to 4.37.7 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/5595ccaf912efad79be6eef63a5619ff05969be3...ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd) --- updated-dependencies: - dependency-name: github/codeql-action/init dependency-version: 4.37.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/analyze dependency-version: 4.37.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.37.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- .github/workflows/trivy.yml | 2 +- .github/workflows/workflow-lint.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 16d3e15cc..41440d557 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -48,7 +48,7 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: languages: ${{ matrix.language }} queries: security-and-quality @@ -73,6 +73,6 @@ jobs: - '**/test/**/fixtures/**' - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 04d722160..f6e54108d 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -53,6 +53,6 @@ jobs: retention-days: 5 - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: results.sarif diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 2bc3701f3..699289656 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -76,7 +76,7 @@ jobs: exit-code: '0' - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: trivy-${{ matrix.image.name }}.sarif category: trivy-${{ matrix.image.name }} diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index f771406a0..9c3b45ab6 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -76,7 +76,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: zizmor.sarif category: zizmor From 0f793558adeb193461c6f6e21779db0ea5b694e7 Mon Sep 17 00:00:00 2001 From: DuduPhudu <34869259+ReidenXerx@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:27:32 +0300 Subject: [PATCH 10/28] fix(group)!: stop group sync claiming matching it never did (#3020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(group)!: remove the matching cascade that was advertised but never built `gitnexus group create` wrote `matching.bm25_threshold` and `matching.embedding_threshold` into every generated group.yaml, and no matcher ever read either one. That was not the whole of it — an entire feature surface described a BM25/embedding cascade that does not exist: - `matching.bm25_threshold` / `matching.embedding_threshold` — parsed, persisted, unread - `detect.embedding_fallback` — defaulted and templated, unread - `MatchType` declared `'bm25' | 'embedding'`; both variants unreachable - `SyncOptions.skipEmbeddings` — declared in sync.ts and never read - `gitnexus group sync --skip-embeddings` — accepted, threaded through GroupService, ignored - CLI help in en and zh-CN promised "Exact + BM25 only (no embedding fallback)" - the MCP `group_sync` schema exposed `skipEmbeddings`, described as "Exact + BM25 only (Demo PR: same as default exact path)" `sync.ts` imports exactly `buildProviderIndex`, `runExactMatch` and `runWildcardMatch`, and the printed cascade has one stage. An operator whose links do not match reaches for those thresholds first, and turning either knob changes nothing — config that silently does nothing is how people conclude a feature is broken. Evidence that the cascade should be deleted rather than implemented, from a real backend/frontend pair: of 165 consumer contracts, 149 link exactly and 16 do not. Nine of the sixteen are third-party APIs (Google OAuth, Apple public keys, PostHog, image annotation) with no in-group provider by construction — similarity matching cannot recover them, it can only invent false links. Two are verb mismatches: the frontend calls `POST /links` and `GET /links/check-exists` while the backend declares `GET /links` and eleven other `/links/*` routes but neither of those, so a fuzzy path match would link a POST consumer to a GET provider. The rest are path-extraction artifacts. Roughly none of the sixteen would be correctly recovered, and several would be actively mis-linked. BREAKING CHANGE: `gitnexus group sync --skip-embeddings` and the MCP `group_sync` `skipEmbeddings` parameter are removed. Both were accepted and ignored, so no behavior changes — but a script passing the flag now fails with `unknown option` instead of being silently misled. Existing group.yaml files keep loading: the removed keys are simply no longer part of the schema, and a regression test pins that a legacy config carrying all three still parses. Closes #3006 Co-Authored-By: Claude Opus 5 (1M context) * fix(group)!: honour --exact-only, drop inert --allow-stale, report every matching stage Addresses the review findings on #3020, all of which are the same defect the PR itself is about: group-sync surface that describes behaviour the pipeline does not have. `exactOnly` was inert in exactly the way `skipEmbeddings` was — declared on `SyncOptions`, threaded through the CLI and the MCP tool, and read by nothing — and strictly worse, because the stage it promised to suppress DOES run and DOES write `matchType:'wildcard'` links into contracts.json and the bridge, which `group impact` and cross-repo `trace` then traverse. It is now honoured rather than deleted: unlike the never-built BM25/embedding stages, the stage it names exists, so the flag describes a real choice. The substituted result is `{ matched: [], remaining: unmatched }`, not an empty result — `wildcard.remaining` IS `SyncResult.unmatched`, so skipping the stage has to leave its input unmatched rather than dropping it from the count an operator reads. `allowStale` had no such stage to gate: `syncGroup` emits no stale warning at any point (the `checkStaleness` call lives in `groupStatus`, a different path), so it is removed under the same rationale as `skipEmbeddings`. `group sync` now prints every matching stage instead of `exact` alone. The old block printed a `Matching cascade:` header and counted only exact links while the next line reported `result.crossLinks.length` — which also includes `manifest` and `wildcard` — so for any group with those the two numbers disagreed with nothing on screen explaining why. Counting is an exhaustive `Record`, so a new MatchType fails the build here instead of going silently uncounted, and reads through `?? 0` so a legacy registry carrying a removed matchType prints an honest count rather than `NaN`. Also: the MCP `group_sync` description no longer omits the wildcard stage that always runs, `exactOnly`'s description no longer refers to a "cascade", and bench/cross-repo-trace/verify.mjs no longer generates the removed threshold keys into a fresh group.yaml. Tests: `sync-exact-only.test.ts` pins both directions of the gate (mutation-verified: removing the gate, or returning `remaining: []`, both go red). `group-tools.test.ts` pins that the MCP schema dropped `skipEmbeddings` and kept `exactOnly`. `group-cli.test.ts` pins that both removed flags are rejected, with `--exact-only` as an accepted-flag control. `config-parser.test.ts` now pins that legacy keys are PRESERVED (measured, not assumed) rather than only that parsing does not throw. The type narrowing's fallout in test files is cleared: `tsc -p tsconfig.test.json` is 987 errors at head against 987 measured on origin/main, with the two error sets identical — zero net, zero new, zero masked. Verification: `tsc --noEmit` exit 0; prettier clean; eslint 0 errors (2 warnings, both pre-existing on base); 69 test files / 1169 tests green across test/unit/group, test/integration/group, tools, cli-i18n and cli-index-help. Co-Authored-By: Claude Opus 5 (1M context) * fix(group): reject malformed and retired group_sync parameters (U1) `GroupService.groupSync` read `exactOnly` off an untyped MCP payload with `Boolean(params.exactOnly)`. While the flag was inert that coercion was harmless; now that it gates the wildcard matching stage, the string "false" -- a routine shape for an LLM caller emitting JSON -- is truthy, so a caller that asked to KEEP wildcard matching got it suppressed and a registry with fewer cross-links persisted to disk. The opposite of the request, written down. Validate instead of coercing, at the service boundary: the MCP SDK does not enforce a tool's advertised inputSchema and `callTool` is reachable directly, so this method is the real gate. The validator mirrors `validateImpactMode`'s `{ value } | { error }` shape -- the established idiom for this boundary, and the one groupSync's other guards already return through. Also refuse `skipEmbeddings` and `allowStale` by name. The CLI rejects them outright because commander errors on an unknown option; the MCP path accepted and silently dropped them, so an agent working from a cached tool schema was never told. Removing them took away discoverability, not acceptance. Both guards run before the group is read off disk, so a rejected call performs no work. Every test asserts the sync did NOT run -- an error string alone cannot distinguish "refused" from "refused but synced anyway". The tool description gains the validation note AFTER the registryOutcome paragraph: `tools.test.ts` slices that description by ordinal position of the 'preserved' / 'superseded' / 'no-prior-registry' literals, so appending past all three leaves those slices intact (verified, 44/44). tsc clean; 1039/1039 group unit tests pass. * fix(group): record the matching stages a sync was told to skip (U2) An `--exact-only` sync wrote a contracts.json and bridge with fewer cross-links and nothing recorded that the wildcard stage had been suppressed by request. `group_impact` and cross-repo `trace` read that registry as authoritative, so a narrowed graph was indistinguishable from a complete one -- and because `group_sync` is MCP-exposed, one agent call durably narrowed the shared answer for every later reader with no signal at all. Add `suppressedMatchStages` to ContractRegistry and SyncResult, following the `unreadableRepos` tri-state end to end: absent means a registry written before the field existed, `[]` is the measurement "this run suppressed nothing", and a populated list names the stages. The writer always emits it, because omitting the empty case is what made "measured, none" unreachable for `unreadableRepos`. Two properties that are easy to get backwards, and are why the split matters: - SyncResult carries the marker on EVERY outcome. The sync genuinely did skip the stage whatever happened to the file afterwards, and the CLI summary (U3) renders from this rather than re-deriving it from the caller's options. - The PERSISTED registry stamps it only on the `written` outcome. The preserve path re-writes `{ ...prior }`, so a carried-forward registry keeps the marker of the sync that actually produced its contracts instead of being relabelled with this run's request. That holds by construction: the registry literal carrying the field is only reachable on the written path. `loadContractRegistryResilient` gains an explicit line, because it rebuilds the envelope field by field with no spread of the parsed root -- a new on-disk field is silently dropped unless named there. Its reader is `recordedMatchStages`, not the existing `recordedRepoList`: that one validates `string[]`, which is right for repo names and one notch too weak here. This repo has already retired MatchType members ('bm25', 'embedding'), so a stale value on disk is a real shape, and dropping non-members keeps an unknown stage name from reaching a caller typed as a live one. Surfaced on `group_contracts` and on `group_sync`'s own return -- deliberately kept separate from the truncated/truncationReason/riskEpistemic triple. That triple reports limits a run hit by accident, whose remedy is to fix the repo; a suppressed stage was asked for, and its remedy is to re-sync without the flag. Conflating them would tell an agent to retry something that returns identically. tsc clean; 1043/1043 group unit tests pass. * fix(group): name a skipped matching stage as skipped, and pin it (U3, U4) Two facts were printing as the same line. `wildcard: 0 cross-links` meant both "the stage ran and matched nothing" and "the stage never ran because you passed --exact-only" -- the same conflation this summary block was introduced to remove one line up, reintroduced by the flag that made the block necessary. Render a suppressed stage as `skipped (--exact-only)`, driven by the sync's own `suppressedMatchStages` rather than by `opts.exactOnly`. The renderer reports what the sync did, not what the caller asked for, so it stays correct on the outcomes where the run ended without writing a registry -- which is where the summary is least legible and a re-derivation from the options would have been wrong. Also drops the `?? 0` fallback and the comment justifying it. The comment claimed a legacy registry could carry a retired matchType into this loop. It cannot: `syncGroup` returns a freshly computed `crossLinks` array on every outcome, and even on the preserve path the prior links go to disk while the fresh array is returned. The code was harmless; the stated reason was false, and a comment that explains an unreachable path is worse than no comment. U4 pins both halves through the CLI. A manifest fixture is sufficient: the stage counts must sum to the total on the `Wrote contracts.json (…)` line, and the skipped rendering does not need a stage to have matched anything, because --exact-only records the suppression whatever the fixture holds. That is why this coverage did not need indexed gRPC/Thrift fixture repos. Verified by mutation, not assertion: removing the skipped-rendering branch turns `names a stage it was told to skip as skipped` red and leaves the other 22 green. A control case pins the opposite direction -- the same group without the flag still reports the stage as zero -- so `skipped` cannot be printed unconditionally and pass. U3 and U4 land together: the test has no value without the renderer, so one commit keeps a revert clean. Both depend on U2, which introduced the field they read. tsc clean; 1066/1066 across the group unit and CLI integration suites. * fix(group): make every description of --exact-only match what it does (U5) Two descriptions this branch wrote or touched still misstated behavior. The MCP `exactOnly` description carries "Manifest links still apply." The CLI help and both locale strings, rewritten in the same commit, omit it -- so the surface most operators read understated what still runs. Manifest cross-links are computed before the gate and are genuinely unaffected by the flag, so the caveat is the accurate half and the CLI now says it too. The `group_sync` tool description opened with "extract HTTP contracts". That clause was carried forward byte-identical while only the trailing cross-linking half was rewritten, and it is wrong: the detect config has six non-HTTP extraction toggles, and this branch's own new test fixture is Thrift. `help-i18n.ts` is deliberately untouched. It maps an option to its translation key and that key already exists; only the commander string and the two locale values carry text, so a text-only change does not reach it. The tool-description edit sits ahead of the registryOutcome paragraph, leaving the relative order of the 'preserved' / 'superseded' / 'no-prior-registry' literals intact -- `tools.test.ts` slices that description by their positions. tsc clean; 64/64 across the locale-parity, help-registration, tool-schema and group-tool suites. * fix(group)!: remove max_candidates_per_step and shared_libs (U6) Both keys were declared, defaulted, written into every generated group.yaml, and read by nothing -- the same three-station dead surface this PR removed for bm25_threshold, embedding_threshold and detect.embedding_fallback. Every other DetectConfig field gates a real extractor in sync.ts; shared_libs gates nothing, because 'lib' contracts come only from the operator-declared manifest extractor. MatchingConfig reaches matching.ts solely through buildNoisyContractFilter, which reads exclude_links_paths and exclude_links_param_only_paths and nothing else. Existing group.yaml files keep loading and keep their keys. parseGroupConfig spreads the raw block over its defaults, so a key the schema no longer knows about survives into the returned config -- which matters because `group add` and `group remove` round-trip the operator's file through loadGroupConfig -> yaml.dump -> write, so anything the parser dropped would be deleted from their checked-in file. The legacy-config test now pins both keys in the same cast form as its three siblings, and the fixture carries shared_libs so that assertion is not vacuous. Two stations that are easy to miss and are swept here: - gitnexus/bench/cross-repo-trace/verify.mjs GENERATES a fresh group.yaml. It is not a preserve-path fixture, so "leave YAML fixtures alone" does not cover it; the repo has two generators and both are updated. It is a .mjs file outside tsconfig's include, so no type gate would have caught it. - config-parser.test.ts asserted the removed default at runtime, which vitest DOES run. That assertion is gone from the defaults case (the key no longer has a default) and re-formed as a preserve assertion in the legacy case. Verification gate, corrected: "zero net new errors against origin/main" would have measured the whole branch delta and been red through no fault of this commit. Measured instead against the branch tip immediately before it -- tsc -p tsconfig.test.json --noEmit reports 989 before and 989 after. Twenty-four typed-literal sites across ten test files, none of them CI-gated, plus the two runtime sites above which are. Note the deliberate side effect: removing a key from the defaults also stops the group add round-trip from re-adding it to a file that never carried it. Nothing in src reads either key, so no behavior changes. BREAKING CHANGE: `matching.max_candidates_per_step` and `detect.shared_libs` are no longer part of the group.yaml schema and are no longer written into generated templates. Existing files carrying them continue to parse and retain them. src tsc clean; 1189/1189 across the group unit, group integration, locale-parity, help-registration and tool-schema suites. * docs(group): map PR #3020 review findings to the commits that close them Retitles the ledger to hold one section per reviewed PR and adds #3020's ten findings. Two things are stated rather than claimed away: `abda0d041` closes three findings because they are one code block plus the test that pins it, and the suppressed-stage marker is a coupled set because the renderer consumes the field the earlier commit introduces. Also records what is NOT closed here -- the PR description's false claim about `max_candidates_per_step` lives outside this branch. * refactor(group): apply simplify-pass findings Four cleanup agents (reuse, simplification, efficiency, altitude) over this run's diff. Efficiency was clean. The rest found five things worth fixing, two of which were real gaps rather than style. `recordedMatchStages` filtered unknown values instead of rejecting the list. That inverted the tri-state on the one field built to prevent exactly this conflation: a stale `['bm25']` -- the scenario its own comment cites as the motivation -- survived as `[]`, which on this field MEANS "measured, nothing was suppressed". A confident clean answer manufactured from a value we could not read. Now all-or-nothing, matching `recordedRepoList`. `gitnexus group contracts` showed nothing after an exact-only sync. The human renderer destructures a fixed field list and gates its incompleteness warning on `truncated`, so the marker reached the MCP payload and the JSON output but not the listing an operator actually reads. It now warns, separately from the `truncated` warning, because the remedies differ: one says fix the repo, this one says re-run without the flag. `verbose` was still coerced with `Boolean()` in the same call whose tool description this branch changed to promise "PARAMETERS ARE VALIDATED". Validated now, and added to the tool schema -- it was read by the backend and advertised nowhere. Reuse: the thrift wildcard-matchable pair existed twice, near-verbatim, in `sync-exact-only` and `registry-suppressed-stages`. Both now call a shared `makeWildcardPair` fixture, so the shape `runWildcardMatch` fires on is defined once. Simplification: dropped a `Set` built per sync over a list that only ever holds zero or one entries; iterating `Object.keys(STAGE_COUNTS) as MatchType[]` also keeps the exhaustiveness the `Record` was built for, which `Object.entries` had discarded. Deliberately not done, with reasons: a schema-driven unknown-parameter layer at the MCP chokepoint (five parameters are read by backends and declared in no schema, so a strict layer rejects working calls today, and it cannot produce the "was removed" message finding 3 is about); folding the marker into `GROUP_IMPACT_TRUNCATION_REASONS` (reverses a recorded plan decision and the bridge scope is an open question for the maintainer); a per-stage suppression cause `Record` (no second suppressor exists -- speculative); collapsing the six `detect` extractor branches into a table (a real generalization, but a refactor outside this diff); and converging an untouched pre-existing CLI test onto the new manifest helper (it captures a value the helper does not return, so the change risks more than the duplication costs). tsc clean; eslint 0 errors (1 pre-existing warning); 1085/1085. * docs(group): remove REVIEW-FINDINGS-MAP.md Removes the findings-to-commits ledger from the source tree. Note for anyone reading this in history: the file was introduced on main by #3012 and carried that PR's findings map; this branch had appended a #3020 section. Deleting it drops both. #3012's content is recoverable with `git show 2c0fb7753:gitnexus/src/core/group/REVIEW-FINDINGS-MAP.md`. * fix(group): stop cross-repo impact and trace claiming a narrowed graph is complete Closes the half of the suppressed-stage finding that was deferred. The reviewers were right that deferring it was the weak point: the motivating harm was named as `group_impact` and cross-repo `trace` traversing a graph missing real edges, and those were exactly the surfaces left uncovered. The deferral rested on an assumption that does not hold. "It is already blind to this, so we do not make it worse" is false: `--exact-only` was inert before this PR, so the number of narrowed registries in the world goes from zero to nonzero exactly when this lands. The blindness was harmless only while narrowing was impossible. And silence there is not neutral -- `cross-impact.ts` documents `truncated: false` as an affirmative completeness claim, so those tools were about to start asserting a complete answer over a knowingly short graph. `suppressedMatchStages` now rides the bridge the same way `unreadableRepos` does: persisted in meta.json (no BRIDGE_SCHEMA_VERSION bump -- meta fields have this precedent), read back all-or-nothing, and carried across the preserve path through `refreshPreservedBridgeMeta`'s diagnostics so a preserved bridge keeps the marker of the sync that actually built it. `crossRepoCompleteness` folds it in, which is what makes this one change reach all three surfaces -- that function is by design the ONE computation behind the truncation triple. Precedence is explicit: an unreadable or unaccounted repo outranks a suppressed stage, because it is the more serious structural gap and its remedy has to be the one reported. `'suppressed-stage'` is a new member of the truncation-reason union rather than a reuse of `'incomplete-sync'`. The earlier decision not to touch that union was about not conflating remedies -- telling an agent to repair a repo that read fine, for a narrowing it requested. A distinct member preserves that reasoning while letting the answer stop claiming completeness, which is what reusing the existing member would have destroyed. The union's guard test did its job: adding a member failed the check that every reason is explained on the agent-facing surface, so the impact tool description now names this one and its distinct remedy (re-run WITHOUT the flag; nothing failed to read). `group status` and its CLI renderer surface it too, on the populated case only -- absent is a registry predating the field and empty is the ordinary clean sync; neither earns a line. Deliberately still not done, and why: a repo-wide unknown-parameter layer for every MCP tool. Five parameters are read by backends and declared in no schema (`subgroupExact`, `unmatchedOnly`, `showClusters`, `showProcesses`, and `verbose` until this branch declared it), and three tools dispatch with no schema entry at all, so a strict layer rejects working calls until each is reconciled. That reconciliation is the work; the layer is the cheap part. It also cannot produce the "was removed and is no longer accepted" message the retired-parameter guard exists to give. tsc clean; eslint 0 errors (2 pre-existing warnings); 1159/1159 across the group unit, group integration and tool-schema suites. * fix(group): make the suppressed-stage signal actually reach its readers Applies the mechanical findings from the code review of the previous commit. That commit claimed cross-repo impact and trace stop reporting a narrowed graph as complete. Trace did; impact did not, and two operator-facing messages said something false. Four reviewers plus the cross-model pass converged on the same two defects, and the untested seams were exactly where they were. `runGroupImpact` recomputed the truncation reason and hardcoded its fallback, so it could never emit 'suppressed-stage' -- the value the previous commit added to the union and documented in the tool description. Every narrowed-but-readable bridge was reported as 'incomplete-sync', telling the caller to repair a repo that read fine. It now propagates the bridge's own reason, as cross-trace.ts already did. The preserve path stamped this run's request onto an older bridge. When no repo can be read the database and registry are kept from an earlier sync, so meta.json has to keep describing that sync; instead `{ ...existing, ...diagnostics }` overwrote its marker, leaving contracts.json, meta.json and bridge.lbug describing three different runs. Currently masked by unreadable-repo precedence, one loosened condition from a live wrong verdict. `group contracts` printed "the last sync did not record which repos it could read" after any exact-only sync: truncated was set with both repo lists empty, so the message fell through to the wrong branch. It is now gated on the reason, not the flag. `group impact` likewise blamed the local walk for a floor the flag caused. The tri-state reader is now defined once, in the leaf module whose own comment says it exists so this exact duplication cannot recur -- it had been copied into bridge-db.ts within one commit of that comment being true. Both agent-facing descriptions now name the field. The previous commit added it to three payloads and documented it on none. Tests cover what shipped green: the preserve path for both artifacts (verified by mutation -- reintroducing the stamp turns exactly one test red), and the reason's REACHABILITY. The existing guard only asserted each reason is described, which is why a documented-but-unemittable value passed it. Also corrects a comment that said the marker is deliberately not folded into the truncation triple. True when written; false one commit later. tsc clean; eslint 0 errors; 1163/1163 across the group unit, group integration and tool-schema suites. * fix(group): drop verbose from the MCP surface, fail a superseded bridge closed Two maintainer-directed findings from the review. verbose is removed from the group_sync MCP schema and from GroupService, and kept on the CLI. The parameter never did what either description claimed: the gates emit workspace-dependency discovery stats and one aggregate manifest line, not "each cross-link". Worse, they emit them through the server's logger, which an MCP caller cannot read at all -- so advertising it introduced precisely the kind of knob this PR exists to delete, in the PR that deletes them. SyncOptions keeps the field and the CLI keeps --verbose, because a CLI user really can see that output; its help now says "Show additional sync diagnostics", which is what it shows. It was added to the MCP schema earlier in this same PR, so there is no published compatibility burden in taking it back out. A caller that still sends it is ignored rather than refused: it was never a documented parameter, and the retired-name guard is reserved for ones this tool actually withdrew. The second fixes a split-brain the completeness work made materially worse. When contracts.json commits and the bridge write then fails, the previous database stays in place describing an EARLIER sync. Until now it kept vouching for itself, so group_impact could traverse the superseded graph and call its answer complete while group_contracts reported the advanced registry -- two public surfaces making contradictory epistemic claims out of one sync. That was tolerable when the disagreement was about counts. It is not, now that suppressed-stage makes completeness a correctness property. markBridgeProvenanceUnknown withdraws the claim without touching the database: bridgeMetaMatchesFile already gives provenanceUnknown highest precedence and refuses to vouch for the pair, so cross-repo answers downgrade to a floor until a sync succeeds. Deliberately not a re-stamp -- the metadata still describes the database it was written for, and saying otherwise recreates the mis-pairing the preserve path avoids. Deliberately not a delete -- the old graph is still worth having as a floor, it just stops being called complete. Best-effort, because it runs inside a failure handler and must not replace a reported bridge failure with an unrelated one; the warning now states which of the two happened. Shared registry+bridge generation identity is the architectural fix and is deliberately NOT attempted here. This is the PR-sized containment. Verified by mutation, both directions: neutering the withdrawal turns the new test red, and a control pins that a healthy sync does not withdraw provenance -- otherwise every successful run would report its own answers as a floor. tsc clean; eslint 0 errors; 1185/1185. * refactor(group): apply simplify-pass findings Four cleanup agents over the last five commits. Efficiency was clean and traced why: the containment helper is failure-path only, the reason ternary sits after the fan-out loop, and the tri-state readers run once per artifact read. The strongest finding was one the diff itself proved. `refreshPreservedBridgeMeta` enforced the never-persisted rule for `repoListsUnreadable` and `pairedWithDatabase` with two deletes in its own body, under a comment noting it was the only code that read metadata and wrote it back. That held exactly as long as there was one such caller. `markBridgeProvenanceUnknown` made it two, and inherited nothing. The strip now lives in `writeBridgeMeta`, so every writer gets it and no future one can forget; `pairedWithDatabase` is the dangerous one, because persisted it tells every later reader the pair was verified when nothing verified it. `group impact` still printed "fan-out stopped early" whenever `truncatedRepos` was non-empty — but the bridge's incomplete repos are unioned into that list even when zero crossings were attempted, so a structural gap was reported as a runtime one, with the only working remedy omitted. That is the same false-cause shape the contract listing was re-gated for one commit ago, left live one command over because the new reason was bolted in front of the old branch rather than replacing the thing it branched on. Now keyed on the reason. The `?? 'incomplete-sync'` arm in cross-impact was unreachable: reaching it needs `truncated` true with all three of its inputs false, which `truncated = runtimeTruncated || bridge.truncated` forbids. Flattened. Also: a `recordedMatchStages` insert had split `crossRepoCompleteness` from its own JSDoc; one new test was a strict subset of another; and the bridge-failure warning interleaved concatenation with a mid-chain ternary. The new invariant assertion was caught being VACUOUS by mutation before it shipped — seeded with a valid repo list, `readBridgeMeta` never sets the reader-only field, so it passed with or without the strip. The fixture now seeds an unreadable list, and both it and the pre-existing assertion go red when the strip is removed. Deliberately skipped, with reasons: a shared `firstTruncated` fold over `TruncationFields` (the right altitude, but it changes cross-trace's return assembly and that surface separately documents a 'timeout' rung it cannot emit — a behavior change, not a cleanup); a reason-keyed `explainFloor` helper across all four CLI renderers (real, but a four-site refactor); narrowing the persisted stage vocabulary to a `SuppressibleStage` alias (would be undone by the very extension the field was modelled as a list to allow); moving `verbose` to `logger.debug` and deleting `SyncOptions.verbose` (the maintainer explicitly directed keeping both); and merging the two tri-state readers behind a predicate (they are adjacent in one file now, so a tightening applies to both by inspection — the duplication the comment warned about was cross-FILE). tsc clean; eslint 0 errors; 1164/1164. * fix(group): address gitnexus-check findings Seven bot comments across two review rounds; five distinct after dedup. Four were valid and are fixed, two were already resolved by later commits the bot had not seen. The validator could throw from its own error path. `JSON.stringify` is the right renderer there — it is what distinguishes the string "false" from the boolean, which is the entire point of the message — but it throws on a BigInt and on a cyclic object. So a validator promising a structured `{ error }` instead rejected, and `callTool` is reachable directly, so neither input is hypothetical. Guarded, keeping the distinction and falling back for the shapes that cannot serialize. An unreadable suppression record read as "nothing was suppressed". `recordedMatchStages` is all-or-nothing by design, so garbage collapses to `undefined` — and the consumer treated `undefined` as an empty measurement, throwing that safety away and reporting a registry it could not parse as complete. Present-but-unreadable now forces the floor, while absent stays legitimate: a registry written before the field existed has no opinion and should not be dragged to a floor for it. Two test-side findings, both real and both invisible to CI because `tsconfig.json` is src-only. Three `mock.calls[0][1]` accesses did not type-check against a zero-arg mock, and four assertions read `truncationReason` / `riskEpistemic` straight off `CrossRepoCompleteness`, which is a discriminated union carrying them on one arm. Also removed a `StoredContract` import that went dead when those fixtures moved to `makeWildcardPair`. Worth recording: U6 set a test-config gate at 989 errors and later commits walked it to 994 without anyone re-measuring — the bot caught three of the five. Now 987, below the original baseline. Already fixed, not by this commit: the preserve-path stamp the bot flagged against 6ceac8b1f (fixed in 1fbe0dc6b) and the displaced completeness JSDoc (fixed in 2d2ef8c47). Both behavior fixes are mutation-verified: restoring the unguarded stringify turns the new unserializable-value test red, and a control pins that an absent record still reads as complete so the fails-closed change cannot pass by forcing every registry to a floor. src tsc clean; eslint clean; 1167/1167. * chore(autofix): apply prettier + eslint fixes via /autofix command --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gergo Magyar Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- gitnexus/bench/cross-repo-trace/verify.mjs | 3 - gitnexus/src/cli/group.ts | 124 ++++++++-- gitnexus/src/cli/help-i18n.ts | 2 - gitnexus/src/cli/i18n/en.ts | 7 +- gitnexus/src/cli/i18n/zh-CN.ts | 7 +- .../src/core/group/REVIEW-FINDINGS-MAP.md | 107 --------- gitnexus/src/core/group/bridge-db.ts | 80 +++++- gitnexus/src/core/group/completeness.ts | 36 ++- gitnexus/src/core/group/config-parser.ts | 5 - gitnexus/src/core/group/cross-impact.ts | 23 +- gitnexus/src/core/group/cross-trace.ts | 1 + gitnexus/src/core/group/service.ts | 118 ++++++++- gitnexus/src/core/group/storage.ts | 5 - gitnexus/src/core/group/sync.ts | 57 ++++- gitnexus/src/core/group/types.ts | 43 +++- gitnexus/src/mcp/resources.ts | 6 +- gitnexus/src/mcp/tools.ts | 12 +- .../test/integration/group/group-cli.test.ts | 98 ++++++++ .../group/group-sync-lock-concurrency.test.ts | 4 +- .../test/unit/group/config-parser.test.ts | 46 +++- gitnexus/test/unit/group/fixtures.ts | 35 +++ gitnexus/test/unit/group/group-tools.test.ts | 19 ++ gitnexus/test/unit/group/matching.test.ts | 24 -- .../group/registry-suppressed-stages.test.ts | 227 ++++++++++++++++++ .../group/service-group-sync-payload.test.ts | 133 ++++++++++ .../test/unit/group/sync-exact-only.test.ts | 81 +++++++ .../group/sync-partial-extraction.test.ts | 4 +- .../unit/group/sync-unreadable-repos.test.ts | 104 +++++++- .../group/sync-windowed-resolution.test.ts | 4 +- gitnexus/test/unit/group/sync.test.ts | 32 +-- gitnexus/test/unit/group/types.test.ts | 8 +- .../repo-manager-registry-strict-read.test.ts | 4 +- 32 files changed, 1198 insertions(+), 261 deletions(-) delete mode 100644 gitnexus/src/core/group/REVIEW-FINDINGS-MAP.md create mode 100644 gitnexus/test/unit/group/registry-suppressed-stages.test.ts create mode 100644 gitnexus/test/unit/group/sync-exact-only.test.ts diff --git a/gitnexus/bench/cross-repo-trace/verify.mjs b/gitnexus/bench/cross-repo-trace/verify.mjs index 8497af38d..e965bfc07 100644 --- a/gitnexus/bench/cross-repo-trace/verify.mjs +++ b/gitnexus/bench/cross-repo-trace/verify.mjs @@ -58,9 +58,6 @@ packages: {} detect: http: true matching: - bm25_threshold: 0.7 - embedding_threshold: 0.65 - max_candidates_per_step: 3 `; } diff --git a/gitnexus/src/cli/group.ts b/gitnexus/src/cli/group.ts index c68f7142d..7bb7bc180 100644 --- a/gitnexus/src/cli/group.ts +++ b/gitnexus/src/cli/group.ts @@ -2,6 +2,7 @@ import { createRequire } from 'node:module'; import type { Command } from 'commander'; import type { RegistryWriteOutcome } from '../core/group/sync.js'; +import type { MatchType } from '../core/group/types.js'; import { logger } from '../core/logger.js'; const _require = createRequire(import.meta.url); @@ -134,6 +135,7 @@ export function registerGroupCommands(program: Command): void { >; missingRepos?: string[]; unreadableRepos?: string[]; + suppressedMatchStages?: string[]; }; console.log(' Repo index / contracts staleness:'); @@ -188,6 +190,18 @@ export function registerGroupCommands(program: Command): void { if ((st.missingRepos || []).length > 0) { console.log(`\n Last sync missing repos: ${st.missingRepos!.join(', ')}`); } + // Only the populated case prints. Absent means a registry that predates + // the field, and empty is the ordinary clean sync — neither is worth a + // line, whereas a narrowed registry changes how every later answer + // should be read. + const skippedStages = st.suppressedMatchStages ?? []; + if (skippedStages.length > 0) { + console.log( + `\n Last sync skipped matching stages: ${skippedStages.join(', ')}` + + `\n Cross-links those stages would have found are absent by request.` + + `\n Re-run \`gitnexus group sync\` without --exact-only for the complete set.`, + ); + } } finally { await backend.dispose().catch(() => {}); } @@ -196,10 +210,11 @@ export function registerGroupCommands(program: Command): void { group .command('sync ') .description('Sync Contract Registry — extract contracts and build cross-links') - .option('--skip-embeddings', 'Exact + BM25 only (no embedding fallback)') - .option('--exact-only', 'Exact match only') - .option('--allow-stale', 'Skip stale index warnings') - .option('--verbose', 'Show each cross-link detail') + .option( + '--exact-only', + 'Skip wildcard service matching; cross-link on exact contract-id match only (manifest links still apply)', + ) + .option('--verbose', 'Show additional sync diagnostics') .option('--json', 'JSON output') .action(async (name: string, opts: Record) => { const { getGroupDir, getDefaultGitnexusDir } = await import('../core/group/storage.js'); @@ -216,9 +231,7 @@ export function registerGroupCommands(program: Command): void { try { result = await syncGroup(config, { groupDir, - allowStale: Boolean(opts.allowStale), verbose: Boolean(opts.verbose), - skipEmbeddings: Boolean(opts.skipEmbeddings), exactOnly: Boolean(opts.exactOnly), }); } catch (err) { @@ -257,10 +270,40 @@ export function registerGroupCommands(program: Command): void { `\n Index them with \`gitnexus analyze\`, or remove them from group.yaml.`, ); } - console.log(`\nMatching cascade:`); - const exactLinks = result.crossLinks.filter((l) => l.matchType === 'exact'); - console.log(` exact: ${exactLinks.length} cross-links (confidence 1.0)`); - console.log(` unmatched: ${result.unmatched.length} contracts`); + // Every stage that produced a link, not just `exact`. This used to print + // `Matching cascade:` and then count `exact` alone, while the `Wrote + // contracts.json (…)` line below reports `result.crossLinks.length` — + // which also includes `manifest` and `wildcard` links. For any group with + // those, the two numbers disagreed with nothing on screen explaining why. + // Summing the stages here makes them reconcile by construction. + console.log(`\nMatching:`); + // Exhaustive by construction, same idiom as OUTCOME_LINE below: adding a + // MatchType fails the build here instead of silently going uncounted and + // reopening the very mismatch this replaced. Every stage prints even at + // zero — a stage that is absent reads as "did not apply", not "found none". + const STAGE_COUNTS: Record = { + exact: 0, + manifest: 0, + wildcard: 0, + }; + for (const link of result.crossLinks) STAGE_COUNTS[link.matchType] += 1; + // A stage the sync was told to skip is reported as skipped, not as a + // zero count. The two are different facts — "ran, matched nothing" and + // "never ran" — and printing both as `0` is the same conflation this + // block replaced. Driven by what the sync did (`suppressedMatchStages`) + // rather than by what the caller asked for, so it stays correct on the + // outcomes where the run ended without writing a registry. + for (const stage of Object.keys(STAGE_COUNTS) as MatchType[]) { + const count = STAGE_COUNTS[stage]; + const label = `${stage}:`.padEnd(10); + if (result.suppressedMatchStages.includes(stage)) { + console.log(` ${label} skipped (--exact-only)`); + continue; + } + const confidence = stage === 'exact' ? ' (confidence 1.0)' : ''; + console.log(` ${label} ${count} cross-links${confidence}`); + } + console.log(` ${'unmatched:'.padEnd(10)} ${result.unmatched.length} contracts`); // Driven by what actually happened to the file. This line used to be // unconditional, so a run that deliberately preserved the previous // registry still announced `Wrote contracts.json (0 contracts, 0 @@ -391,11 +434,28 @@ export function registerGroupCommands(program: Command): void { // repos — reporting it as crossings understates a fan-out cap the // same way #2787's totals did. const dropped = (raw as { truncatedRepos?: string[] })?.truncatedRepos ?? []; - console.log( - dropped.length > 0 - ? ` risk is a LOWER BOUND — fan-out stopped early; crossings to ${dropped.length} repo(s) not traversed: ${dropped.join(', ')}` - : ' risk is a LOWER BOUND — the local impact walk did not complete (every bridge crossing was traversed)', - ); + const reason = (raw as { truncationReason?: string })?.truncationReason; + // Keyed on the REASON, not on which incidental fact happens to be + // non-empty. `truncatedRepos` is populated for a structural gap too + // — the bridge's incomplete repos are unioned into it even when ZERO + // crossings were attempted — so branching on its length first + // reported "fan-out stopped early" for a run where nothing stopped + // early, and omitted the only remedy that works. Same false-cause + // shape the contract listing was just re-gated for, one command over. + const floorReason = (): string => { + if (reason === 'suppressed-stage') { + return 'the last sync skipped a matching stage (--exact-only); re-run `gitnexus group sync` without it for the complete graph'; + } + if (reason === 'incomplete-sync') { + return dropped.length > 0 + ? `the last sync could not account for ${dropped.join(', ')}; their contracts are absent from every query against this bridge — re-run \`gitnexus group sync\`` + : 'the last sync could not say which repos it read — re-run `gitnexus group sync`'; + } + return dropped.length > 0 + ? `fan-out stopped early; crossings to ${dropped.length} repo(s) not traversed: ${dropped.join(', ')}` + : 'the local impact walk did not complete (every bridge crossing was traversed)'; + }; + console.log(` risk is a LOWER BOUND — ${floorReason()}`); } } } finally { @@ -480,7 +540,15 @@ export function registerGroupCommands(program: Command): void { return; } - const { contracts, crossLinks, truncated, unreadableRepos, missingRepos } = raw as { + const { + contracts, + crossLinks, + truncated, + unreadableRepos, + missingRepos, + suppressedMatchStages, + truncationReason, + } = raw as { contracts: Array<{ role: string; contractId: string; @@ -495,6 +563,8 @@ export function registerGroupCommands(program: Command): void { contractId: string; }>; truncated?: boolean; + suppressedMatchStages?: string[]; + truncationReason?: string; unreadableRepos?: string[]; missingRepos?: string[]; }; @@ -518,7 +588,27 @@ export function registerGroupCommands(program: Command): void { ` ${l.from.repo} -> ${l.to.repo} [${l.matchType}, conf=${l.confidence}] ${l.contractId}`, ); } - if (truncated) { + // Separate from `truncated` below, and deliberately so: that one means + // the sync could not read something and the remedy is to fix the repo. + // This one means the sync was ASKED to skip a stage, and the remedy is + // to re-run without the flag. A listing narrowed on purpose is still + // narrowed, and without this the human view showed nothing at all. + if (suppressedMatchStages && suppressedMatchStages.length > 0) { + console.log( + `\n⚠️ This listing is a lower bound: the last sync skipped ${suppressedMatchStages.join(', ')} matching` + + `\n (--exact-only), so cross-links that stage would have found are absent.` + + `\n Re-run \`gitnexus group sync\` without --exact-only for the complete set.`, + ); + } + // Gated on the REASON, not just the flag. A suppressed stage sets + // `truncated` with both repo lists empty, which sent this block down + // its else-branch and printed "the last sync did not record which + // repos it could read" — a false statement, with the wrong remedy, + // about a sync that recorded them fine. The suppressed-stage warning + // above already said the true thing. When a repo gap co-occurs the + // reason is 'incomplete-sync' (the repo side takes precedence in + // `crossRepoCompleteness`), so this block still runs for it. + if (truncated && truncationReason !== 'suppressed-stage') { // Counts above are a floor, not a census. Name the repos when the // registry recorded them, and say so plainly when it did not — a // listing that cannot say what it is missing is still incomplete. diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index 58f28d11a..eeea3eeae 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -155,9 +155,7 @@ const OPTION_DESCRIPTION_KEYS = { 'embeddings install|--cuda': 'help.option.embeddings.install.cuda', 'embeddings install|--force': 'help.option.embeddings.install.force', 'group create|--force': 'help.option.group.create.force', - 'group sync|--skip-embeddings': 'help.option.group.sync.skipEmbeddings', 'group sync|--exact-only': 'help.option.group.sync.exactOnly', - 'group sync|--allow-stale': 'help.option.group.sync.allowStale', 'group sync|--verbose': 'help.option.group.sync.verbose', 'group sync|--json': 'help.option.json', 'group impact|--target ': 'help.option.group.impact.target', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index aaaf442cb..e893e61ac 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -293,10 +293,9 @@ export const en = { 'help.option.embeddings.install.force': 'Install into the runtime prefix even when the stack already resolves', 'help.option.group.create.force': 'Overwrite existing group', - 'help.option.group.sync.skipEmbeddings': 'Exact + BM25 only (no embedding fallback)', - 'help.option.group.sync.exactOnly': 'Exact match only', - 'help.option.group.sync.allowStale': 'Skip stale index warnings', - 'help.option.group.sync.verbose': 'Show each cross-link detail', + 'help.option.group.sync.exactOnly': + 'Skip wildcard service matching; cross-link on exact contract-id match only (manifest links still apply)', + 'help.option.group.sync.verbose': 'Show additional sync diagnostics', 'help.option.status.json': 'Emit machine-readable index and analyzer provenance', 'help.option.json': 'JSON output', 'help.option.group.impact.target': 'Symbol or file name to analyze', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 0ed1c7f1e..2c066f010 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -273,10 +273,9 @@ export const zhCN = { '同时下载 CUDA GPU 二进制文件（运行 onnxruntime-node 的 NuGet postinstall；代理后请设置 GLOBAL_AGENT_HTTPS_PROXY）', 'help.option.embeddings.install.force': '即使嵌入组件已可解析，也强制安装到运行时目录', 'help.option.group.create.force': '覆盖现有仓库组', - 'help.option.group.sync.skipEmbeddings': '仅使用 exact + BM25（不使用嵌入回退）', - 'help.option.group.sync.exactOnly': '仅精确匹配', - 'help.option.group.sync.allowStale': '跳过过期索引警告', - 'help.option.group.sync.verbose': '显示每条跨仓库链接详情', + 'help.option.group.sync.exactOnly': + '跳过通配符服务匹配，仅按契约 ID 精确匹配建立跨仓链接（清单声明的链接仍然生效）', + 'help.option.group.sync.verbose': '显示额外的同步诊断信息', 'help.option.status.json': '输出机器可读的索引和分析器来源信息', 'help.option.json': 'JSON 输出', 'help.option.group.impact.target': '要分析的符号或文件名', diff --git a/gitnexus/src/core/group/REVIEW-FINDINGS-MAP.md b/gitnexus/src/core/group/REVIEW-FINDINGS-MAP.md deleted file mode 100644 index aebe73303..000000000 --- a/gitnexus/src/core/group/REVIEW-FINDINGS-MAP.md +++ /dev/null @@ -1,107 +0,0 @@ -# Review findings → commits (PR #3012) - -Every finding raised in review of this PR, and the commit that closes it. The -Definition of Done claims each finding has exactly one commit and that reverting -that commit reintroduces that finding and no other; this is what makes the claim -checkable without the reviewer's report in hand. - -**Not under `docs/`** — that path is gitignored, so a map written there would -never reach the PR and nobody but its author could perform the audit. It lives -beside the code it describes, as `PIPELINE.md` does. - -## Revert contract - -Revertability is **dependency-aware**. Where one commit extracts a helper that -later commits consume, reverting the helper alone does not build. The contract -is: reverting a commit reintroduces its own finding and no other _finding_, with -its prerequisite commits retained. - -One coupled set exists: - -| Set | Commits | Why coupled | -| -------------------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------- | -| Shared completeness helper | `4c203ac7b` ← `79f6f5bcb`, `0fe6fc9d4`, `dbc3953b0` | The three consumers call `crossRepoCompleteness`; reverting it alone breaks the build. | - -## Primary findings - -| # | Finding | Commit | -| --- | -------------------------------------------------------------------------------- | ----------- | -| 1 | Malformed `meta.json` crashes cross-repo impact and leaks the bridge handle | `27b0069f2` | -| 2 | Unreadable repos still contribute contracts through deferred manifest resolution | `7037e8441` | -| 3 | Strict read accepts a registry row that cannot identify a repo | `5245b22d7` | -| 4 | Unstamped bridge metadata is trusted without any check | `94f2a8757` | -| 5 | A subgroup-scoped query is marked incomplete by repos it excluded | `79f6f5bcb` | -| 6 | The preserved registry and the bridge disagree about the same sync | `4676abf03` | -| 7 | Three surfaces compute completeness three different ways | `4c203ac7b` | -| 8 | `group_contracts` has no channel for its own completeness | `0fe6fc9d4` | -| 9 | `group status` cannot tell a missing entry from an unreadable registry | `a12b846c9` | -| 10 | The sync summary describes a write that did not happen that way | `5a668455c` | -| 11 | The total-failure log promises preservation where there is nothing to preserve | `c4b356b29` | -| 12 | The bridge-failure warning promises a truncation the code never reports | `1df79bb9a` | -| 13 | Two concurrent syncs of one group lose each other's writes | `4f07359bf` | -| 14 | The bridge swap needs the lock its caller already holds | `3b6215862` | -| 15 | The byte guard misses most tracked text files, and all extensionless ones | `07bf8be75` | -| 16 | The byte guard reads the vendored grammar tree it does not need to judge | `3ef831a0a` | -| 17 | The strict-read test cannot see which registry read ran | `eccc3c682` | -| 18 | The CLI branches this PR introduced have no assertions | `535d2ad29` | -| 19 | The MCP payloads have no assertions | `2c253b4a8` | -| 20 | Corrupt-registry errors quote the file's bytes, credentials included | `24ba2a537` | -| 21 | The mtime pairing's limits are recorded nowhere a reader will look | `ca0aca106` | -| 22 | The bridge-input docstring narrows what `unreadableRepos` means | `8c930f470` | -| 23 | The strict-read docstring's call-site count is wrong | `a95838954` | -| 24 | Contract staging crashes on the engine's argument limit | `57eac7558` | -| 25 | The sync tool's description names two of three reachable outcomes | `8bfd1a6ab` | -| 26 | The impact tool and status resource do not explain incompleteness | `dbc3953b0` | -| 27 | A lock timeout blames an `analyze` it cannot establish | `2d2a0119e` | -| 28 | A losing sync downgrades the one that beat it to the lock | `e407f05cf` | - -## Findings raised in review and deliberately not implemented as suggested - -| Finding | Suggested fix | What shipped, and why | -| ---------------------------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Unstamped metadata is trusted | Treat every absent stamp as incomplete | Rejected. It would mark every pre-existing bridge a lower bound until re-synced — a repo-wide regression traded for a narrow window. The write-order pairing in `94f2a8757` is the narrower fix. | -| Stale bridge signal after a failed write | Re-stamp the metadata so the warning's promise becomes true | Rejected. Re-stamping recreates the metadata/database mis-pairing that stamping exists to prevent. `1df79bb9a` corrects the warning instead. | -| Strict row gate | Require all three fields non-blank | Narrowed to `name` and `storagePath`. This gate rejects the whole registry, which is machine-wide, so a field tightened past what identification needs lets one blank value break every group sync on the machine. | - -## Found during execution, not in the review - -| What | Commit | -| --------------------------------------------------------------------------------------------- | ----------- | -| A half-written bridge stamp read as a verified match (found by the repo's own contract check) | `066f2d802` | -| `readBridgeMeta`'s widened return type blocked the merge on contract drift | `a9d281dd4` | -| `group contracts --json` discarded every field it did not re-serialize | `b7753575d` | -| `sync.ts` renders as a binary diff because the base blob carries a NUL | `1667c24b4` | - -## Corrections to the plan, found while executing it - -Recorded because each was a claim in the plan that the code contradicted. - -| Claim | Reality | -| ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| The strict gate should require the fields "the resolution path consumes" | `defaultResolveHandle` **does** consume `path`. The distinction is what _identifies_ the repo. | -| Pass the trace's two endpoint repos as the scope predicate | A destination trace declares no `to`. Narrowing to `from` would report an unreadable provider as "no outgoing link". | -| Filter the incomplete set by the subgroup prefix | The query's own repo must stay in scope, or an unreadable origin becomes a confident "nothing depends on this". | -| `group status`'s third failure mode is a row that resolves but cannot be opened | Unreachable — `loadMeta` returns `null` on every error and `checkStaleness` catches everything. The reachable case is `resolveRepo` throwing. | -| The mtime rule can only demote pairs already broken | False. `cp -r` and `rsync` without `-t` demote an intact pair. Recorded at the code in `ca0aca106`. | -| `.scm` files are "edited constantly" here | Every tracked `.scm` is vendored. This repo writes tree-sitter queries inline in TypeScript. | - -## Residual risks, recorded rather than closed - -- **Credentials in the registry.** HTTPS remote URLs are persisted with their - userinfo intact. `24ba2a537` stops one channel echoing them; it does not stop - them being written. Pre-existing, tracked separately. -- **`readRegistryFile`'s read error.** The ENOENT-guarded outer catch still - rethrows the raw `fs.readFile` error into `unresolvableReason`. Node embeds - the path, not file contents, so no registry bytes leak — but it is the one - remaining foreign error object on that path. -- **Abstract-socket lock scope.** Linux abstract sockets are - network-namespace-scoped, so two containers sharing a bind-mounted group - directory do not contend unless the file backend is forced. Recorded at - `group-lock.ts`. -- **Scope filter at depth > 1.** The declared-scope intersection is sound only - while `MAX_SUPPORTED_CROSS_DEPTH` is 1. At depth 2 an out-of-scope repo can - sit between two in-scope ones. Recorded at the intersection site. -- **R14 is unmet on this PR.** `.gitattributes` makes TypeScript diffs render as - text, and it works locally — but GitHub resolves the attribute from the base - side, which does not carry it. `sync.ts` renders as binary in this PR's web - view and will render as text for every PR after this one merges. diff --git a/gitnexus/src/core/group/bridge-db.ts b/gitnexus/src/core/group/bridge-db.ts index 17eb24b76..3c8c07bf2 100644 --- a/gitnexus/src/core/group/bridge-db.ts +++ b/gitnexus/src/core/group/bridge-db.ts @@ -3,9 +3,16 @@ import path from 'node:path'; import { createHash } from 'node:crypto'; import lbug from '@ladybugdb/core'; import type { LbugValue } from '@ladybugdb/core'; -import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js'; +import type { + BridgeHandle, + BridgeMeta, + StoredContract, + CrossLink, + RepoSnapshot, + MatchType, +} from './types.js'; import { BRIDGE_SCHEMA_QUERIES, BRIDGE_SCHEMA_VERSION } from './bridge-schema.js'; -import { recordedRepoList } from './completeness.js'; +import { recordedMatchStages, recordedRepoList } from './completeness.js'; import { closeLbugConnection, openLbugConnection, @@ -649,7 +656,15 @@ export async function closeBridgeDb(handle: BridgeHandle): Promise { /* ------------------------------------------------------------------ */ export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise { - await writeFileAtomic(path.join(groupDir, 'meta.json'), JSON.stringify(meta, null, 2)); + // Strip the reader-only fields HERE rather than at each writer. `readBridgeMeta` + // sets both on what it returns, so any caller that reads-modifies-writes would + // round-trip them to disk — and `pairedWithDatabase` is the poisonous one: + // persisted, it tells every future reader the pair was verified when nothing + // verified it. That rule used to live in the body of the only such caller, + // which held exactly as long as there was one. There are now three writers and + // two of them read first. Enforced at the boundary, no writer can get it wrong. + const { repoListsUnreadable: _reader1, pairedWithDatabase: _reader2, ...persisted } = meta; + await writeFileAtomic(path.join(groupDir, 'meta.json'), JSON.stringify(persisted, null, 2)); } /** @@ -826,6 +841,10 @@ export async function readBridgeMeta(groupDir: string): Promise { // was there and could not be read. if (unreadableRepos) meta.unreadableRepos = unreadableRepos; else delete meta.unreadableRepos; + // Same absent-vs-empty rule, through the one shared reader. + const suppressed = recordedMatchStages(raw.suppressedMatchStages); + if (suppressed) meta.suppressedMatchStages = suppressed; + else delete meta.suppressedMatchStages; if (repoListsUnreadable) meta.repoListsUnreadable = true; return meta; } @@ -902,6 +921,12 @@ async function fileExists(filePath: string): Promise { */ export async function refreshPreservedBridgeMeta( groupDir: string, + // Deliberately NOT `suppressedMatchStages`. This path preserves an EARLIER + // sync's database, so stamping it with this run's request would claim the + // untouched bridge was built with a flag it never saw. The registry's own + // preserve write (`{ ...prior, missingRepos, unreadableRepos }`) omits it for + // exactly this reason, and the two artifacts have to agree about which run + // they describe. diagnostics: { missingRepos: string[]; unreadableRepos: string[] }, ): Promise { const dbPath = path.join(groupDir, 'bridge.lbug'); @@ -921,10 +946,8 @@ export async function refreshPreservedBridgeMeta( const refreshed: BridgeMeta = { ...existing, ...diagnostics }; // NEVER PERSISTED (see `BridgeMeta`): both are things a READER computes ABOUT // a file, and this is the first code in the repo that reads metadata and - // writes it back. `pairedWithDatabase` is the poisonous one — persisted, it - // would tell every future reader that the pair had been verified. - delete refreshed.repoListsUnreadable; - delete refreshed.pairedWithDatabase; + // writes it back. The strip itself now lives in `writeBridgeMeta`, so every + // writer inherits it rather than each remembering. if (paired) { const stat = await fsp.stat(dbPath).catch(() => null); @@ -944,6 +967,39 @@ export async function refreshPreservedBridgeMeta( return 'provenance-unknown'; } +/** + * Withdraw the bridge's claim to be complete, without touching the database. + * + * The one path this exists for: `contracts.json` committed, then the bridge + * replacement failed. The old database is still physically usable and still + * answers queries, but it now describes an EARLIER sync than the canonical + * registry beside it — so `group_contracts` can report a narrowed or advanced + * contract set while `group_impact` traverses the old graph and calls its + * answer complete. Two public surfaces, contradictory epistemic claims, from + * one sync. + * + * Setting `provenanceUnknown` is the smallest thing that makes that safe: + * `bridgeMetaMatchesFile` gives it highest precedence and refuses to vouch for + * the pair, so every cross-repo answer downgrades to a floor until a sync + * succeeds. Deliberately NOT a re-stamp — the metadata still describes the + * database it was written for, and claiming otherwise is the mis-pairing the + * preserve path is careful to avoid. Deliberately not a delete either: the + * previous graph is better than nothing as long as nobody calls it complete. + * + * Best-effort by construction. It runs inside a failure handler, so a throw + * here would replace a reported bridge failure with an unrelated one. + */ +export async function markBridgeProvenanceUnknown(groupDir: string): Promise { + try { + const existing = await readBridgeMeta(groupDir); + if (existing.version === 0) return false; + await writeBridgeMeta(groupDir, { ...existing, provenanceUnknown: true }); + return true; + } catch { + return false; + } +} + /* ------------------------------------------------------------------ */ /* writeBridge — atomic write-to-temp-then-rename */ /* ------------------------------------------------------------------ */ @@ -969,6 +1025,13 @@ export interface WriteBridgeInput { * contract those repos own. */ unreadableRepos?: string[]; + /** + * Matching stages the sync was asked to skip. Recorded here for the same + * reason `unreadableRepos` is: a later cross-repo query reads this bridge + * with no access to the run that built it, and a graph narrowed by request + * looks exactly like a complete one. + */ + suppressedMatchStages?: MatchType[]; } /** @@ -1322,6 +1385,9 @@ export async function writeBridgeUnlocked( // different claim from a bridge that never recorded the field. Omitted // only when the caller passed nothing to record. ...(input.unreadableRepos ? { unreadableRepos: input.unreadableRepos } : {}), + ...(input.suppressedMatchStages + ? { suppressedMatchStages: input.suppressedMatchStages } + : {}), }); return report; diff --git a/gitnexus/src/core/group/completeness.ts b/gitnexus/src/core/group/completeness.ts index 326d10b2d..16aee0473 100644 --- a/gitnexus/src/core/group/completeness.ts +++ b/gitnexus/src/core/group/completeness.ts @@ -13,7 +13,7 @@ * Nothing here imports anything but types. Keep it that way: the moment this * file gains a runtime import, every consumer pays for it again. */ -import type { GroupImpactTruncationReason } from './types.js'; +import type { GroupImpactTruncationReason, MatchType } from './types.js'; /** * A union rather than `Pick` so the two states are @@ -69,6 +69,12 @@ export interface CrossRepoCompletenessInput { */ unreadableRepos?: readonly string[]; missingRepos?: readonly string[]; + /** + * Matching stages the sync was asked to skip. Absent or empty means it + * suppressed none; a populated list makes the answer a floor for a reason + * that is neither a runtime limit nor an unreadable repo. + */ + suppressedMatchStages?: readonly string[]; /** Computed by the caller; see `bridgeProvenanceUnknown` for the bridge one. */ provenanceUnknown: boolean; /** @@ -93,6 +99,23 @@ export type CrossRepoCompleteness = TruncationFields & { incompleteRepos: string[]; }; +/** + * Read a persisted `suppressedMatchStages` list. + * + * Sibling of `recordedRepoList` and here for the same stated reason: it had + * lived in two files verbatim, so tightening one would silently leave the other. + * All-or-nothing like its sibling — a stale member (this repo has already + * retired `'bm25'` and `'embedding'`) makes the whole list unreadable rather + * than filtering down to `[]`, which on this field would mean "measured, + * nothing suppressed": a clean answer manufactured from a value we could not + * read. + */ +export function recordedMatchStages(value: unknown): MatchType[] | undefined { + if (!Array.isArray(value)) return undefined; + const known: MatchType[] = ['exact', 'manifest', 'wildcard']; + return value.every((v): v is MatchType => known.includes(v as MatchType)) ? value : undefined; +} + /** * The ONE computation of "is this cross-repo answer complete?" (KTD10). * @@ -111,8 +134,17 @@ export function crossRepoCompleteness(input: CrossRepoCompletenessInput): CrossR const incompleteRepos = [ ...new Set([...(input.unreadableRepos ?? []), ...(input.missingRepos ?? [])]), ].filter((repoPath) => input.inScope(repoPath)); + // An unreadable or unaccounted repo outranks a suppressed stage: it is the + // more serious structural gap and its remedy (repair the repo, re-sync) has + // to be the one reported. A suppressed stage only decides the reason when + // the repo side is otherwise clean. + const suppressed = (input.suppressedMatchStages ?? []).length > 0; + const repoSideIncomplete = input.provenanceUnknown || incompleteRepos.length > 0; return { - ...truncationFields(input.provenanceUnknown || incompleteRepos.length > 0, 'incomplete-sync'), + ...truncationFields( + repoSideIncomplete || suppressed, + repoSideIncomplete ? 'incomplete-sync' : 'suppressed-stage', + ), incompleteRepos, }; } diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index 29c868171..754f578f9 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -29,16 +29,11 @@ const DEFAULT_DETECT = { grpc: true, thrift: true, topics: true, - shared_libs: true, - embedding_fallback: true, includes: false, workspace_deps: false, }; const DEFAULT_MATCHING = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: [] as string[], exclude_links_param_only_paths: false, }; diff --git a/gitnexus/src/core/group/cross-impact.ts b/gitnexus/src/core/group/cross-impact.ts index 21cbbe055..19ddd6fee 100644 --- a/gitnexus/src/core/group/cross-impact.ts +++ b/gitnexus/src/core/group/cross-impact.ts @@ -856,6 +856,7 @@ export async function runGroupImpact( const bridge = crossRepoCompleteness({ unreadableRepos: bridgePrep.meta.unreadableRepos, missingRepos: bridgePrep.meta.missingRepos, + suppressedMatchStages: bridgePrep.meta.suppressedMatchStages, provenanceUnknown, inScope: (candidate) => repoInSubgroup(candidate, subgroup) || repoInSubgroup(candidate, repoPath, true), @@ -878,15 +879,23 @@ export async function runGroupImpact( // and under-reporting a blast radius is the unsafe direction (an agent told // LOW proceeds; told CRITICAL it stops). Marking the floor keeps the // warning intact while making the incompleteness legible. - // Runtime limits first — they are what the caller can retry. 'incomplete-sync' - // is the remaining cause once nothing was merely cut short, and its remedy is - // a different one: re-run `gitnexus group sync`, not the query. Computed - // inline because `truncationFields` reads the reason ONLY on the truncated - // branch — naming it in a variable invited reading it on the complete path, - // where it would say 'incomplete-sync' about a complete result. + // Runtime limits first — they are what the caller can retry. Past those, the + // BRIDGE's own reason wins: it already distinguished an unreadable repo + // ('incomplete-sync', remedy: re-sync) from a stage the sync was asked to + // skip ('suppressed-stage', remedy: re-sync WITHOUT the flag). Hardcoding + // the fallback here overrode that and told every caller to repair a repo + // that read fine — and made the second value unreachable from this surface + // while the tool description promised it. `cross-trace.ts` re-spreads the + // bridge's fields for the same reason. ...truncationFields( truncated, - fanoutTimedOut ? 'timeout' : runtimeTruncated ? 'partial' : 'incomplete-sync', + fanoutTimedOut + ? 'timeout' + : runtimeTruncated + ? 'partial' + : bridge.truncated + ? bridge.truncationReason + : 'incomplete-sync', ), truncatedRepos: [...new Set([...truncatedRepos, ...bridge.incompleteRepos])], summary: { diff --git a/gitnexus/src/core/group/cross-trace.ts b/gitnexus/src/core/group/cross-trace.ts index ddb771703..92c52e789 100644 --- a/gitnexus/src/core/group/cross-trace.ts +++ b/gitnexus/src/core/group/cross-trace.ts @@ -264,6 +264,7 @@ function bridgeCompletenessFor( return crossRepoCompleteness({ unreadableRepos: meta.unreadableRepos, missingRepos: meta.missingRepos, + suppressedMatchStages: meta.suppressedMatchStages, provenanceUnknown: bridgeProvenanceUnknown(meta), inScope, }); diff --git a/gitnexus/src/core/group/service.ts b/gitnexus/src/core/group/service.ts index 79f67abe6..af1f1385a 100644 --- a/gitnexus/src/core/group/service.ts +++ b/gitnexus/src/core/group/service.ts @@ -15,7 +15,7 @@ import { type RepoMeta, } from '../../storage/repo-manager.js'; import { crossRepoCompleteness } from './completeness.js'; -import { recordedRepoList } from './completeness.js'; +import { recordedMatchStages, recordedRepoList } from './completeness.js'; import { GroupNotFoundError, loadGroupConfig } from './config-parser.js'; import { fileMatchesServicePrefix, @@ -262,7 +262,8 @@ function registryIdentifies(entries: RegistryEntry[], registryName: string): boo async function loadContractRegistryResilient( groupDir: string, ): Promise< - { ok: true; registry: ContractRegistry; skippedCorrupt: number } | { ok: false; error: string } + | { ok: true; registry: ContractRegistry; skippedCorrupt: number; suppressionUnreadable: boolean } + | { ok: false; error: string } > { const filePath = path.join(groupDir, 'contracts.json'); let raw: string; @@ -327,6 +328,15 @@ async function loadContractRegistryResilient( // Bound once: the gate is a full array scan and the ternary below used it twice. const recordedUnreadable = recordedRepoList(base.unreadableRepos); + const recordedSuppressed = recordedMatchStages(base.suppressedMatchStages); + // Present-but-unreadable is NOT the same as absent. `recordedMatchStages` is + // all-or-nothing, so garbage collapses to `undefined` — and a consumer that + // reads `undefined` as "nothing was suppressed" would throw that safety away + // and report a registry it could not parse as complete. Absent stays + // legitimate (a registry predating the field); only a value that was there + // and unreadable forces the answer to a floor. + const suppressionUnreadable = + base.suppressedMatchStages !== undefined && recordedSuppressed === undefined; const registry: ContractRegistry = { version: typeof base.version === 'number' ? base.version : 0, generatedAt: typeof base.generatedAt === 'string' ? base.generatedAt : '', @@ -348,11 +358,75 @@ async function loadContractRegistryResilient( // rendered as a clean result, which is the same conflation this whole // change removes. ...(recordedUnreadable ? { unreadableRepos: recordedUnreadable } : {}), + // Same omit-when-unrecorded rule. This reader rebuilds the envelope field + // by field with no spread of `base`, so a new on-disk field is dropped + // unless it is named here. + ...(recordedSuppressed ? { suppressedMatchStages: recordedSuppressed } : {}), contracts, crossLinks, }; - return { ok: true, registry, skippedCorrupt }; + return { ok: true, registry, skippedCorrupt, suppressionUnreadable }; +} + +/** + * Validate a boolean MCP parameter — reject, never coerce. + * + * `Boolean(params.x)` is the trap this exists to close: the string `"false"` + * is truthy, and an LLM caller emitting JSON produces that shape routinely. + * While `exactOnly` was inert the coercion was harmless; now that it gates a + * matching stage, a coerced `"false"` suppresses that stage and persists a + * registry with fewer cross-links than the caller asked for. + * + * Absent stays absent-as-false (the unchanged default). Anything that is not + * a real boolean returns a structured `{ error }`, mirroring + * `validateImpactMode` — the established shape for this boundary, and the one + * `groupSync`'s other guards already use. + */ +function validateBooleanParam(name: string, raw: unknown): { value: boolean } | { error: string } { + if (raw === undefined) return { value: false }; + if (typeof raw === 'boolean') return { value: raw }; + return { error: `Invalid "${name}": expected true or false, got ${describeValue(raw)}.` }; +} + +/** + * Render an untrusted value for an error message, without throwing. + * + * `JSON.stringify` is the right shape here — it distinguishes the string + * `"false"` from the boolean, which is the whole point of the message — but it + * throws on a BigInt and on a cyclic object. A validator whose ERROR path can + * throw does not return the structured `{ error }` it promises: the caller gets + * a rejected promise instead of feedback it can act on, and `callTool` is + * reachable directly, so neither input is hypothetical. + */ +function describeValue(raw: unknown): string { + try { + const rendered = JSON.stringify(raw); + // `undefined`, a function, or a symbol serialize to `undefined`. + return rendered ?? String(raw); + } catch { + return typeof raw === 'bigint' ? `${raw}n` : Object.prototype.toString.call(raw); + } +} + +/** + * Refuse parameters this tool used to accept and no longer does. + * + * The CLI rejects a removed flag outright because commander errors on an + * unknown option. The MCP path had no equivalent, so an agent working from a + * cached tool schema kept sending a retired key and was told nothing — the + * removal took away discoverability, not acceptance. Naming the parameter is + * what lets the caller correct itself on the next call. + */ +function rejectRetiredSyncParams(params: Record): { error: string } | null { + for (const retired of ['skipEmbeddings', 'allowStale']) { + if (params[retired] !== undefined) { + return { + error: `"${retired}" was removed and is no longer accepted. Drop it from the call.`, + }; + } + } + return null; } export class GroupService { @@ -384,6 +458,13 @@ export class GroupService { async groupSync(params: Record): Promise { const name = String(params.name ?? '').trim(); if (!name) return { error: 'name is required' }; + // Before anything reads the group off disk: the MCP SDK does not enforce a + // tool's advertised `inputSchema` and `callTool` is reachable directly, so + // this method is the real validation boundary. + const exactOnly = validateBooleanParam('exactOnly', params.exactOnly); + if ('error' in exactOnly) return exactOnly; + const retired = rejectRetiredSyncParams(params); + if (retired) return retired; const groupDir = getGroupDir(getDefaultGitnexusDir(), name); let config: GroupConfig; try { @@ -404,10 +485,11 @@ export class GroupService { try { result = await syncGroup(config, { groupDir, - exactOnly: Boolean(params.exactOnly), - skipEmbeddings: Boolean(params.skipEmbeddings), - allowStale: Boolean(params.allowStale), - verbose: Boolean(params.verbose), + exactOnly: exactOnly.value, + // `verbose` is deliberately NOT accepted here. It gates diagnostics on + // the server's logger, which an MCP caller cannot observe — advertising + // it would be exactly the kind of knob that does not do what the caller + // expects. `SyncOptions.verbose` stays for the CLI, which can see them. }); } catch (err) { // Fails closed (R9): this sync could not be protected against a concurrent @@ -423,6 +505,10 @@ export class GroupService { unmatched: result.unmatched.length, missingRepos: result.missingRepos, unreadableRepos: result.unreadableRepos, + // The agent-facing half of the skipped-stage signal. A human sees it in + // the CLI summary; without this an agent would have to issue a second + // `group_contracts` call to discover its own sync was narrowed. + suppressedMatchStages: result.suppressedMatchStages, // An agent that calls group_sync and then group_contracts a moment later // can otherwise see contract counts that disagree with this payload, with // nothing here explaining why the write was skipped. @@ -465,9 +551,14 @@ export class GroupService { const { incompleteRepos: _incompleteRepos, ...truncation } = crossRepoCompleteness({ unreadableRepos, missingRepos, + suppressedMatchStages: registry.suppressedMatchStages, // An unrecorded `unreadableRepos` means this listing cannot say which // repos the sync failed to read — so it cannot claim to be complete. - provenanceUnknown: unreadableRepos === undefined, + // Either kind of unreadable provenance forces the floor: a sync that + // could not say which repos it read, or a suppression record that was + // present and could not be parsed. Reading the second as "nothing was + // suppressed" would report an unparseable registry as complete. + provenanceUnknown: unreadableRepos === undefined || loaded.suppressionUnreadable, // A contract LISTING declares no scope to intersect with: it is the whole // registry, so every configured repo is in scope by construction. The // `type`/`repo`/`unmatchedOnly` filters above narrow which rows are shown, @@ -482,6 +573,13 @@ export class GroupService { // convention `skippedCorrupt` follows below, and the difference between // "the sync measured zero unreadable repos" and "the sync never said". ...(unreadableRepos ? { unreadableRepos } : {}), + // Same omit-when-unrecorded rule, and deliberately NOT folded into the + // truncation triple below: that triple reports limits a run hit by + // accident, whose remedy is to fix the repo. A suppressed stage was + // asked for, and its remedy is to re-sync without that flag. + ...(registry.suppressedMatchStages + ? { suppressedMatchStages: registry.suppressedMatchStages } + : {}), // The structured triple, verbatim from the impact surface (KTD10): // `truncated` always, `truncationReason` + `riskEpistemic` with it. ...truncation, @@ -802,6 +900,10 @@ export class GroupService { // "none" (see ContractRegistry), and a value we could not read is equally // unrecorded. Reporting either as an empty list is the same conflation. unreadableRepos: recordedRepoList(registry?.unreadableRepos), + // Same tri-state, same reason: `group status` is where an operator goes + // to ask "is this group's answer trustworthy right now", and a registry + // narrowed on purpose is a different answer from a complete one. + suppressedMatchStages: recordedMatchStages(registry?.suppressedMatchStages), repos: repoStatuses, }; } diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index ab12599d5..6e82fbb0a 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -98,13 +98,8 @@ detect: http: true grpc: true topics: true - shared_libs: true - embedding_fallback: true matching: - bm25_threshold: 0.7 - embedding_threshold: 0.65 - max_candidates_per_step: 3 # exclude_links_paths: [/ping, /health, /healthcheck] # exclude_links_param_only_paths: false `; diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 992378236..7efd65ef7 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -19,6 +19,7 @@ import type { StoredContract, CrossLink, GroupManifestLink, + MatchType, } from './types.js'; import { HttpRouteExtractor } from './extractors/http-route-extractor.js'; import { GrpcExtractor } from './extractors/grpc-extractor.js'; @@ -28,10 +29,15 @@ import { IncludeExtractor } from './extractors/include-extractor.js'; import { ManifestExtractor } from './extractors/manifest-extractor.js'; import { discoverWorkspaceLinks } from './extractors/workspace-extractor.js'; import { buildProviderIndex, runExactMatch, runWildcardMatch } from './matching.js'; +import type { WildcardMatchResult } from './matching.js'; import { detectServiceBoundaries, assignService } from './service-boundary-detector.js'; import type { CypherExecutor } from './contract-extractor.js'; import { getContractRegistryPath, readContractRegistry, writeContractRegistry } from './storage.js'; -import { refreshPreservedBridgeMeta, writeBridgeUnlocked } from './bridge-db.js'; +import { + markBridgeProvenanceUnknown, + refreshPreservedBridgeMeta, + writeBridgeUnlocked, +} from './bridge-db.js'; import { withGroupSyncLock } from './group-lock.js'; import type { ContractRegistry } from './types.js'; @@ -43,10 +49,8 @@ export interface SyncOptions { resolveRepoHandle?: (registryName: string, groupPath: string) => Promise; skipWrite?: boolean; groupDir?: string; - allowStale?: boolean; verbose?: boolean; exactOnly?: boolean; - skipEmbeddings?: boolean; } /** @@ -95,6 +99,14 @@ export interface SyncResult { */ unreadableRepos: string[]; repoSnapshots: Record; + /** + * Matching stages this run was asked to skip. Populated on EVERY outcome, + * not just `written`: the sync genuinely did skip the stage whatever + * happened to the file afterwards, and the CLI summary renders from this + * rather than re-deriving it from the caller's options. Only the persisted + * registry stamps it conditionally — see the write below. + */ + suppressedMatchStages: MatchType[]; /** * What this sync did to `contracts.json`. Callers must not announce a write * they did not get: without this, `group sync` printed "Wrote contracts.json @@ -561,7 +573,21 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis const providerIndex = buildProviderIndex(autoContracts, config.matching); const { matched, unmatched } = runExactMatch(autoContracts, providerIndex, config.matching); - const wildcard = runWildcardMatch(unmatched, providerIndex); + // `exactOnly` had the same defect this PR removes `skipEmbeddings` for: it was + // declared, threaded through the CLI and MCP, and never read, so `--exact-only` + // silently produced wildcard links anyway. It is honoured here rather than + // deleted because — unlike the never-built BM25/embedding stages — the stage it + // names does exist, so the flag describes a real choice. Contracts left + // unmatched by the exact pass stay unmatched, which is exactly what it promises. + // `remaining`, not `unmatched`: this feeds SyncResult.unmatched below, so every + // contract the exact pass could not place has to be reported as unmatched + // rather than silently dropped from the count. + const wildcard: WildcardMatchResult = opts?.exactOnly + ? { matched: [], remaining: unmatched } + : runWildcardMatch(unmatched, providerIndex); + // Measured, not assumed: `[]` says this run suppressed nothing, which is a + // different statement from a registry that never recorded the field at all. + const suppressedMatchStages: MatchType[] = opts?.exactOnly ? ['wildcard'] : []; // Dedupe cross-links. Manifest contracts participate in runExactMatch, so a // manifest-declared link can also emit a matchType:'exact' CrossLink with the @@ -583,6 +609,11 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis // not recorded, telling the operator to re-run the sync that had just // succeeded. The tri-state only works if the writer commits to it. unreadableRepos, + // Stamped only on the path that writes THIS run's contracts. The preserve + // path below re-writes `{ ...prior }`, so a carried-forward registry keeps + // the marker of the sync that actually produced its contracts instead of + // being relabelled with this run's request. + suppressedMatchStages, contracts: allContracts, crossLinks, }; @@ -765,6 +796,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis repoSnapshots, missingRepos, unreadableRepos, + suppressedMatchStages, }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -777,11 +809,23 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis // true is the one thing not to do — it would recreate exactly the // metadata/database mis-pairing the stamping on the preserve path above // exists to prevent. + // Fail the stale bridge CLOSED. contracts.json has advanced and the + // database has not, so the two describe different syncs; leaving the + // bridge vouching for itself lets `group_impact` call a superseded + // graph complete while `group_contracts` reports the new one. This + // withdraws the claim without touching the database — the previous + // graph stays queryable, it just stops being called complete. + const withdrawn = await markBridgeProvenanceUnknown(groupDir); + const provenanceNote = withdrawn + ? 'Its metadata has been marked provenance-unknown, so those answers now report as ' + + 'a lower bound rather than as complete.' + : 'Its metadata could NOT be marked provenance-unknown, so those answers may still ' + + 'report as complete despite describing an older sync.'; logger.warn( - { err: msg, groupDir }, + { err: msg, groupDir, bridgeProvenanceWithdrawn: withdrawn }, '⚠️ writeBridge failed; contracts.json is intact and is the canonical copy, ' + 'but bridge.lbug was not replaced: cross-repo queries may still answer from ' + - "the previous sync's contracts, and nothing marks them as superseded. " + + `the previous sync's contracts. ${provenanceNote} ` + 'Re-run `gitnexus group sync` to retry.', ); } @@ -792,6 +836,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis return { contracts: allContracts, crossLinks, + suppressedMatchStages, unmatched: wildcard.remaining, missingRepos, unreadableRepos, diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index ee6b7cda5..cf7191664 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,5 +1,5 @@ export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include'; -export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding'; +export type MatchType = 'exact' | 'manifest' | 'wildcard'; export type ContractRole = 'provider' | 'consumer'; export interface GroupConfig { @@ -26,16 +26,11 @@ export interface DetectConfig { grpc: boolean; thrift: boolean; topics: boolean; - shared_libs: boolean; - embedding_fallback: boolean; includes: boolean; workspace_deps: boolean; } export interface MatchingConfig { - bm25_threshold: number; - embedding_threshold: number; - max_candidates_per_step: number; /** * HTTP paths to exclude from cross-link matching. Contracts at these paths * are still extracted and visible in the registry, but they don't produce @@ -114,6 +109,20 @@ export interface ContractRegistry { * absent means "not recorded", not "none". */ unreadableRepos?: string[]; + /** + * Matching stages this sync was ASKED to skip, so a later reader can tell a + * short cross-link list from a complete one. `--exact-only` / `exactOnly` + * suppresses the wildcard stage, and the registry it writes is otherwise + * indistinguishable from one where that stage ran and matched nothing. + * + * Same tri-state as `unreadableRepos` and for the same reason: absent means + * "not recorded" (written before this field existed), `[]` means "measured, + * nothing was suppressed", and a populated list names the stages. Distinct + * from `truncated` / `truncationReason`, which report limits this run hit by + * accident — a suppressed stage is a deliberate request, and its remedy is + * "re-sync without exactOnly", not "fix the unreadable repo". + */ + suppressedMatchStages?: MatchType[]; contracts: StoredContract[]; crossLinks: CrossLink[]; } @@ -137,6 +146,13 @@ export interface RepoHandle { * a retry. `'incomplete-sync'` is structural: the bridge itself was built from a * sync that could not read every configured repo, so those repos' contracts are * absent from every query against it until `gitnexus group sync` succeeds. + * `'suppressed-stage'` is structural too but has its own remedy: the sync was + * ASKED to skip a matching stage (`--exact-only`), so cross-links that stage + * would have found are absent by request. Retrying returns the same floor, and + * so does re-running the sync — the fix is to re-run it WITHOUT the flag. Kept a + * separate member rather than folded into `'incomplete-sync'` precisely because + * that remedy differs; telling an agent to repair a repo it read fine is the + * failure this distinction exists to prevent. * * A runtime array rather than a bare type union: every value here has to be * explained on the agent-facing surface that returns it, and only an enumerable @@ -145,7 +161,12 @@ export interface RepoHandle { * catch, so the list an agent is promised and the list the code can emit have * to come from the same place. */ -export const GROUP_IMPACT_TRUNCATION_REASONS = ['timeout', 'partial', 'incomplete-sync'] as const; +export const GROUP_IMPACT_TRUNCATION_REASONS = [ + 'timeout', + 'partial', + 'incomplete-sync', + 'suppressed-stage', +] as const; export type GroupImpactTruncationReason = (typeof GROUP_IMPACT_TRUNCATION_REASONS)[number]; @@ -357,4 +378,12 @@ export interface BridgeMeta { * Optional: a bridge written before this field existed does not record it. */ unreadableRepos?: string[]; + /** + * Matching stages the sync that built this bridge was asked to skip. + * PERSISTED, like `unreadableRepos` and unlike `repoListsUnreadable` — a + * later `group_impact` or `trace` reads this bridge with no access to the run + * that produced it, and a narrowed graph is otherwise indistinguishable from + * a complete one. Same tri-state: absent is "not recorded". + */ + suppressedMatchStages?: MatchType[]; } diff --git a/gitnexus/src/mcp/resources.ts b/gitnexus/src/mcp/resources.ts index d58a13a79..3411878bd 100644 --- a/gitnexus/src/mcp/resources.ts +++ b/gitnexus/src/mcp/resources.ts @@ -112,7 +112,11 @@ export function getResourceTemplates(): ResourceTemplate[] { 'three-state, and an ABSENT key is not an empty one: absent means the last sync never ' + 'recorded which repos it could read (provenance unknown — treat cross-repo answers for ' + 'this group as a floor), an empty list means the sync measured none, and a populated list ' + - 'names the repos whose contracts are missing from the registry.', + 'names the repos whose contracts are missing from the registry. suppressedMatchStages is ' + + 'three-state the same way: absent is a registry predating the field, an empty list means ' + + 'the sync skipped no matching stage, and a populated list names stages it was ASKED to ' + + 'skip — those cross-link counts are a lower bound by request, and the remedy is to re-sync ' + + 'without that flag rather than to repair a repo.', mimeType: 'text/yaml', }, ]; diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 7fea547a5..e7e453a1b 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -505,7 +505,7 @@ Handles disambiguation: when multiple symbols share the target name, returns ran EdgeType: CALLS, IMPORTS, EXTENDS, IMPLEMENTS, HAS_METHOD, HAS_PROPERTY, METHOD_OVERRIDES, METHOD_IMPLEMENTS, ACCESSES Confidence: 1.0 = certain, <0.8 = fuzzy match -GROUP MODE: set "repo" to "@" for cross-repo impact anchored at the default member (lexicographically first key in group.yaml "repos"), or "@/" to choose the member (same path keys as in group.yaml). Phase-1 walk runs in that member; cross-boundary fan-out uses the group bridge. A cross entry with fanout_status:"not_attempted" proves the declared repository boundary, but its far endpoint has no graph symbol; do not interpret empty by_depth or affected_processes on that entry as a completed zero-impact walk. The fan-out attempts at most 50 neighbour crossings, strongest-confidence first. Any short answer carries truncated:true, truncatedRepos, riskEpistemic:"lower-bound" AND a truncationReason — dropping a crossing can only move risk DOWN, so treat that risk as a floor, never as a verdict. truncated:true does NOT always mean the fan-out ran out of room, so branch on truncationReason: the remedy differs. 'timeout' (the fan-out's wall-clock budget expired) and 'partial' (a neighbour crossing, or the local walk, was cut short) are runtime limits — the same query can return more on a retry or with a larger timeoutMs. 'incomplete-sync' is structural: the group bridge was built by a sync that could not say which repos it read, or that could not read an in-scope repo, so those repos' contracts are absent from EVERY query against this bridge, and truncatedRepos names them even when ZERO crossings to them were attempted. Retrying returns the same floor — run group_sync (\`gitnexus group sync\`) and query again. +GROUP MODE: set "repo" to "@" for cross-repo impact anchored at the default member (lexicographically first key in group.yaml "repos"), or "@/" to choose the member (same path keys as in group.yaml). Phase-1 walk runs in that member; cross-boundary fan-out uses the group bridge. A cross entry with fanout_status:"not_attempted" proves the declared repository boundary, but its far endpoint has no graph symbol; do not interpret empty by_depth or affected_processes on that entry as a completed zero-impact walk. The fan-out attempts at most 50 neighbour crossings, strongest-confidence first. Any short answer carries truncated:true, truncatedRepos, riskEpistemic:"lower-bound" AND a truncationReason — dropping a crossing can only move risk DOWN, so treat that risk as a floor, never as a verdict. truncated:true does NOT always mean the fan-out ran out of room, so branch on truncationReason: the remedy differs. 'timeout' (the fan-out's wall-clock budget expired) and 'partial' (a neighbour crossing, or the local walk, was cut short) are runtime limits — the same query can return more on a retry or with a larger timeoutMs. 'incomplete-sync' is structural: the group bridge was built by a sync that could not say which repos it read, or that could not read an in-scope repo, so those repos' contracts are absent from EVERY query against this bridge, and truncatedRepos names them even when ZERO crossings to them were attempted. Retrying returns the same floor — run group_sync (\`gitnexus group sync\`) and query again. 'suppressed-stage' is also structural but has a DIFFERENT remedy: the sync was asked to skip a matching stage (\`--exact-only\` / exactOnly), so cross-links that stage would have found are absent BY REQUEST. Re-running the sync unchanged returns the same floor — re-run it WITHOUT that flag. Do not report a repo as broken for this reason; nothing failed to read. SERVICE: optional monorepo path prefix (case-sensitive path segments). When "repo" starts with "@", scopes the local impact walk and cross-repo symbol paths to files under that prefix; ignored for a normal indexed repo name.`, annotations: READ_ONLY_TOOL_ANNOTATIONS, @@ -850,11 +850,11 @@ WHEN TO USE: Discover groups before group_sync. Optional "name" returns a single }, { name: 'group_sync', - description: `Rebuild the Contract Registry (contracts.json) for a group: extract HTTP contracts, apply manifest links, exact-match cross-links. + description: `Rebuild the Contract Registry (contracts.json) for a group: extract contracts (HTTP, gRPC, Thrift, topics, includes), apply manifest links, then cross-link by exact contract-id match followed by wildcard service match. WHEN TO USE: After changing group.yaml or re-indexing member repos. -READ THE RESULT: \`missingRepos\` are configured repos with no entry in the registry (index them, or drop them from group.yaml); \`unreadableRepos\` ARE registered but this sync could not extract from them — the index would not open (version skew, lock, corruption), or an extractor failed partway — so NONE of their contracts are in this sync and a following group_impact / group_contracts is a lower bound, not a verdict. \`registryOutcome\` says what happened to the file, and the three values a call here can return each need a different response: 'written' — this run's contracts replaced contracts.json; 'preserved' — nothing could be read, so contracts.json was rewritten keeping the previous sync's contracts and cross-links verbatim and refreshing only \`missingRepos\`/\`unreadableRepos\` to describe THIS run (the file changed, the contracts in it did not, and they are as old as the last sync that succeeded); 'superseded' — nothing could be read, and another sync replaced contracts.json while this one waited for the group lock; that file was left untouched and this run's lists were NOT recorded, because they describe an older group state than what is on disk (so the registry is fresher than this response's diagnostics, not staler); 'no-prior-registry' — nothing could be read AND there was no previous contracts.json to carry forward, so none was written and this group has no contract registry on disk. Only 'no-prior-registry' means there is nothing to read: after it, group_contracts / group_impact have no registry at all rather than a stale one, so fix the repos above and re-run before trusting either.`, +READ THE RESULT: \`missingRepos\` are configured repos with no entry in the registry (index them, or drop them from group.yaml); \`unreadableRepos\` ARE registered but this sync could not extract from them — the index would not open (version skew, lock, corruption), or an extractor failed partway — so NONE of their contracts are in this sync and a following group_impact / group_contracts is a lower bound, not a verdict. \`registryOutcome\` says what happened to the file, and the three values a call here can return each need a different response: 'written' — this run's contracts replaced contracts.json; 'preserved' — nothing could be read, so contracts.json was rewritten keeping the previous sync's contracts and cross-links verbatim and refreshing only \`missingRepos\`/\`unreadableRepos\` to describe THIS run (the file changed, the contracts in it did not, and they are as old as the last sync that succeeded); 'superseded' — nothing could be read, and another sync replaced contracts.json while this one waited for the group lock; that file was left untouched and this run's lists were NOT recorded, because they describe an older group state than what is on disk (so the registry is fresher than this response's diagnostics, not staler); 'no-prior-registry' — nothing could be read AND there was no previous contracts.json to carry forward, so none was written and this group has no contract registry on disk. Only 'no-prior-registry' means there is nothing to read: after it, group_contracts / group_impact have no registry at all rather than a stale one, so fix the repos above and re-run before trusting either. \`suppressedMatchStages\` names matching stages this sync was ASKED to skip, with the same three states as the repo lists: ABSENT means a registry written before the field existed, \`[]\` means this sync suppressed nothing, and a populated list means the cross-link set is a lower bound BY REQUEST — a later group_impact / group_contracts on it reports truncationReason 'suppressed-stage'.\n\nPARAMETERS ARE VALIDATED: \`exactOnly\` must be a real boolean — the string "false" is rejected, not coerced to true. The retired \`skipEmbeddings\` and \`allowStale\` parameters are refused by name; drop them from the call.`, // Usually writes contracts.json, so conservatively non-idempotent even // though output is deterministic for identical input. When no configured // repo could be read it still rewrites the file, keeping the previous @@ -866,11 +866,11 @@ READ THE RESULT: \`missingRepos\` are configured repos with no entry in the regi type: 'object', properties: { name: { type: 'string', description: 'Group name' }, - skipEmbeddings: { + exactOnly: { type: 'boolean', - description: 'Exact + BM25 only (Demo PR: same as default exact path)', + description: + 'Skip the wildcard service-match stage; cross-link only on exact contract-id match. Manifest links still apply.', }, - exactOnly: { type: 'boolean', description: 'Exact match only in cascade' }, }, required: ['name'], }, diff --git a/gitnexus/test/integration/group/group-cli.test.ts b/gitnexus/test/integration/group/group-cli.test.ts index 8e51ce1c5..a9c1840cf 100644 --- a/gitnexus/test/integration/group/group-cli.test.ts +++ b/gitnexus/test/integration/group/group-cli.test.ts @@ -71,6 +71,41 @@ describe('group CLI', () => { expect(source).not.toMatch(blanketClosePattern); }); + /** + * `--skip-embeddings` and `--allow-stale` were both accepted by commander and + * then read by nothing: the first named a BM25/embedding cascade that was + * never built, the second a stale-index warning that no sync path ever + * emitted. An operator who passed either got a silent no-op and a clean exit, + * which is worse than the flag not existing — so they are gone, and the CLI + * must now say so. + * + * `unknown option` is asserted rather than just a nonzero exit because + * `group sync ` ALSO exits nonzero (GroupNotFoundError), so + * the exit code alone cannot tell "the flag is rejected" from "the group is + * not there". The control below is what makes that distinction visible. + */ + it('test_sync_rejects_removed_skip_embeddings_flag', () => { + const r = runGroup(['sync', 'acme', '--skip-embeddings']); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain("unknown option '--skip-embeddings'"); + }); + + it('test_sync_rejects_removed_allow_stale_flag', () => { + const r = runGroup(['sync', 'acme', '--allow-stale']); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain("unknown option '--allow-stale'"); + }); + + it('control: the surviving --exact-only flag is still parsed', () => { + // Without this, the two cases above would also pass against a `group sync` + // that rejected EVERY option. This one reaches the action handler and + // fails on the group instead, which is the proof that commander accepted + // the flag itself. + const r = runGroup(['sync', 'no-such-group', '--exact-only']); + expect(r.stderr).not.toContain('unknown option'); + expect(`${r.stderr}\n${r.stdout}`).toContain('no-such-group'); + }); + it('group impact requires --target and --repo', () => { const c = runGroup(['create', 'impcli']); expect(c.status).toBe(0); @@ -441,6 +476,69 @@ describe('group sync says what it did to contracts.json', () => { expect(fs.existsSync(path.join(groupDir, 'contracts.json'))).toBe(true); }); + /** + * The per-stage `Matching:` block. It used to print `Matching cascade:` and + * count `exact` alone, while the `Wrote contracts.json (…)` line beneath it + * reported every cross-link — so for any group with manifest or wildcard + * links the two numbers disagreed with nothing on screen explaining why. + * + * A manifest fixture is enough to pin both halves. The stage counts have to + * sum to the printed total, and the skipped rendering does not depend on a + * stage having matched anything: `--exact-only` records the suppression + * whatever the fixture contains. + */ + const writeManifestGroup = (name: string): void => { + writeGroupYaml( + home, + name, + { 'app/backend': `${name}-backend`, 'app/frontend': `${name}-frontend` }, + ` + - from: app/frontend + to: app/backend + type: custom + contract: rotateSigningKey + role: consumer`, + ); + fs.writeFileSync(path.join(home, 'registry.json'), '[]', 'utf8'); + }; + + it('prints a count for every matching stage, and they sum to the written total', () => { + writeManifestGroup('stages'); + + const r = runGroupIn(home, ['sync', 'stages']); + + expect(r.status).toBe(0); + expect(r.stdout).toContain('exact: 0 cross-links (confidence 1.0)'); + expect(r.stdout).toContain('manifest: 1 cross-links'); + expect(r.stdout).toContain('wildcard: 0 cross-links'); + // The reconciliation this block exists for: 0 + 1 + 0 is the total below. + expect(r.stdout).toContain('Wrote contracts.json (2 contracts, 1 cross-links)'); + }); + + it('names a stage it was told to skip as skipped, not as zero', () => { + writeManifestGroup('skipped'); + + const r = runGroupIn(home, ['sync', 'skipped', '--exact-only']); + + expect(r.status).toBe(0); + expect(r.stdout).toContain('wildcard: skipped (--exact-only)'); + // "ran and matched nothing" must not be printable for a stage that never ran. + expect(r.stdout).not.toContain('wildcard: 0 cross-links'); + // Manifest links are unaffected by the flag, so the total still says so. + expect(r.stdout).toContain('manifest: 1 cross-links'); + }); + + // control: the skipped rendering tracks the flag, not the fixture. Without + // this, printing `skipped` unconditionally would pass the case above. + it('control: the same group without the flag reports the stage as zero', () => { + writeManifestGroup('unskipped'); + + const r = runGroupIn(home, ['sync', 'unskipped']); + + expect(r.stdout).toContain('wildcard: 0 cross-links'); + expect(r.stdout).not.toContain('skipped (--exact-only)'); + }); + it('says the previous contracts.json was KEPT when no repo could be read', () => { // "Did NOT write contracts.json" was false here: this path REWRITES the // file, keeping the previous sync's contracts and replacing only the two diff --git a/gitnexus/test/integration/group/group-sync-lock-concurrency.test.ts b/gitnexus/test/integration/group/group-sync-lock-concurrency.test.ts index fff0dd746..84a90c55d 100644 --- a/gitnexus/test/integration/group/group-sync-lock-concurrency.test.ts +++ b/gitnexus/test/integration/group/group-sync-lock-concurrency.test.ts @@ -51,12 +51,10 @@ const makeConfig = (name: string): GroupConfig => ({ grpc: false, thrift: false, topics: false, - shared_libs: false, includes: false, workspace_deps: false, - embedding_fallback: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }); const parentContract: StoredContract = { diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index 22bb2ad26..2ede066f4 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -59,11 +59,55 @@ repos: expect(config.links).toEqual([]); expect(config.packages).toEqual({}); expect(config.detect.http).toBe(true); - expect(config.matching.bm25_threshold).toBe(0.7); expect(config.matching.exclude_links_paths).toEqual([]); expect(config.matching.exclude_links_param_only_paths).toBe(false); }); + it('still parses a legacy config carrying the removed matching knobs', () => { + // `bm25_threshold`, `embedding_threshold` and `detect.embedding_fallback` + // were written into every generated group.yaml but read by no matcher, so + // they are gone from the schema and the template. Every group.yaml already + // on disk still has them, and must keep loading without complaint. + const legacy = ` +version: 1 +name: test +repos: + app: my-app +detect: + http: true + embedding_fallback: true + shared_libs: true +matching: + bm25_threshold: 0.7 + embedding_threshold: 0.65 + max_candidates_per_step: 3 +`; + const config = parseGroupConfig(legacy); + expect(config.name).toBe('test'); + expect(config.repos).toEqual({ app: 'my-app' }); + expect(config.detect.http).toBe(true); + + // Pinned behavior: PRESERVE, not strip. The parser spreads the raw block + // over its defaults (`{ ...DEFAULT_MATCHING, ...raw.matching }`), so a key + // it no longer knows about survives into the returned config. + // + // Every assertion above is satisfied by the defaults alone, so without this + // the test only proves "does not throw" — it would stay green under a + // future strict validator that silently DROPPED the operator's legacy keys. + // That is not a harmless drop: `group add` and `group remove` in + // gitnexus/src/cli/group.ts round-trip the file through `loadGroupConfig` + // → `yaml.dump` → write, so anything the parser discards is deleted from + // the operator's checked-in group.yaml the next time they add a repo. + expect((config.matching as unknown as Record).bm25_threshold).toBe(0.7); + expect((config.matching as unknown as Record).embedding_threshold).toBe(0.65); + expect((config.detect as unknown as Record).embedding_fallback).toBe(true); + // The two keys this commit removes, pinned the same way and for the same + // reason: an operator's group.yaml carries them today because + // `gitnexus group create` wrote them there. + expect((config.matching as unknown as Record).max_candidates_per_step).toBe(3); + expect((config.detect as unknown as Record).shared_libs).toBe(true); + }); + it('defaults thrift detection to true', () => { const minimal = ` version: 1 diff --git a/gitnexus/test/unit/group/fixtures.ts b/gitnexus/test/unit/group/fixtures.ts index ad28a7166..a496c332a 100644 --- a/gitnexus/test/unit/group/fixtures.ts +++ b/gitnexus/test/unit/group/fixtures.ts @@ -35,6 +35,41 @@ export function makeContract(overrides: Partial = {}): StoredCon }; } +/** + * The one contract shape `runWildcardMatch` fires on: a thrift service-wildcard + * consumer and a matching method-level provider. `runExactMatch` skips wildcard + * consumers, so this pair links through the wildcard stage or through no stage + * at all — which is what makes it the fixture for anything testing that stage + * being run or skipped. + * + * Shared because two suites need exactly this pair; if the predicate in + * `isServiceWildcard` ever changes, it changes here once. + */ +export function makeWildcardPair(): { provider: StoredContract; consumer: StoredContract } { + return { + provider: makeContract({ + contractId: 'thrift::billing.v1.OrderService/PlaceOrder', + type: 'thrift', + role: 'provider', + symbolUid: 'uid-provider-place-order', + symbolRef: { filePath: 'src/provider.ts', name: 'OrderService.PlaceOrder' }, + symbolName: 'OrderService.PlaceOrder', + confidence: 0.9, + repo: 'app/provider', + }), + consumer: makeContract({ + contractId: 'thrift::OrderService/*', + type: 'thrift', + role: 'consumer', + symbolUid: 'uid-consumer-order-service', + symbolRef: { filePath: 'src/consumer.ts', name: 'callOrderService' }, + symbolName: 'callOrderService', + confidence: 0.9, + repo: 'app/consumer', + }), + }; +} + /** * Write the `waveful` group's `group.yaml` into `groupDir` (creating it), with * every detector disabled so a suite's own bridge rows are the only thing that diff --git a/gitnexus/test/unit/group/group-tools.test.ts b/gitnexus/test/unit/group/group-tools.test.ts index 85ba45f0a..a5412e198 100644 --- a/gitnexus/test/unit/group/group-tools.test.ts +++ b/gitnexus/test/unit/group/group-tools.test.ts @@ -18,4 +18,23 @@ describe('Group MCP tools', () => { const tool = GITNEXUS_TOOLS.find((t) => t.name === 'group_sync')!; expect(tool.inputSchema.required).toContain('name'); }); + + it('group_sync no longer advertises skipEmbeddings, and still advertises exactOnly', () => { + // `skipEmbeddings` named a BM25/embedding cascade that was never built — + // the handler read the parameter and every value took the same code path, + // so the schema advertised a choice an agent could not actually make. It is + // gone from `SyncOptions`, the CLI and here. + // + // `exactOnly` is asserted in the same test on purpose: it is the parameter + // NEXT TO the deleted one, it survived, and it now does what it says (see + // sync-exact-only.test.ts). Pinning only the absence would stay green if a + // later edit removed the wrong one of the two. + const tool = GITNEXUS_TOOLS.find((t) => t.name === 'group_sync')!; + expect(tool.inputSchema.properties).not.toHaveProperty('skipEmbeddings'); + // `verbose` gates diagnostics on the server's logger, which an MCP caller + // cannot observe. Advertising it would promise a knob whose effect is + // invisible to the only audience that reads this schema. + expect(tool.inputSchema.properties).not.toHaveProperty('verbose'); + expect(tool.inputSchema.properties.exactOnly).toMatchObject({ type: 'boolean' }); + }); }); diff --git a/gitnexus/test/unit/group/matching.test.ts b/gitnexus/test/unit/group/matching.test.ts index 6e5b80786..a10f1d017 100644 --- a/gitnexus/test/unit/group/matching.test.ts +++ b/gitnexus/test/unit/group/matching.test.ts @@ -636,9 +636,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('exclude_links_paths prevents cross-links for configured paths', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: ['/ping'], exclude_links_param_only_paths: false, }; @@ -659,9 +656,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('excluded providers do not appear in matched', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: ['/health'], exclude_links_param_only_paths: false, }; @@ -679,9 +673,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('excluded contracts do not appear in unmatched', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: ['/ping'], exclude_links_param_only_paths: false, }; @@ -700,9 +691,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('exclude_links_param_only_paths filters /{param} and /{param}/{param}', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: [], exclude_links_param_only_paths: true, }; @@ -723,9 +711,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('mixed routes like /users/{param} are NOT excluded by param_only', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: [], exclude_links_param_only_paths: true, }; @@ -757,9 +742,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('trailing slash on contractId still matches configured exclusion', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: ['/ping'], exclude_links_param_only_paths: false, }; @@ -778,9 +760,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('root path exclusion ["/"] suppresses http::GET::/ contracts', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: ['/'], exclude_links_param_only_paths: false, }; @@ -802,9 +781,6 @@ describe('buildNoisyContractFilter (via runExactMatch)', () => { it('non-HTTP contracts are never filtered', () => { const matchingConfig: MatchingConfig = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: ['/ping'], exclude_links_param_only_paths: true, }; diff --git a/gitnexus/test/unit/group/registry-suppressed-stages.test.ts b/gitnexus/test/unit/group/registry-suppressed-stages.test.ts new file mode 100644 index 000000000..88a9ecea5 --- /dev/null +++ b/gitnexus/test/unit/group/registry-suppressed-stages.test.ts @@ -0,0 +1,227 @@ +/** + * `suppressedMatchStages` — what a sync says about the stages it was told to skip. + * + * `--exact-only` / `exactOnly` suppresses the wildcard matching stage. The + * registry it writes is otherwise indistinguishable from one where that stage + * ran and matched nothing, and `group_impact` / cross-repo `trace` read that + * registry as authoritative. So the sync has to say so. + * + * The tri-state is the same one `unreadableRepos` uses, and the reason is the + * same: ABSENT means a registry written before the field existed and therefore + * has no opinion; EMPTY is a measurement — this run suppressed nothing; + * POPULATED names the stages. Normalizing absent to `[]` would report an + * unmeasured registry as a clean one. + * + * Two properties here are easy to get wrong and are pinned deliberately: + * the returned result carries the marker on EVERY outcome (the sync really did + * skip the stage whatever happened to the file), while the persisted registry + * stamps it only on the outcome that writes this run's contracts. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import { makeWildcardPair } from './fixtures.js'; +import { + crossRepoCompleteness, + type CrossRepoCompleteness, +} from '../../../src/core/group/completeness.js'; +import { GROUP_IMPACT_TRUNCATION_REASONS } from '../../../src/core/group/types.js'; +import type { GroupConfig, ContractRegistry } from '../../../src/core/group/types.js'; + +const config: GroupConfig = { + version: 1, + name: 'suppressed', + description: '', + repos: { 'app/provider': 'provider-repo', 'app/consumer': 'consumer-repo' }, + links: [], + packages: {}, + detect: { + http: true, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: false, + }, + matching: { + exclude_links_paths: [], + exclude_links_param_only_paths: false, + }, +}; + +const { provider, consumer } = makeWildcardPair(); + +let groupDir: string; + +beforeEach(() => { + groupDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-suppressed-')); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(groupDir, { recursive: true, force: true }); +}); + +const run = (exactOnly: boolean, opts: { write: boolean } = { write: false }) => + syncGroup(config, { + extractorOverride: async () => [provider, consumer], + exactOnly, + ...(opts.write ? { groupDir } : { skipWrite: true }), + }); + +const readRegistry = (): ContractRegistry => + JSON.parse(fs.readFileSync(path.join(groupDir, 'contracts.json'), 'utf8')) as ContractRegistry; + +/** + * `CrossRepoCompleteness` is a discriminated union: `truncationReason` and + * `riskEpistemic` exist only on the `truncated: true` arm, so reading them off + * the union directly does not type-check. These read them positionally, the + * same way this suite reads a preserved key its type no longer carries. + */ +const fieldOf = (out: CrossRepoCompleteness, key: string): unknown => + (out as unknown as Record)[key]; +const reasonOf = (out: CrossRepoCompleteness): unknown => fieldOf(out, 'truncationReason'); + +describe('a sync records the matching stages it was told to skip', () => { + it('names the wildcard stage when exactOnly suppressed it', async () => { + const result = await run(true); + + expect(result.suppressedMatchStages).toEqual(['wildcard']); + expect(result.crossLinks).toEqual([]); + }); + + // control: the marker tracks the request, not a constant. Without this, a + // hardcoded `['wildcard']` would pass the case above. + it('control: measures an empty list when no stage was suppressed', async () => { + const result = await run(false); + + expect(result.suppressedMatchStages).toEqual([]); + expect(result.crossLinks).toHaveLength(1); + expect(result.crossLinks[0].matchType).toBe('wildcard'); + }); + + it('persists the marker into contracts.json on a written sync', async () => { + await run(true, { write: true }); + + expect(readRegistry().suppressedMatchStages).toEqual(['wildcard']); + }); + + it('persists an empty measurement, not an absent key, on an unsuppressed sync', async () => { + await run(false, { write: true }); + + const registry = readRegistry(); + expect(registry.suppressedMatchStages).toEqual([]); + // The distinction the tri-state exists for: a measured zero is not silence. + expect(registry).toHaveProperty('suppressedMatchStages'); + }); +}); + +/** + * The half that matters to a later reader: does a narrowed graph still claim + * to be complete? `crossRepoCompleteness` is the ONE computation behind the + * truncation triple that `group_impact`, cross-repo `trace` and the contract + * listing all return, so pinning it here covers all three. + */ +describe('cross-repo completeness reflects a suppressed stage', () => { + it('reports a floor, with its own reason, when a stage was suppressed', () => { + const out = crossRepoCompleteness({ + unreadableRepos: [], + missingRepos: [], + suppressedMatchStages: ['wildcard'], + provenanceUnknown: false, + inScope: () => true, + }); + + expect(out.truncated).toBe(true); + expect(reasonOf(out)).toBe('suppressed-stage'); + expect(fieldOf(out, 'riskEpistemic')).toBe('lower-bound'); + }); + + // control: without a suppressed stage the same clean input is complete. + // Without this, hardcoding `truncated: true` would pass the case above. + it('control: a clean sync with nothing suppressed is not truncated', () => { + const out = crossRepoCompleteness({ + unreadableRepos: [], + missingRepos: [], + suppressedMatchStages: [], + provenanceUnknown: false, + inScope: () => true, + }); + + expect(out.truncated).toBe(false); + expect(reasonOf(out)).toBeUndefined(); + }); + + // An unreadable repo is the more serious gap and its remedy differs, so it + // has to win the reason slot rather than being masked by the flag. + it('lets an unreadable repo outrank a suppressed stage in the reason', () => { + const out = crossRepoCompleteness({ + unreadableRepos: ['app/backend'], + missingRepos: [], + suppressedMatchStages: ['wildcard'], + provenanceUnknown: false, + inScope: () => true, + }); + + expect(out.truncated).toBe(true); + expect(reasonOf(out)).toBe('incomplete-sync'); + }); +}); + +/** + * The reason must stay a DECLARED member of the union agents branch on. + * + * That it is reachable from real code is already pinned above, by a case that + * drives `crossRepoCompleteness` and gets the value back. What that cannot see + * is the union itself shrinking: a guard in `tools.test.ts` asserts every + * member is documented, so dropping a member keeps that guard green while every + * consumer silently loses the value. + */ +describe('the suppressed-stage reason is reachable, not just documented', () => { + it('is a declared member of the reason union agents branch on', () => { + // If a future change drops it from the union, the tool description guard + // would still pass while every consumer lost the value. + expect(GROUP_IMPACT_TRUNCATION_REASONS).toContain('suppressed-stage'); + }); +}); + +/** + * An UNREADABLE suppression record must not read as "nothing was suppressed". + * + * `recordedMatchStages` is all-or-nothing on purpose: garbage collapses to + * `undefined`. A consumer that then treats `undefined` as an empty measurement + * throws that safety away and reports a registry it could not parse as + * complete. Absent stays legitimate — a registry written before the field + * existed has no opinion and should not be forced to a floor. + */ +describe('an unreadable suppression record fails closed', () => { + it('does not report a registry it could not parse as complete', () => { + const garbage = crossRepoCompleteness({ + unreadableRepos: [], + missingRepos: [], + suppressedMatchStages: [], + // What `loadContractRegistryResilient` now passes when the stored value + // was present and could not be read. + provenanceUnknown: true, + inScope: () => true, + }); + + expect(garbage.truncated).toBe(true); + expect(reasonOf(garbage)).toBe('incomplete-sync'); + }); + + // control: an absent record is not an unreadable one. Without this, forcing + // every pre-existing registry to a floor would pass the case above. + it('control: a clean registry with nothing recorded stays complete', () => { + const clean = crossRepoCompleteness({ + unreadableRepos: [], + missingRepos: [], + provenanceUnknown: false, + inScope: () => true, + }); + + expect(clean.truncated).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/group/service-group-sync-payload.test.ts b/gitnexus/test/unit/group/service-group-sync-payload.test.ts index f96841d77..ddfa32d71 100644 --- a/gitnexus/test/unit/group/service-group-sync-payload.test.ts +++ b/gitnexus/test/unit/group/service-group-sync-payload.test.ts @@ -82,10 +82,20 @@ const syncResult = (overrides: Partial = {}): SyncResult => ({ missingRepos: [], unreadableRepos: [], repoSnapshots: {}, + suppressedMatchStages: [], registryOutcome: 'written', ...overrides, }); +/** + * `syncGroupMock` is declared zero-arg, so `mock.calls` is typed as an array of + * the empty tuple and indexing `[1]` does not type-check. The runtime call + * genuinely has two arguments (config, options); this reads the second without + * restating a signature the rest of the suite does not need. + */ +const syncOptsOf = (call: number): Record => + (syncGroupMock.mock.calls[call] as unknown as unknown[])[1] as Record; + const CONTRACT = makeContract({ repo: 'app/backend' }); const CROSS_LINK: CrossLink = { contractId: CONTRACT.contractId, @@ -172,6 +182,7 @@ describe('group_sync forwards what the sync learned about the repos and the file unmatched: 1, missingRepos: ['app/frontend'], unreadableRepos: ['app/backend'], + suppressedMatchStages: [], registryOutcome: 'preserved', }); }); @@ -190,6 +201,7 @@ describe('group_sync forwards what the sync learned about the repos and the file unmatched: 0, missingRepos: [], unreadableRepos: [], + suppressedMatchStages: [], registryOutcome: 'written', }); }); @@ -302,3 +314,124 @@ describe('group_contracts forwards its structured incompleteness', () => { }); }); }); + +/** + * What `group_sync` REFUSES to run on. + * + * The MCP SDK does not enforce a tool's advertised `inputSchema` and + * `callTool` is reachable directly, so this service method is the real + * validation boundary. Two consequences this block pins: + * + * - `exactOnly` now gates a matching stage, so `Boolean(params.exactOnly)` + * turned the string `"false"` — a common shape for an LLM caller emitting + * JSON — into `true` and persisted a registry with the wildcard stage + * suppressed. The opposite of what the caller asked for, written to disk. + * - `skipEmbeddings` and `allowStale` were retired. The CLI rejects them + * outright; the MCP path accepted and silently dropped them, so an agent + * working from a cached schema was told nothing. + * + * Every case asserts `syncGroupMock` was NOT called: a rejection that still + * runs the sync is the failure mode, and an error string alone cannot tell + * the two apart. + */ +describe('group_sync rejects malformed and retired parameters', () => { + it.each([['false'], ['true'], [0], [1], [null], [{}], [[]]])( + 'rejects a non-boolean exactOnly (%j) and runs no sync', + async (bad) => { + const payload = await new GroupService(port).groupSync({ name: GROUP, exactOnly: bad }); + + expect(payload).toEqual({ + error: `Invalid "exactOnly": expected true or false, got ${JSON.stringify(bad)}.`, + }); + expect(syncGroupMock).not.toHaveBeenCalled(); + }, + ); + + // `verbose` is no longer part of this tool's surface: not advertised, not + // validated, not forwarded. A caller that still sends it is ignored rather + // than refused — it was never a documented parameter, so there is nothing to + // reject on behalf of, and the retired-name guard is reserved for parameters + // this tool actually withdrew. + it('ignores verbose entirely rather than validating or forwarding it', async () => { + syncGroupMock.mockResolvedValue(syncResult()); + + const payload = (await new GroupService(port).groupSync({ + name: GROUP, + verbose: 'not-a-boolean', + })) as Record; + + expect(payload.error).toBeUndefined(); + expect(syncGroupMock).toHaveBeenCalledTimes(1); + expect(syncOptsOf(0)).not.toHaveProperty('verbose'); + }); + + // The error path must not throw. `JSON.stringify` — the right renderer here, + // because it distinguishes the string "false" from the boolean — throws on a + // BigInt and on a cyclic object, and `callTool` is reachable directly, so a + // validator that rejects instead of returning `{ error }` breaks its own + // contract on inputs a caller can actually send. + it('returns a structured error rather than throwing on an unserializable value', async () => { + const cyclic: Record = {}; + cyclic.self = cyclic; + + const fromBigInt = (await new GroupService(port).groupSync({ + name: GROUP, + exactOnly: 1n, + })) as Record; + const fromCyclic = (await new GroupService(port).groupSync({ + name: GROUP, + exactOnly: cyclic, + })) as Record; + + expect(String(fromBigInt.error)).toContain('Invalid "exactOnly"'); + expect(String(fromCyclic.error)).toContain('Invalid "exactOnly"'); + expect(syncGroupMock).not.toHaveBeenCalled(); + }); + + it.each([['skipEmbeddings'], ['allowStale']])( + 'rejects the retired %s parameter by name and runs no sync', + async (retired) => { + const payload = await new GroupService(port).groupSync({ name: GROUP, [retired]: true }); + + expect(payload).toEqual({ + error: `"${retired}" was removed and is no longer accepted. Drop it from the call.`, + }); + expect(syncGroupMock).not.toHaveBeenCalled(); + }, + ); + + it.each([[true], [false]])( + 'passes a real boolean exactOnly (%j) through unchanged', + async (ok) => { + syncGroupMock.mockResolvedValue(syncResult()); + + await new GroupService(port).groupSync({ name: GROUP, exactOnly: ok }); + + expect(syncGroupMock).toHaveBeenCalledTimes(1); + expect(syncOptsOf(0)).toMatchObject({ exactOnly: ok }); + }, + ); + + it('treats an omitted exactOnly as false', async () => { + syncGroupMock.mockResolvedValue(syncResult()); + + await new GroupService(port).groupSync({ name: GROUP }); + + expect(syncOptsOf(0)).toMatchObject({ exactOnly: false }); + }); + + // control: the guards above reject specific shapes, not every call. Without + // this, deleting the whole method body and returning an error would pass. + it('control: a valid call with only a name still syncs', async () => { + syncGroupMock.mockResolvedValue(syncResult({ registryOutcome: 'written' })); + + const payload = (await new GroupService(port).groupSync({ name: GROUP })) as Record< + string, + unknown + >; + + expect(payload.error).toBeUndefined(); + expect(payload.registryOutcome).toBe('written'); + expect(syncGroupMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/test/unit/group/sync-exact-only.test.ts b/gitnexus/test/unit/group/sync-exact-only.test.ts new file mode 100644 index 000000000..32623763f --- /dev/null +++ b/gitnexus/test/unit/group/sync-exact-only.test.ts @@ -0,0 +1,81 @@ +/** + * `--exact-only` / `exactOnly` had the same defect this PR removes + * `skipEmbeddings` for: it was declared on `SyncOptions`, threaded through the + * CLI and the MCP tool, and never read. A sync asked for exact matching only + * still ran the wildcard stage and still emitted `matchType: 'wildcard'` + * cross-links, so the flag was a promise the pipeline never kept. + * + * It is kept (rather than deleted alongside the never-built BM25/embedding + * stages) because the stage it names does exist, so the flag describes a real + * choice. These two cases run the SAME synthetic input through `syncGroup` + * twice and differ only in the flag, so the wildcard link is the only thing + * that can move between them. + */ +import { describe, it, expect } from 'vitest'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import { makeWildcardPair } from './fixtures.js'; +import type { GroupConfig } from '../../../src/core/group/types.js'; + +describe('syncGroup exactOnly gates the wildcard stage', () => { + const config: GroupConfig = { + version: 1, + name: 'test', + description: '', + repos: { 'app/provider': 'provider-repo', 'app/consumer': 'consumer-repo' }, + links: [], + packages: {}, + detect: { + http: true, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: false, + }, + matching: {}, + }; + + const { provider, consumer } = makeWildcardPair(); + + it('runs the wildcard stage when exactOnly is not set', async () => { + const result = await syncGroup(config, { + extractorOverride: async () => [provider, consumer], + skipWrite: true, + }); + + expect(result.crossLinks).toHaveLength(1); + expect(result.crossLinks[0].matchType).toBe('wildcard'); + expect(result.crossLinks[0].contractId).toBe('thrift::OrderService/*'); + expect(result.crossLinks[0].from.repo).toBe('app/consumer'); + expect(result.crossLinks[0].to.repo).toBe('app/provider'); + // The consumer was placed, so only the provider is left over. + expect(result.unmatched.map((c) => c.contractId)).toEqual([ + 'thrift::billing.v1.OrderService/PlaceOrder', + ]); + }); + + it('emits no wildcard cross-link when exactOnly is true, and still reports the contract as unmatched', async () => { + const result = await syncGroup(config, { + extractorOverride: async () => [provider, consumer], + exactOnly: true, + skipWrite: true, + }); + + expect(result.crossLinks).toEqual([]); + + // The second half of the gate, and the reason it substitutes + // `{ matched: [], remaining: unmatched }` rather than an empty result: + // `wildcard.remaining` IS `SyncResult.unmatched`. A gate that returned + // `remaining: []` would also produce zero cross-links and pass the + // assertion above, while silently deleting both contracts from the + // unmatched count an operator reads to decide whether the flag cost them + // anything. Skipping the stage must leave its input unmatched, not gone. + expect(result.unmatched.map((c) => c.contractId)).toEqual([ + 'thrift::billing.v1.OrderService/PlaceOrder', + 'thrift::OrderService/*', + ]); + // Extraction itself is untouched by the flag — both contracts are still in + // the registry, only the link between them is withheld. + expect(result.contracts).toHaveLength(2); + }); +}); diff --git a/gitnexus/test/unit/group/sync-partial-extraction.test.ts b/gitnexus/test/unit/group/sync-partial-extraction.test.ts index a8fbc2c45..06b3bf450 100644 --- a/gitnexus/test/unit/group/sync-partial-extraction.test.ts +++ b/gitnexus/test/unit/group/sync-partial-extraction.test.ts @@ -92,12 +92,10 @@ const config = (): GroupConfig => ({ grpc: true, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }); describe('syncGroup when one extractor fails partway through a repo', () => { diff --git a/gitnexus/test/unit/group/sync-unreadable-repos.test.ts b/gitnexus/test/unit/group/sync-unreadable-repos.test.ts index d5b38cfd5..12a85439c 100644 --- a/gitnexus/test/unit/group/sync-unreadable-repos.test.ts +++ b/gitnexus/test/unit/group/sync-unreadable-repos.test.ts @@ -169,12 +169,10 @@ const makeConfig = (repos: Record): GroupConfig => ({ grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }); /** @@ -331,6 +329,55 @@ describe('syncGroup with an unreadable index', () => { expect(onDisk.unreadableRepos).toEqual(['app/backend']); }); + it('keeps the prior suppressedMatchStages instead of stamping this run request', async () => { + // The preserved registry describes an EARLIER sync. If this run's request + // were stamped onto it, a graph narrowed by `--exact-only` would be + // relabelled complete the moment a later plain sync failed to read + // anything — and `group_impact` reads exactly that field to decide whether + // its answer is a floor. + initLbugMock.mockRejectedValue(new Error(LBUG_VERSION_ERROR)); + + const contractsPath = path.join(groupDir, 'contracts.json'); + fs.writeFileSync( + contractsPath, + JSON.stringify({ ...PRIOR_REGISTRY, suppressedMatchStages: ['wildcard'] }), + ); + + // This run asks for NO suppression, and fails to read anything. + const result = await syncGroup(makeConfig({ 'app/backend': 'backend-repo' }), { groupDir }); + + expect(result.registryOutcome).toBe('preserved'); + // The result describes THIS run — it really did suppress nothing. + expect(result.suppressedMatchStages).toEqual([]); + + const onDisk = JSON.parse(fs.readFileSync(contractsPath, 'utf8')) as Record; + // The file still describes the sync that produced its contracts. + expect(onDisk.suppressedMatchStages).toEqual(['wildcard']); + }); + + it('does not stamp this run request onto the preserved bridge metadata', async () => { + // Same property one artifact over. `bridge.lbug` is untouched on this path, + // so its meta.json must keep describing the sync that built it; otherwise + // contracts.json, meta.json and the database describe three different runs. + initLbugMock.mockRejectedValue(new Error(LBUG_VERSION_ERROR)); + + fs.writeFileSync(path.join(groupDir, 'contracts.json'), JSON.stringify(PRIOR_REGISTRY)); + await writeBridgeMeta(groupDir, { + version: 1, + generatedAt: '2026-01-01T00:00:00.000Z', + missingRepos: [], + unreadableRepos: [], + suppressedMatchStages: ['wildcard'], + }); + + await syncGroup(makeConfig({ 'app/backend': 'backend-repo' }), { groupDir }); + + const meta = await readBridgeMeta(groupDir); + expect(meta.suppressedMatchStages).toEqual(['wildcard']); + // ...while the diagnostics describing THIS run are refreshed, as before. + expect(meta.unreadableRepos).toEqual(['app/backend']); + }); + it('writes nothing at all when there is no previous registry to preserve', async () => { initLbugMock.mockRejectedValue(new Error(LBUG_VERSION_ERROR)); @@ -923,6 +970,57 @@ describe('the warning after a failed bridge write', () => { const bridgeWarning = (cap: ReturnType) => cap.records().find((r) => r.level === 40 && typeof r.groupDir === 'string'); + it('withdraws the old bridge provenance when the registry advanced but the bridge write failed', async () => { + // The split-brain this guards: contracts.json commits, the bridge write + // then fails, and the previous database stays in place describing an + // EARLIER sync. Left vouching for itself, `group_impact` traverses that + // older graph and calls its answer complete while `group_contracts` + // reports the new registry — two public surfaces, contradictory claims, + // out of one sync. Marking provenance unknown withdraws the completeness + // claim without deleting a graph still useful as a floor. + // `unreadableRepos` is deliberately UNREADABLE here, not merely empty: that + // is what makes `readBridgeMeta` set the reader-only `repoListsUnreadable` + // on what it returns, so the assertion below can actually catch a + // read-modify-write writer round-tripping it back to disk. Seeded with a + // valid list the check passes whether or not the strip exists. + fs.writeFileSync( + path.join(groupDir, 'meta.json'), + JSON.stringify({ + version: 1, + generatedAt: '2026-01-01T00:00:00.000Z', + missingRepos: [], + unreadableRepos: 'not-a-list', + }), + ); + expect((await readBridgeMeta(groupDir)).repoListsUnreadable).toBe(true); + expect((await readBridgeMeta(groupDir)).provenanceUnknown).toBeUndefined(); + + writeBridgeFailure = new Error('ENOSPC: no space left on device'); + await runSync(); + + expect((await readBridgeMeta(groupDir)).provenanceUnknown).toBe(true); + + // ...and the withdrawal must not persist the reader-only fields. + // `readBridgeMeta` sets both on what it returns, so a read-modify-write + // writer round-trips them unless the write boundary strips them. + // `pairedWithDatabase` is the poisonous one: persisted, it would tell every + // later reader the pair was verified when nothing verified it. + const raw = JSON.parse(fs.readFileSync(path.join(groupDir, 'meta.json'), 'utf8')) as Record< + string, + unknown + >; + expect(raw).not.toHaveProperty('pairedWithDatabase'); + expect(raw).not.toHaveProperty('repoListsUnreadable'); + }); + + // control: a sync whose bridge write SUCCEEDS must not withdraw provenance — + // otherwise every healthy sync would report its own answers as a floor. + it('control: a successful bridge write leaves provenance intact', async () => { + await runSync(); + + expect((await readBridgeMeta(groupDir)).provenanceUnknown).toBeFalsy(); + }); + it('names the registry as intact and does not promise a truncation this branch never reports', async () => { writeBridgeFailure = new Error('ENOSPC: no space left on device'); const cap = _captureLogger(); diff --git a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts index 4c44ef778..aaf361b87 100644 --- a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts +++ b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts @@ -213,12 +213,10 @@ describe('syncGroup windowed resolution bounds pool residency (real pool, #2189) grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; await syncGroup(config, { diff --git a/gitnexus/test/unit/group/sync.test.ts b/gitnexus/test/unit/group/sync.test.ts index 51140d775..68fe13c50 100644 --- a/gitnexus/test/unit/group/sync.test.ts +++ b/gitnexus/test/unit/group/sync.test.ts @@ -26,12 +26,10 @@ describe('syncGroup', () => { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }); it('returns SyncResult with contracts and cross-links', async () => { @@ -223,12 +221,10 @@ describe('syncGroup', () => { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; const result = await syncGroup(config, { @@ -678,12 +674,10 @@ service OrderService { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; const cap = _captureLogger(); @@ -749,12 +743,10 @@ service OrderService { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: workspaceDeps, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; } @@ -907,12 +899,10 @@ service OrderService { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: true, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; const result = await syncGroup(config, { @@ -999,12 +989,10 @@ service OrderService { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; const poolAdapter = await import('../../../src/core/lbug/pool-adapter.js'); @@ -1084,12 +1072,10 @@ service OrderService { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; const result = await syncGroup(config, { @@ -1129,12 +1115,10 @@ describe('syncGroup windowed manifest resolution (issue #2189 / PR #2191 review) grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; }; diff --git a/gitnexus/test/unit/group/types.test.ts b/gitnexus/test/unit/group/types.test.ts index e1ffb77d7..cfa9abba2 100644 --- a/gitnexus/test/unit/group/types.test.ts +++ b/gitnexus/test/unit/group/types.test.ts @@ -23,12 +23,10 @@ describe('Group types', () => { grpc: true, thrift: true, topics: true, - shared_libs: true, - embedding_fallback: true, includes: true, workspace_deps: true, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; expect(config.version).toBe(1); expect(config.name).toBe('company'); @@ -93,12 +91,10 @@ describe('Group types', () => { grpc: true, thrift: true, topics: true, - shared_libs: true, - embedding_fallback: true, includes: true, workspace_deps: true, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }; expect(config.detect.thrift).toBe(true); }); diff --git a/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts b/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts index d992b0083..76d79e295 100644 --- a/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts +++ b/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts @@ -68,12 +68,10 @@ describe('readRegistryStrict', () => { grpc: false, thrift: false, topics: false, - shared_libs: false, - embedding_fallback: false, includes: false, workspace_deps: false, }, - matching: { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3 }, + matching: {}, }); beforeEach(async () => { From 6088d2e309de134688cb465fc76988ce801e06c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Thu, 27 Aug 2026 23:21:40 +0100 Subject: [PATCH 11/28] chore: release v1.6.10 (#3064) * chore: release v1.6.10 * fix(eval): derive the pinned runtime version from package.json The containment suite mounts a GitNexus runtime built from this checkout and asserts its version equals PINNED_GITNEXUS_VERSION, a constant hardcoded to "1.6.9" when the harness landed in #2566. The first release after that lands 1.6.10 in gitnexus/package.json, the built runtime reports 1.6.10, and `eval / containment (ubuntu)` fails on drift the release itself created. Read the version from gitnexus/package.json instead. The check keeps its real job -- proving the mounted runtime came from this checkout rather than a published package -- without a copy that only ever drifts on release day. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Gergo Magyar Co-authored-by: Claude Opus 5 (1M context) --- .agents/plugins/marketplace.json | 2 +- .claude-plugin/marketplace.json | 2 +- eval/workflow_bench/runtime_mounts.py | 4 +- .../.claude-plugin/plugin.json | 2 +- .../.codex-plugin/plugin.json | 2 +- .../skills/gitnexus-cli/mcp.json | 2 +- .../skills/gitnexus-debugging/mcp.json | 2 +- .../skills/gitnexus-exploring/mcp.json | 2 +- .../skills/gitnexus-guide/mcp.json | 2 +- .../skills/gitnexus-impact-analysis/mcp.json | 2 +- .../skills/gitnexus-lfg/mcp.json | 2 +- .../skills/gitnexus-plan/mcp.json | 2 +- .../skills/gitnexus-refactoring/mcp.json | 2 +- .../skills/gitnexus-review/mcp.json | 2 +- .../skills/gitnexus-work/mcp.json | 2 +- gitnexus/CHANGELOG.md | 91 +++++++++++++++++++ gitnexus/package-lock.json | 4 +- gitnexus/package.json | 2 +- 18 files changed, 111 insertions(+), 18 deletions(-) diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 6494fac53..77a48e8ce 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "source": { "source": "local", "path": "./gitnexus-claude-plugin" diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 576586d48..717a4c132 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "source": "./gitnexus-claude-plugin", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase." } diff --git a/eval/workflow_bench/runtime_mounts.py b/eval/workflow_bench/runtime_mounts.py index 3f2e6fcf2..c532ac1b8 100644 --- a/eval/workflow_bench/runtime_mounts.py +++ b/eval/workflow_bench/runtime_mounts.py @@ -28,8 +28,10 @@ from .proposer_sandbox import ( SandboxError, ) -PINNED_GITNEXUS_VERSION = "1.6.9" HARNESS_ROOT = Path(__file__).resolve().parents[2] +# The mounted runtime is built from this checkout, so the pin tracks the harness' +# own package version. A hardcoded copy only drifts on release day (#3064). +PINNED_GITNEXUS_VERSION = json.loads((HARNESS_ROOT / "gitnexus" / "package.json").read_text())["version"] CE_ARMS = frozenset({"ce_workflow", "ce_workflow_direct", "ce_review"}) SANDBOX_CE_PLUGIN = "/opt/compound-engineering-plugin" diff --git a/gitnexus-claude-plugin/.claude-plugin/plugin.json b/gitnexus-claude-plugin/.claude-plugin/plugin.json index ace058dad..9e9372dea 100644 --- a/gitnexus-claude-plugin/.claude-plugin/plugin.json +++ b/gitnexus-claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.9", + "version": "1.6.10", "author": { "name": "GitNexus" }, diff --git a/gitnexus-claude-plugin/.codex-plugin/plugin.json b/gitnexus-claude-plugin/.codex-plugin/plugin.json index c9a03db4d..67ef62af2 100644 --- a/gitnexus-claude-plugin/.codex-plugin/plugin.json +++ b/gitnexus-claude-plugin/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.9", + "version": "1.6.10", "skills": "./skills", "mcpServers": "./.mcp.json", "hooks": "./hooks/hooks.json", diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index def77dce8..6b4520d63 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -4,6 +4,97 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +## [1.6.10] - 2026-08-27 + +### Added + +- **Spring framework modeling expanded end to end** — AOP transactions, caching and security (#2783), `@Bean` factories and `@Resource` injection (#2740), profiles/conditions/auto-configuration (#2678), constructor and standard injection (#2632), bean candidate inventory (#2494), configuration-property consumers, and non-HTTP handler entry points (#2891) +- **Receiver chains typed from AST structure across all 14 languages**, with an explicit epistemic lower bound on what the graph can claim (#2708, #2744, #2747) +- **Java enum constant bodies modeled as first-class instances**, with JLS 13.1 anonymous-class naming (#2558) +- **More route surfaces indexed** — Java constant-based route paths such as `@PostMapping(ApiPathConstants.X)` (#2980) and JavaScript data route tables (#2972) +- **MCP server hardening** — repository allowlist, fail-closed read-only mode, deterministic output budgets, and normalized `impact`/`context` aliases +- **`bunx` lane so bun-only machines can run GitNexus** (#2765) +- **Codex support** — hooks, plugin marketplace and setup (#2328, #2369) — plus CodeBuddy and Qoder coding-agent integrations (#2368) +- **Skills mirrored to `.agents/skills/`** when an `.agents/` directory exists +- **One-click Render deploy** (#2804) +- **`serve` origin/proxy configuration is validated and port-scoped** (#2820) +- **Expanded TypeScript/JavaScript taint sink model** (#2490) +- **Wiki generation accepts explicit HTTP LLM hosts** (#2491) +- **Embedding request-body dimensions configurable** via `GITNEXUS_EMBEDDING_REQUEST_DIMS` (#2574) +- **Refreshed MiniMax model and endpoint configuration** (#2780) +- **`MAX_CALLABLE_VALUE_TARGETS` and `MAX_PROPERTY_DISPATCH_FANOUT` configurable via env** (#2725, #2726) +- **Opt-in `analyze --self-commit`** for AGENTS.md/CLAUDE.md churn (#2640) +- **Buffer pool sized to the graph before the database opens**, with an adaptive size hint +- **CI review agent runs as a coordinated reviewer swarm** on Sonnet 5 with structured, linked reviews (#2570, #2572), alongside the GitNexus Engineering Tool Kit skills (#2566) and an online skill-evolution loop (#2571) +- **Icebug community-engine prototype behind a gate** (#2376) + +### Fixed + +- **`group sync` stops claiming matching it never did** — the advertised BM25/embedding cascade was config, help text and MCP schema with no matcher behind it; the unread `matching.bm25_threshold`, `matching.embedding_threshold`, `detect.embedding_fallback` and `--skip-embeddings` surfaces are removed (#3020) +- **Emitted Next.js build output is ignored during ingestion**, and the inert `public/build` entry is deleted (#3018) +- **NestJS decorator routes are indexed** so `api_impact` and `route_map` stop reporting live endpoints as non-existent (#3017) +- **Import resolution gated by real module configuration** instead of path-suffix guessing — TypeScript config (#2953, #2956), Java and Kotlin declared packages (#2955, #2990), Go module paths (#2984), PHP Composer autoload maps (#2987), Python `__init__.py` re-exports (#2864) and unaliased dotted namespace imports (#2826, #2828), and JavaScript module extensions (#3034) +- **Interface dispatch is generic-instantiation aware** (#2912, #2939), fans out from Case 3b receivers (#2832, #2842) and from C# record interface calls (#2904), and resolves through generic-typed field receivers in every language (#2833, #2855) +- **Go method sets modeled exactly** so interface satisfaction is decidable (#2813, #2829), out-of-repo package qualifiers resolve, and an undecided interface check is no longer reported as a decided negative (#2873, #2921) +- **Go pointer-receiver calls resolve**, reporting the program boundary instead of hedging (#2766, #2782) +- **Java record support** — graph nodes for `record_declaration`, component accessors, enum and record interface heritage (#2564, #2916, #2935, #2936), plus `E.CONST.method()` enum-constant receiver dispatch (#2561) and JLS binary-name identities for local classes, enums, records and interfaces (#2562, #2653) +- **Rust module-qualified calls resolve against the module tree** (#2730, #2741), items are qualified by their enclosing `mod` chain (#2742, #2745), duplicate type names stay ambiguous in range binding (#2514, #2652), and `Box` names normalize +- **Closure bindings are call sources in every language**, and function-local values carry their own identity (#2693, #2695, #2699, #2718) +- **A named receiver's member never resolves lexically** (#2714), platform builtins stop resolving to unrelated same-file symbols (#2549), and inline constructor receivers are typed in every spelling (#2708, #2737) +- **Python calls resolve through constructor-injected fields** (#2628) and module-imported classes (#2770) +- **Package directories that repeat higher in the path resolve correctly** (#2881, #2929) +- **`check` stops reporting erased and deferred imports as initialization cycles** (#2934) +- **`detect_changes` no longer scales its query with the diff's hunk count** (#2915, #2930), and CR-only line-ending diffs are ignored (#2839) +- **`group` stops reporting what could not be measured as a measurement of zero** (#3012), resolves HTTP consumers through configured clients and constant route tables (#3008), and preserves manifest-only impact crossings (#2784) +- **`impact` and `context` are reproducible** — deterministic ordering on every capped query (#2787, #2796) — and Convex caller results are marked incomplete rather than empty (#3044) +- **Object handler identity is preserved** during ingestion (#3046), nested source directories are discovered (#3043), and parse-node insertion is canonicalized +- **Large-repo analyze OOM and the false worker-timeout cascade are fixed** (#2649, #2679) +- **Single-writer lock on the index write path** (#2658, #2677), atomic index swap with read-pool staleness invalidation (#2614), and reliable large incremental writeback commits (#2409, #2425) +- **Remote URLs are stripped of credentials before they are persisted** (#2914, #2928), and every registry write gets its own tmp path (#2888, #2920) +- **Schema version derived from a DDL fingerprint** instead of a hand-incremented constant (#2798, #2808), and the scope-resolution relation cross product is fully declared (#2792, #2793) +- **FTS reliability** — binary payloads stay out of the description column and an unbuildable index is confined to its own table (#2919), FTS-indexed DML is gated before the incremental writeback (#2841, #2854), analyze degrades instead of aborting on index-build failure (#2548), real LOAD errors surface and broken extension files self-heal (#2374, #2375), and Windows missing-dependency load failures are diagnosed (#2383) +- **`VECTOR` is loaded only when needed** (#3045) and before the incremental writeback touches embedding rows (#2623, #2624) +- **Buffer pool bounded instead of taking the native 80%-of-RAM default** (#2560), scaled by the OS page-size granule ratio (#2631, #2636), with a COPY-safe floor and actionable diagnostics for non-4K page sizes (#2424) +- **`Napi::Error` SIGABRT on analyze eliminated** — C++ type lookups are indexed and workers terminate only at JS-safe points (#2432, #2436) +- **Native-load failures fail closed**, including truncated-binary SIGBUS (#2441, #2651), and glibc-too-old loads are no longer misdiagnosed (#2672, #2689) +- **Index staleness reporting fixed** — no false-stale status after analyze, with inline staleness in `query`/`context`/`impact`/`cypher` (#2655, #2668, #2683) +- **Windows path handling** — the `\\?\` long-path prefix no longer breaks repo path matching (#2667, #2700), `parts` negation is honored (#2720), and missing-shadow errors let `serve` repo-switch recover (#2382, #2387) +- **Embeddings survive partial failures** — unparseable 200 responses are retried (#2790, #2795), batch inserts are retry-safe (#2453), HTTP generation is resumable, resume checkpoints bind to their provider, and proxy-blocked installs self-heal (#2370, #2372) +- **Custom HTTP embedding endpoint failures are reported as themselves**, not as Hugging Face download errors (#2385, #2386) +- **Exact symbol content with 0-based line storage and 1-based MCP display** (#2377, #2379, #2380) +- **`rename` reports every edit that apply writes** and reconciles its report on partial failure (#2605) +- **Global registry transactions serialized across processes** (#2716) +- **Swift indented conditional directives are preprocessed** so class bodies survive parsing (#2771), and Swift member-containment pairs are declared in the `CONTAINS` DDL (#2769) +- **JavaScript `exports.foo = function () {}` CommonJS exports are indexed** (#2723, #2729), and `const X = () => {}` is no longer double-indexed as a Function plus an edgeless Const twin (#2687, #2691) +- **JVM sibling injection is proximity-bounded** (#2732), and C#/Kotlin free calls are gated by instance ownership (#2563, #2654) +- **Dart extension type symbols are extracted** (#2539), and declarations recover after embedded NUL bytes (#2430) +- **CLI and hooks fail loudly on backend error payloads**, with an MCP query hint when the server owns the DB lock (#2396, #2397) +- **Committed agent guides stop churning**, with an `--index-only` nudge (#2907, #2927), and `gitnexus-plan` artifacts publish on macOS without an interpreter (#2905, #2922) +- **The 300-flows cap is removed for large repositories** (#2198) + +### Changed + +- **BREAKING: Node `^22.18.0 || >=24.11.0` is now the supported floor**; the `@types/uuid` stub is dropped +- **BREAKING: the non-functional `group` matching knobs are gone** — `matching.bm25_threshold`, `matching.embedding_threshold`, `detect.embedding_fallback` in `group.yaml`, the `gitnexus group sync --skip-embeddings` flag, and the MCP `group_sync` `skipEmbeddings` argument (#3020) +- **Structural relationships are held out of the JS heap by default** during analyze (#2680, #2685) +- **Global ignore support** — `core.excludesFile`, `.git/info/exclude`, and a user-level global ignore file are honored (#2606) +- **Plugin manifests sync on every version bump** (#2445), and planning output under `docs/plans` is no longer tracked + +### Performance + +- **Import resolution indexed instead of scanned** — every scanning resolver with a consolidated memo (#2911), a per-run workspace index for Go/C#/Dart/Ruby (#2898), and Kotlin import resolution (#2872) +- **MCP server startup drops the analyze-only language-provider closure** (#2802, #2806) +- **C++ qualified namespace members indexed once per pipeline run** (#2788, #2794) +- **Vendored Leiden O(communities × N) copy removed**, with Icebug wired to its real API (#2337, #2692) +- **`core.excludesFile` / `info/exclude` resolution memoized** (#2606) + +### Chore / Dependencies + +- **`@ladybugdb/core` bumped to ^0.18.3** for the rel-property IN-predicate fix (#2508, #2634) +- **Security overrides** — `sharp` >=0.35.0 for libvips vulnerabilities (#2993) and `adm-zip` >=0.6.0 for a memory-allocation vulnerability (#2992) +- **~130 dependency bumps** across the CLI, web app and GitHub Actions, including `@modelcontextprotocol/sdk`, LangChain, Vite, Vitest, TypeScript, React and the Docker/CodeQL action suite +- **CI hardening** — Windows shard watchdog widened with exit diagnostics (#2449), platform-sensitive matrix sharded to fix the Windows cross-platform timeout (#2394), and CI Report no longer dies silently when the tests job fails (#2728) + ## [1.6.9] - 2026-07-04 ### Added diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 6d4e37d12..08aa3a863 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,12 +1,12 @@ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { diff --git a/gitnexus/package.json b/gitnexus/package.json index 7b496063d..87d83a090 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", From 880f94d147893c14ba3dc88a752424a5f7e32c0e Mon Sep 17 00:00:00 2001 From: glier Date: Fri, 28 Aug 2026 16:30:09 +0300 Subject: [PATCH 12/28] feat(group): resolve Kotlin constant-based route paths (@PostMapping(ApiPaths.X)) (#3059) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ingestion): resolve Kotlin constant-based route paths `route-extractors/constant-resolver.ts` is the language-agnostic core for folding constant references into literal route paths. Bindings existed for Java, JavaScript/TypeScript and Python, but not Kotlin — so `@PostMapping(ApiPaths.ORDERS)` resolved on Java sources and silently produced no route on the identical Kotlin code. Add `route-extractors/kotlin-const-resolver.ts` as the fourth binding, mirroring `java-const-resolver.ts` (Kotlin shares the JVM package/import model) in structure, naming and skip-floor discipline: * `resolveKotlinImport` — import specifier -> file path. Tier 1 matches the `/.kt` convention; tier 2 falls back to the unique constant-defining file in the package directory, because Kotlin does not require a file to be named after the declaration it holds. Each tier is unique-or-nothing. * `extractKotlinModuleConstants` — parse tree -> `ModuleConstants`. * `parseKotlinConstOperands` / `foldKotlinOperands` — pre-bound wrappers so the calling side stays language-neutral, matching how `spring.ts` consumes `parseJavaConstOperands`. Kotlin-specific forms handled explicitly rather than translated from Java: top-level `const val`, `object` members, `companion object` members (keyed under the enclosing class, since `Companion` never appears in a reference), import aliases (`import a.b.C as D`), and the absence of a `String` type gate (Kotlin infers property types, so the initializer decides). String templates and multi-line raw strings are refused rather than folded with the interpolation dropped, which would publish a path the application does not serve. `var`, custom getters and delegates are not constants and are skipped. Wiring: `group/extractors/http-patterns/kotlin.ts` gains a `prepareRepo` pre-pass that builds the repo-wide constant map once per `extract()` run, and `scan` now folds constant-valued `@(Get|Post|Put|Delete|Patch)Mapping` arguments against it — the same shape the Java plugin already implements. A constant-valued class-level `@RequestMapping` prefix suppresses the routes under that class, as in `java.ts`: emitting them unprefixed would turn a missing fact into a wrong one. An ambiguous import returns null, never a guess — a duplicate fully-qualified name across modules, a package with two constant files, and a wildcard import all floor to skip. A wrong resolution is a false edge in the graph; a missing one is only a missing fact. The ingestion provider (`languages/kotlin.ts`) is deliberately left alone: the ingestion fold runs only over `decoratorRoutes`, and Kotlin declares no `extractDecoratorRoutes` because `spring.ts` is bound to tree-sitter-java. Declaring the constant hooks there today would harvest a map nothing consumes. The reasoning is recorded in the new module's header. Tests cover each reference form (qualified, fully-qualified, single-name import, concatenation incl. 3+ operand chains) plus every ambiguity case, at both the resolver layer and through `KOTLIN_HTTP_PLUGIN.prepareRepo` + `scan`. * test(group): cover the Kotlin route-fold guards left unpinned Follow-up to the Kotlin constant-route binding, closing the test gaps a review found. No behavior change: the only source edit is a comment. * Class-prefix suppression is now asserted for the NAMED spelling (`@RequestMapping(value = ApiPaths.BASE)`) as well as the positional one. The two take different branches of `kotlinRouteArgumentExpression`, and only the method-path side of the named branch was covered; a regression there would let a constant-prefixed class escape suppression and publish every method under it at an unprefixed path. * `MAX_FOLD_LENGTH` and both recursion caps are pinned from both sides. Output doubles per level while depth only increments, so 13 doublings of a one-character leaf land exactly on the limit and 14 overrun it; a 30-link reference chain resolves where a 40-link one hits the cross-file cap; an 80-term `+` chain hits the operand-parse cap where a 60-term one folds. A 30-level shared-descendant DAG folding inside a 5 s budget pins the success memo that keeps it out of O(2^depth). These are the guards that keep a pathological constant graph from building a gigabyte-scale string or recursing without bound during a group sync; they were inherited from the audited Java binding but nothing held them in place. * OpenFeign consumers are covered on both paths: a constant method path folds, and a constant interface-level `@RequestMapping` prefix suppresses the consumer. The latter is deliberate — Spring Cloud prepends a type-level `@RequestMapping` to every method of the client, so an unfoldable prefix makes the remote URL unknowable whether or not `@FeignClient(path)` is present, and a dropped edge beats a wrong one. The suppression reaches an interface because tree-sitter-kotlin models `interface` as a `class_declaration`; `java.ts` misses this case only because its `findEnclosingClass` skips `interface_declaration`, and aligning Java changes Java's behavior, so it is left to its own change. Documented at the guard so the divergence is not read as an oversight. Note for reviewers of the parent change: re-indexing a Kotlin Spring service will REMOVE routes that were previously emitted, unprefixed, from classes whose `@RequestMapping` prefix is a constant. Those paths were never served by the application; the drop is the fix, not a regression. * fix(group): suppress Kotlin routes only when the class prefix resolves to no literal Class-prefix suppression decided "is this prefix unresolvable?" from a three-element allow-list of node types (`simple_identifier`, `navigation_expression`, `additive_expression`). An allow-list is safe for FOLDING, where a forgotten shape yields no route, but it is the wrong shape for SUPPRESSION, where a forgotten shape means "emit unprefixed" — a route the application does not serve. `java.ts` gates on the ABSENCE of a literal (`if (!valueNode)`) for exactly this reason. The predicate is now inverted: a class is marked unless its `path`/`value` argument is provably literal, recursing into `[…]` and `arrayOf(…)` elements and refusing an interpolated `string_literal`. Measured against the previous behavior, with `@PostMapping(ApiPaths.ORDERS)` under each class prefix, on an app serving `/api/v1/orders`: * `[ApiPaths.BASE]` `POST /orders` -> dropped * `arrayOf(ApiPaths.BASE)` `POST /orders` -> dropped * `value = [ApiPaths.BASE]` `POST /orders` -> dropped * `buildPath()` `POST /orders` -> dropped * `if (USE_V2) "/api/v2" else …` `POST /orders` -> dropped * `"${ApiPaths.BASE}"` `POST /${ApiPaths.BASE}/orders` -> dropped The last one published raw source text as a served path; refusing an interpolated literal also fixes it for LITERAL method routes, which emitted `/${ApiPaths.BASE}/list` before this branch existed. Two regressions this suppression had introduced are repaired, both by consulting the literal-prefix map that the pass above already built and declining to mark a class that has an entry in it: * `@RequestMapping("/lit", ApiPaths.BASE)` + `@GetMapping("/list")` lost `GET /lit/list` entirely. Kotlin's vararg spelling leaves a resolvable arm behind, and suppression exists to avoid wrong routes, not to discard right ones. * `@FeignClient(path = "/api")` + `@RequestMapping(ApiPaths.BASE)` lost its consumer, though `path` outranks `@RequestMapping` when the URL is assembled and made the prefix perfectly knowable. Two Feign emission paths never consulted the unfoldable set at all: * `@FeignClient(path = CONST)` was invisible to the analysis, which matches `@RequestMapping` only, so the client fell through to the no-prefix fallback and published `GET /orders` for a call the service makes to `/api/v1/orders`. Collected as its own set, kept separate because `path` outranks `@RequestMapping` in both directions. * The `@RequestLine` loop resolves through the identical "path wins" fallback chain but had no guard, so one interface could suppress its `@(Get|…)Mapping` route and publish its `@RequestLine` route under the very same unresolvable prefix. Both lanes now judge alike. Note for reviewers: the `@RequestLine` guard is not a regression fix — that lane emitted a wrong unprefixed consumer before this branch too. It moves a wrong route to no route, on both sides of the change. Co-Authored-By: Claude Opus 5 (1M context) * fix(group): fold Kotlin route constants on Windows and when they fold to "" Two defects in the Kotlin constant-path fold, plus the documentation corrections review asked for. Windows keys. `resolveKotlinImport` turns an import specifier into `com/example/ApiPaths.kt` and asks whether a repository key ends with it. The orchestrator's key list comes from glob v13, which has no `posix: true` and joins with the platform separator, so on Windows every key arrives backslashed and that test can never pass: the pre-pass still ran, the repo context was still built, and every cross-file fold returned null — the headline feature silently absent on one platform. Every unit fixture spelled its keys POSIX, so CI could not see it. Normalized at the one boundary that produces the keys — `prepareRepo`'s map keys and `scan`'s `fileRel` — which is the fix `node.ts` and `python.ts` already apply for the same reason. `readFile` still receives the raw path. Normalizing inside the resolver cannot work: it returns the key it matched, so a normalized return value would miss in a map nobody normalized. Empty fold. `foldKotlinOperands` collapsed `''` into `null`, conflating "folded to the empty string" with "unresolvable". `const val ROOT = ""` is Spring's spelling for the class prefix itself, so under `@RequestMapping("/api")` the literal `@PostMapping("")` published `POST /api/` while `@GetMapping(ApiPaths.ROOT)` published nothing. Return the fold unfiltered: callers already guard on `=== null`, `resolveKotlinConstant` already returned `''` for the same constant, and this matches `foldJavaOperands`. Docs. The module header claimed the fold, the cycle guard and the depth cap all live in the agnostic core. They do not — roughly 200 lines are a local fork of the Java binding's already forked state machine, because the core keys its maps by simple name while a Kotlin operand can be qualified at any position. Say that, with the reason and the follow-up. The stated `isKotlinConstantFile` invariant ("never rejects a file the extractor accepts") is false: the extractor harvests a top-level non-`const` `val` that fails both gate arms. The cost is not nil, either — measured, such a constant in its own file loses every cross-file route, while the same declaration beside the route still folds through `scan`'s on-demand re-extract. Both recorded on the gate. The depth caps are now `MAX_OPERAND_PARSE_DEPTH` (64) and `MAX_FOLD_DEPTH` (32); the core's own `MAX_RESOLVE_DEPTH` is 8 and module-private, so it cannot simply be reused. Measured with a differential probe over 41 Kotlin fixtures against the PR base, in both key styles. Exactly one POSIX row moves — the empty fold — and every other row, controls included, is byte-identical to before. POSIX and Windows keys now yield identical detections on every fixture, on both sides. Deliberately not done: the `isKotlinConstantFile` gap is documented, not closed, because closing it means parsing every file that contains any `val`. `java-const-resolver.ts` still spells 64 and 32 inline. The PR body's rollout note still says re-indexing activates the change — `HttpRouteExtractor` runs during `group sync` (`sync.ts:297`), so that is a PR-body fix, not a code one. Co-Authored-By: Claude Opus 5 (1M context) * fix(group): key Kotlin constants by visibility and resolve imports on the declared package Two ways the Kotlin route fold could publish a path the application does not serve. Both were inherited from the merged Java binding, which documents each as accepted; the notes were wrong, not merely conservative, and both are left open as a Java follow-up rather than changed here. Simple-name flattening. Every `object`/companion member was recorded under BOTH its qualified name `Owner.NAME` and its bare `NAME` in one file-level namespace, so an initializer naming a sibling resolved through whichever object was walked LAST: object A { const val BASE = "/right"; const val ROUTE = BASE + "/m" } object B { const val BASE = "/wrong" } @GetMapping(A.ROUTE) // Kotlin serves /right/m; this emitted /wrong/m Swapping the two objects flipped the answer back — the same source, merely reordered, changed the emitted route. The bare key is also a binding Kotlin does not have: `BASE` alone never names `A.BASE` from outside `object A`'s body, and because the fold consults literals before imports, that fabricated key outranked a genuine `import com.example.api.Paths.ORDERS` and published the local object's value instead of the imported one. Keys now follow Kotlin's own visibility. A member of a named `object` gets only `Owner.NAME`; the simple name is recorded for a top-level `val` and for a companion member, which really is in scope unqualified throughout its enclosing class. Initializers resolve against their scope chain, innermost first, so `BASE` inside `object A` means `A.BASE` — collecting every declaration before recording any is what makes that independent of declaration order. An unfoldable object member no longer drops a same-named import either, since it shadows nothing. The known limit is now stated rather than argued away: a companion's bare key is still file-wide, so two companions in one file whose members collide still resolve last-wins for an unqualified reference. Kotlin scopes that to the enclosing class and this map cannot express it — the fold is entered with a file key and a name, and nothing says which class body the annotation sat in. Initializers are unaffected; only a bare annotation reference can land wrong. Import binding never read the `package` header. Both tiers picked candidates purely from the path, so a file whose PATH ended with the imported FQN beat the real declaration — and when the decoy declared the same constant the fold did not skip, it invented a value. Measured: `object ApiPaths { const val ORDERS = "/right" }` in `src/generated/Constants.kt` (`package com.example.api`) plus a decoy at `src/x/com/example/api/ApiPaths.kt` (`package x.com.example.api`) emitted `GET /wrong`. This falsifies the old docstring's safety argument, which only covered a wrong file that LACKS the name. Two further triggers: a root-level `package data` was impersonated by `com/example/data` on a path-suffix test, while the real root-level file was invisible to the package-directory tier at all; and a unique constant file under a test source tree folded into a production route. The declared `package` is now recorded per file and matched exactly. Candidates that declare a different package are rejected rather than guessed at, an entry with no recorded package is rejected too, and two files declaring the same fully-qualified name resolve to nothing — a duplicated FQN names no single declaration, so the test-source copy of a production constant is a skip, not a guess about build configuration this layer cannot see. The file-name convention survives only as a tie-break among candidates that already declare the right package. `packageName` rides on a Kotlin-local `KotlinModuleConstants` rather than widening the agnostic `ModuleConstants`, which Java, JS and Python share and none of them needs it. Measured with a differential probe over all 41 Kotlin fixtures, in both key styles. Seven rows move, all of them from a wrong route: * sibling shadow, A first /wrong/m -> /right/m * bare key beats import /wrong -> /right * path-suffix decoy /wrong -> /right * root-package suffix match /wrong -> /right * root-package suffix only /wrong -> (skip; not in the repo) * test copy into production /test-only -> (skip; FQN declared twice) * wrong file lacks the name (skip) -> /right The last row is the one control that changes, and it changes from emitting nothing to emitting the route Kotlin serves: its decoy declares a different package, so the unconventionally named real file is now the sole candidate. Every other row, all six remaining controls included, is byte-identical to before, and POSIX and Windows keys still agree on every fixture. Deliberately not done: `resolveKotlinImport` does not PREFER the candidate that declares the sought name when several share the package — it only rejects when two do. Preferring it would resolve more imports correctly (a package holding `ApiPaths.kt` that declares something else and `Constants.kt` that declares `ApiPaths`), but it is a separate skip-to-route improvement that would rewrite an assertion this suite already pins, and the review round did not ask for it. Co-Authored-By: Claude Opus 5 (1M context) * fix(group): scope Kotlin companion constants to their class, and stop an empty path array suppressing routes Three ways the Kotlin route fold still published the wrong answer, all measured on fixtures rather than reasoned about, plus one comment that was false. Empty path array. `hasResolvableLiteralPathElement` answered "does any element resolve to a literal?", and `[].some(...)` is `false`, so `@RequestMapping(arrayOf())` read as an UNRESOLVABLE prefix and suppressed every route under the class — including a plain `@GetMapping("/lit")` that no constant fold ever touched. Spring treats an empty array as NO prefix, so `/lit` is genuinely served. The same arithmetic hit `@FeignClient(path = arrayOf())`, dropping a consumer. Measured against the pre-suppression branch point: * `@RequestMapping(arrayOf())` + `@GetMapping("/lit")` nothing -> GET /lit * `@FeignClient(path = arrayOf())` nothing -> consumer GET /orders The predicate is now a three-valued `classifyPathArgument`: `'literal'`, `'none'` (an empty array — no prefix), `'unresolvable'`. Only the last may suppress, because "no prefix" is not "an unresolvable prefix" and only one of them makes the served path unknowable. `@RequestMapping([])` is the same idea spelled differently, but tree-sitter-kotlin does not parse the class carrying it as a `class_declaration` at all, so no class-prefix pattern matches it and no arm here can be reached — recorded rather than guarded against. Companion scope. A companion member's simple name was recorded into the SAME file-level namespace as top-level constants, and companions are recorded last, so it won every unqualified reference in the file — including from a class that is not its own. Kotlin binds it unqualified inside its enclosing class body and nowhere else: * top-level `/top` vs an unrelated `Holder`'s companion `/companion`, referenced from a third class `/companion` -> `/top` * two companions colliding on one name `/h2` -> `/h1` * `const val ROUTE = BASE + "/m"` at file level beside a companion `BASE` `/comp/m` -> `/top/m` * a single-name import losing to a same-named companion outside its class Only a TOP-LEVEL `val` now writes a bare key. The unqualified binding is reached from the reference site instead: `scan` collects the enclosing type chain of the annotation and `foldKotlinOperands` rewrites a bare operand to `.` when an enclosing type declares it — innermost first, before the file-level maps and before imports, which is Kotlin's own order. So the companion still wins inside its own class (the control that pinned this behavior keeps passing) and loses everywhere else. Nothing was skipped to get there: every one of the four cases now emits the route the application serves. An unfoldable companion member still drops a same-named import file-wide. The import map has no scopes, and over-deleting costs a route while under-deleting publishes the imported value at a reference the compiler binds to the unfoldable member. Backtick quoting. `` package com.example.`api` `` and `package com.example.api` are the same package to the compiler — the quotes are lexical syntax, not part of the name — but the grammar keeps them in the node text, `declaredPackage` joined them verbatim and `resolveKotlinImport` required an exact match, so the sole real candidate was rejected and `GET /right` was lost. Every identifier that becomes a map key or a lookup name is now read through `unquoteKotlinIdentifier`: package segments, import specifiers and aliases, declaration and member names, and references. Both directions matter — an import may quote a segment the declaration spells plainly, and the reverse — and a KEYWORD segment, which can only be spelled quoted, still folds. Comment correction. The previous commit's note claimed "Sibling INITIALIZERS are unaffected (they go through the scope chain above); only a bare reference from a route annotation can land on the wrong companion." That is false, and the `/comp/m` case above is the counterexample: a top-level initializer has an EMPTY scope chain, so `qualifyRef` leaves its operand bare and the file-wide companion key answered it. The source comment now describes what the code does; the claim also appears in the `ef402a4a` commit body, which is already published and is left as written. Not changed. `resolveKotlinImport` computes `declaring` — the unique in-package file that declares the sought name — and uses it only to REJECT when two files declare it, never to resolve. With two or more in-package candidates it falls through to the file-name convention and returns null, dropping a case Kotlin resolves unambiguously. Returning it would flip the pinned assertion "returns null when the package holds two constant files and no name matches" from skip to route (that fixture's `Paths.kt` does declare `object ApiPaths`, so `declaring` is not null there despite the test's title), so it is left open as a follow-up rather than traded against a skip-floor assertion. Fixture sweep: 82 cases in both POSIX and Windows key styles. Six move, all listed above; the other 76 are byte-identical on both key styles, including the companion-inside-its-own-class control, the qualified-reference cases, and the pre-existing interface-inheritance gap on a constant controller prefix, which is unchanged and remains a separate follow-up. Co-Authored-By: Claude Opus 5 (1M context) * fix(group): admit backtick-quoted constants, and overlay a same-file val that imports nothing Two gate defects, both reported by the review bot and both reproduced before being fixed. `isKotlinConstantFile` matched only `\w+` for a declaration's name, so a file whose constants are backtick-quoted — `const val ` + "`ORDERS`" + ` = "/orders"` — failed both arms and was never parsed into the repo constant map. The resolver supports backtick identifiers everywhere else: `unquoteKotlinIdentifier` strips the quoting at every point a name becomes a key or a lookup. So the gate was NARROWER than the extractor, which is the one direction its arms exist to exclude, and a cross-file reference to such a constant floored to skip. Measured: the route emitted nothing, and emits `GET /orders` now. The on-demand overlay in `scan` admitted the file's extraction only when it had imports. A file declaring a top-level non-`const` `val` is already excluded from the pre-pass map (no `const`, no `object`), so this branch is its only chance, and an import-only test discarded exactly the constants the route needed. The guard now matches the admission test the pre-pass itself applies. The bot stated this second one more broadly than it holds. Measured, any import at all masks it — a realistic Spring controller always has one — so the failure needs all three of: a top-level non-`const` `val`, no `object` in the file, and no imports. Narrow, but real, and the fix costs one predicate. Verified with the differential probe: both cases go from no detection to the correct route, and all 41 existing fixtures are byte-identical before and after. Co-Authored-By: Claude Opus 5 (1M context) * refactor(group): let the resolver own Kotlin enclosing-type qualification The route caller gated kotlinEnclosingTypeNames on its own copy of foldKotlinOperands' bare-ref predicate. That gate could not change the result — qualifyKotlinRefInEnclosingTypes returns a dotted name unchanged — so it only spread one rule across two modules that can drift apart. Also corrects a trimmed comment that claimed a collection_literal never reaches classifyPathArgument, which the non-empty branch there disproves. Co-authored-by: Cursor * fix(group): keep unfoldable Kotlin constants as skip, not a wrong route Record declared-but-unfoldable names so a companion or duplicate FQN cannot fall through to a foldable twin, and treat empty [] as no prefix on parsed RequestMapping arrays. Prefer the unique declaring file before package filename fallbacks so extra unfoldable files in the same package do not drop a real route. Co-authored-by: Cursor * fix(group): preserve full paths for nested Kotlin constants (#3059) Key nested objects and companions by their full enclosing type path so same-file, imported, and bare nested references resolve consistently. Co-authored-by: Cursor * fix(group): qualify a PARTIALLY qualified Kotlin reference, not just a bare one Both qualification points short-circuited on `name.includes('.')` — "already carries its owner". A dotted reference carries AN owner, not necessarily its own full one, and Kotlin resolves a partially qualified name against the enclosing scopes exactly as it resolves a bare one. Measured on the branch before this change: * `object Outer { object Inner { const val Q = "/orders" } @GetMapping(Inner.Q) … }` emitted NOTHING. The key is `Outer.Inner.Q`; left unchanged, `Inner.Q` matches nothing. * a top-level `object ApiPaths { ORDERS = "/orders" }` beside `class OrderController { object ApiPaths { ORDERS = "/inner" } }`, with `@GetMapping(ApiPaths.ORDERS)` inside that class, emitted `/orders`. The compiler binds the NESTED object, so the application serves `/inner`. That is a wrong route, not a missing one. * the same defect on the initializer side: `const val ROUTE = Inner.Q + "/m"` inside `object Outer` emitted nothing, where Kotlin gives `/orders/m`. Fixing only one side would leave the two halves disagreeing about what a dotted name means, which is the asymmetry the earlier defects in this file came from, so both move together: * `qualifyKotlinRefInEnclosingTypes` drops the early return. The scopes it walks are already qualified, so prefixing them onto whatever the reference spells is the whole rule. * `qualifyRef` splits at the last dot and prefixes the scope onto the OWNER, so the bare case is byte-for-byte what it was. The allocation gate in `foldKotlinOperands` loses its `!includes('.')` clause for the same reason. It was not merely a missed optimization: it decided the result per OPERAND LIST, so the same `Inner.Q` folded or not depending on whether a sibling operand happened to be bare. Verified with the differential probe: the three cases above go from wrong or missing to correct, and all 41 existing fixtures are byte-identical to face59ae in both POSIX and Windows key styles. Co-Authored-By: Claude Opus 5 (1M context) * fix(group): close Kotlin val gate and reuse import indexes (#3059) Recognize standalone top-level vals without parsing local declarations, and prepare exact-package/FQN constant indexes once per extraction so route folds avoid repeated repo scans while preserving ambiguity floors. Co-authored-by: Cursor --------- Co-authored-by: Gergő Magyar Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- .../group/extractors/http-patterns/kotlin.ts | 550 ++++++- .../route-extractors/kotlin-const-resolver.ts | 1400 ++++++++++++++++ .../group/kotlin-const-route-fold.test.ts | 1265 ++++++++++++++ .../unit/kotlin-route-const-resolver.test.ts | 1448 +++++++++++++++++ 4 files changed, 4656 insertions(+), 7 deletions(-) create mode 100644 gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts create mode 100644 gitnexus/test/unit/group/kotlin-const-route-fold.test.ts create mode 100644 gitnexus/test/unit/kotlin-route-const-resolver.test.ts diff --git a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts index 10f195111..d9a7c1705 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts @@ -18,6 +18,19 @@ import { joinPath, type SharedSpringType, } from '../../../ingestion/route-extractors/spring-shared.js'; +import { + buildKotlinConstantIndex, + extractKotlinModuleConstants, + foldKotlinOperands, + isKotlinConstantFile, + overlayKotlinConstantIndex, + parseKotlinConstOperands, + unfoldableDeclarationsOf, + unquoteKotlinIdentifier, + type KotlinConstantIndex, + type ModuleConstants, + type RepoConstants, +} from '../../../ingestion/route-extractors/kotlin-const-resolver.js'; import { REST_TEMPLATE_TO_HTTP, WEB_CLIENT_SHORT_TO_HTTP, @@ -42,6 +55,24 @@ import { * named annotation arguments (`@GetMapping(value = "/x")` and * `@GetMapping(path = "/x")`) are supported. * + * A method path that is a CONSTANT rather than a literal — + * `@GetMapping(ApiPaths.ORDERS)`, `@PostMapping(value = ApiPaths.BASE + "/create")` — + * is folded against a repo-wide Kotlin constant map built once per `extract()` + * run by `prepareRepo`, mirroring what the Java plugin does for the same shape + * in `java.ts`. An unresolvable fold skips the route (never a guessed path), and + * a class prefix that resolves to NO literal at all suppresses every method + * route under that class — the rule `java.ts` applies too, because emitting + * those routes unprefixed would publish paths the application does not serve. + * A prefix that resolves only PARTLY (Kotlin's vararg spelling + * `@RequestMapping("/lit", ApiPaths.BASE)`) still publishes its resolvable arm: + * suppression exists to avoid wrong routes, not to discard right ones. An EMPTY + * path array (`@RequestMapping(arrayOf())`) is not a prefix at all and + * suppresses nothing — see `classifyPathArgument`. On a + * `@FeignClient` the same rule is applied to whichever prefix GOVERNS, in the + * "path wins" order the URL is assembled in — `@FeignClient(path)` first, then + * the interface's `@RequestMapping` — and to both consumer lanes, `@(Get|...)Mapping` + * and `@RequestLine`. + * * **Consumers** — four call-site patterns common in Kotlin * Spring projects: * @@ -131,6 +162,180 @@ const arrayOfArg = (cap: string): string => `(call_expression (simple_identifier) @arrayOf (#eq? @arrayOf "arrayOf") (call_suffix (value_arguments (value_argument (string_literal) ${cap}))))`; +/** + * Expression node types a METHOD route path can be FOLDED from. A + * `string_literal` is deliberately absent: literal paths are already captured by + * the dedicated literal patterns, so admitting one here would emit the same + * route twice. + * + * This is an allow-list on purpose, and only safe because it gates FOLDING: a + * shape missing from it yields no route, which is the skip floor. The + * unfoldable-CLASS-PREFIX analysis must not be written this way — there a shape + * missing from the list means "emit unprefixed", a wrong route — so it inverts + * the test instead (see `classifyPathArgument`). + */ +const FOLDABLE_PATH_EXPRESSIONS: ReadonlySet = new Set([ + 'simple_identifier', + 'navigation_expression', + 'additive_expression', +]); + +/** + * Repo-relative path in the POSIX form the Kotlin constant map is keyed by. + * + * The orchestrator's file list comes from glob v13, which has no `posix: true` + * option and joins with the platform separator, so on Windows `prepareRepo` + * receives `src\main\kotlin\com\example\ApiPaths.kt` and `scan` receives the + * same for `fileRel`. `resolveKotlinImport` turns an import specifier into + * `com/example/ApiPaths.kt` and asks whether a key ENDS WITH it — a test no + * backslashed key can pass. Left unnormalized, every cross-file constant fold + * returns null on Windows and on Windows only: the pre-pass still runs, the + * context is still built, and the feature is simply, silently absent. The unit + * fixtures build POSIX keys by hand, so CI cannot see it. + * + * Normalizing at this boundary — write side (the map keys below) and read side + * (`fileRel`) — is the same fix `node.ts` (`normalizeRel`) and `python.ts` + * (`fileShortKey` / `fileLongKey`) already apply for the same reason, and it is + * the only coherent place: the resolver returns the key it matched, so + * normalizing inside it would hand back a value that misses in a map nobody + * normalized. `readFile` still receives the ORIGINAL `rel`, since the filesystem + * wants the platform's own spelling. + */ +function normalizeRel(rel: string): string { + return rel.replace(/\\/g, '/').replace(/^\.\//, ''); +} + +/** + * The path expression carried by one route-annotation argument, or null when the + * argument does not designate a path. + * + * tree-sitter-kotlin gives positional and named arguments the same + * `value_argument` node, distinguished only by a leading `simple_identifier` and + * an `=` token — so the key must be read here rather than constrained in the + * query. Non-route keys (`produces`, `consumes`, `headers`, …) return null, + * matching the `#match? @key "^(path|value)$"` guard the literal patterns use. + */ +function kotlinRouteArgumentExpression(arg: Parser.SyntaxNode): Parser.SyntaxNode | null { + const first = arg.namedChild(0); + if (!first) return null; + if (!arg.children.some((c) => c.type === '=')) return first; // positional + if (first.type !== 'simple_identifier') return null; + if (first.text !== 'path' && first.text !== 'value') return null; + return arg.namedChild(1); +} + +/** + * The `path = …` expression of one `@FeignClient` argument, or null. + * + * Deliberately narrower than {@link kotlinRouteArgumentExpression}: on a Feign + * client the positional argument and `value =` name a SERVICE, not a path, so + * only the explicit `path` key contributes a URL prefix. This mirrors the + * `#eq? @key "path"` guard the literal `@FeignClient` patterns use, and the + * `keyNode.text !== 'path'` guard `java.ts` applies to the same annotation. + */ +function kotlinFeignPathArgumentExpression(arg: Parser.SyntaxNode): Parser.SyntaxNode | null { + const first = arg.namedChild(0); + if (!first || first.type !== 'simple_identifier') return null; + if (!arg.children.some((c) => c.type === '=')) return null; + if (first.text !== 'path') return null; + return arg.namedChild(1); +} + +/** + * Is `node` a string literal whose value is fully known at parse time — that is, + * a literal carrying no interpolation? + * + * tree-sitter-kotlin models `"$base/x"` and `"${base}/x"` as a `string_literal` + * whose named children INTERLEAVE `string_content` runs with interpolation nodes + * — `interpolation_identifier_start`/`interpolated_identifier` for the `$name` + * form, `interpolation_expression_start`/`interpolated_expression`/ + * `interpolation_expression_end` for `${…}` — so the test has to be `every`, not + * `some`: `"pre${A.B}post"` carries `string_content` too. The route layer + * unquotes the RAW TEXT, so treating one as a literal publishes the source + * spelling — `/${ApiPaths.BASE}/orders` — as though the application served it. + * Escape sequences are NOT separate nodes in this grammar (`"/a\nb"` is one + * `string_content`), so this accepts exactly what it accepted before; a future + * grammar that split them would floor to "unknown" rather than to a de-escaped + * guess. Same test the constant resolver's `stringLiteralValue` applies, so a + * path is either literal on both sides or folded on neither. + */ +function isPlainStringLiteral(node: Parser.SyntaxNode): boolean { + if (node.type !== 'string_literal') return false; + return node.namedChildren.every((child) => child.type === 'string_content'); +} + +/** + * Element expressions of a Kotlin `arrayOf(...)` call, or null when `node` is + * not one. The JS mirror of the {@link arrayOfArg} query fragment, so the + * unfoldable-prefix analysis inspects exactly the elements the literal prefix + * patterns harvest. + */ +function kotlinArrayOfElements(node: Parser.SyntaxNode): Parser.SyntaxNode[] | null { + if (node.type !== 'call_expression') return null; + const callee = node.namedChild(0); + if (callee?.type !== 'simple_identifier' || callee.text !== 'arrayOf') return null; + const suffix = node.namedChildren.find((c) => c.type === 'call_suffix'); + const args = suffix?.namedChildren.find((c) => c.type === 'value_arguments'); + if (!args) return null; + return args.namedChildren + .filter((c) => c.type === 'value_argument') + .map((c) => c.namedChild(0)) + .filter((c): c is Parser.SyntaxNode => c !== null); +} + +/** + * What a route-annotation path argument says about the prefix it designates. + * Only `'unresolvable'` may suppress a route: + * + * - `'literal'` — at least one element is a plain literal, already harvested by + * the literal prefix patterns, so there is nothing to suppress. + * - `'none'` — no prefix. Empty `arrayOf()` or `[]` is Spring's "map at the root". + * Kept distinct from `'unresolvable'` because conflating them suppressed even + * plain literal routes below such a class, which no constant fold was ever + * involved in. tree-sitter-kotlin (fwcd) represents empty `[]` with a + * zero-width recovery child; filtering it is required for route interfaces, + * which do parse as `class_declaration`. + * - `'unresolvable'` — a non-empty argument with no literal element + * (`ApiPaths.BASE`, `buildPath()`, a template). Served path is unknowable. + */ +type PathArgumentPrefix = 'literal' | 'none' | 'unresolvable'; + +function classifyPathArgument(expr: Parser.SyntaxNode): PathArgumentPrefix { + if (isPlainStringLiteral(expr)) return 'literal'; + if (expr.type === 'collection_literal') { + const elements = expr.namedChildren.filter((child) => child.text.length > 0); + if (elements.length === 0) return 'none'; + return elements.some(isPlainStringLiteral) ? 'literal' : 'unresolvable'; + } + const elements = kotlinArrayOfElements(expr); + if (elements) { + if (elements.length === 0) return 'none'; + return elements.some(isPlainStringLiteral) ? 'literal' : 'unresolvable'; + } + return 'unresolvable'; +} + +/** + * Type declarations enclosing `node`, innermost first, by qualified type path. + * + * The scope a bare constant in a route annotation is resolved against; passed to + * `foldKotlinOperands`, which applies it. Collects `class_declaration` (including + * interfaces) and `object_declaration`. A `companion_object` adds no link of + * its own — members are keyed under the enclosing class one hop up. For a node + * inside `Outer.Inner`, returns `['Outer.Inner', 'Outer']`, matching the keys + * produced by `extractKotlinModuleConstants`. Skips unnamed types rather than + * guessing. + */ +function kotlinEnclosingTypeNames(node: Parser.SyntaxNode): string[] { + const simpleNames: string[] = []; + for (let cur = node.parent; cur; cur = cur.parent) { + if (cur.type !== 'class_declaration' && cur.type !== 'object_declaration') continue; + const ident = cur.children.find((c) => c.type === 'type_identifier'); + if (ident) simpleNames.push(unquoteKotlinIdentifier(ident.text)); + } + return simpleNames.map((_, index) => simpleNames.slice(index).reverse().join('.')); +} + // ─── Kotlin OkHttp builder verb-walk (parity with java-static-path.ts) ── // Mirrors `inferOkHttpMethod`, adapted to the Kotlin grammar: a call `X.name(args)` // is a `call_expression` whose callee is a `navigation_expression` (receiver + @@ -399,6 +604,151 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { ], } satisfies LanguagePatterns>); + // ─── Provider: constant-valued @RequestMapping / @(Get|...)Mapping ──── + // The literal patterns above pin the path node itself (`(string_literal) @path`), + // which structurally cannot match `@GetMapping(ApiPaths.ORDERS)`. These two + // capture the whole `value_argument` instead and let + // `kotlinRouteArgumentExpression` sort out positional vs `path =`/`value =` + // in JS — a query-level split is not available here, because tree-sitter-kotlin + // uses one `value_argument` node for both forms and 0.21.x has no negation to + // test the `=` token with. + // + // These deliberately match LITERAL arguments too (any `value_argument` does). + // The method-route loop drops those via `FOLDABLE_PATH_EXPRESSIONS` so a + // literal route is emitted once, by the literal patterns; the class-prefix + // collector instead KEEPS them and tests them for literalness, which is how a + // prefix that no literal pattern could resolve gets noticed at all. + const SPRING_CONST_CLASS_PREFIX_PATTERNS = compilePatterns({ + name: 'kotlin-spring-const-class-prefix', + language, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + (modifiers + (annotation + (constructor_invocation + (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (value_arguments (value_argument) @arg)))) + (type_identifier) @cls) @class + `, + }, + ], + } satisfies LanguagePatterns>); + + const SPRING_CONST_METHOD_ROUTE_PATTERNS = compilePatterns({ + name: 'kotlin-spring-const-method-route', + language, + patterns: [ + { + meta: {}, + query: ` + (function_declaration + (modifiers + (annotation + (constructor_invocation + (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (value_arguments (value_argument) @arg)))) + (simple_identifier) @method_name) @method + `, + }, + ], + } satisfies LanguagePatterns>); + + const SPRING_CONST_FEIGN_PATH_PATTERNS = compilePatterns({ + name: 'kotlin-spring-const-feign-path', + language, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + (modifiers + (annotation + (constructor_invocation + (user_type (type_identifier) @ann (#eq? @ann "FeignClient")) + (value_arguments (value_argument) @arg))))) @class + `, + }, + ], + } satisfies LanguagePatterns>); + + /** + * Ids of classes whose `@RequestMapping` prefix cannot be resolved to any + * literal, so no route under them can be published at a path the application + * actually serves. + * + * The predicate is INVERTED rather than an allow-list of non-literal node + * types: a class is marked unless its `path`/`value` argument is provably + * literal (recursing into `[…]` and `arrayOf(…)` elements, and refusing an + * interpolated `string_literal`). An allow-list has to enumerate every + * non-literal spelling and silently passes the ones it forgot — + * `[ApiPaths.BASE]`, `arrayOf(ApiPaths.BASE)`, `buildPath()`, + * `if (…) "/a" else "/b"` — each of which then publishes its methods at their + * UNPREFIXED path, a route the application does not serve. `java.ts` gates on + * the ABSENCE of a literal (`if (!valueNode)`) for the same reason. + * + * `resolvedPrefixes` is the literal prefix map built by the pass ABOVE, and a + * class holding an entry there is deliberately NOT marked: Kotlin's vararg + * spelling `@RequestMapping("/lit", ApiPaths.BASE)` leaves a resolvable `/lit` + * behind, and suppressing it would drop a route that IS derivable — trading a + * wrong route for a missing one, which is not the bargain this suppression + * exists to make. The prefix set is then partial (the constant arm is absent) + * exactly as it was before constant folding existed. + * + * The prefix is never folded here: it also feeds the cross-file + * interface-inheritance pass, which has no repo context, so folding it in + * `scan` alone would make the two views disagree. Same rule `java.ts` applies + * (`typesWithUnfoldablePrefix`); folding class prefixes cross-file is a + * follow-up on both sides. Used by BOTH `scan` and the inheritance-view + * collector — with the prefix map each has already built — so the two cannot + * drift apart. + */ + const collectUnfoldablePrefixClassIds = ( + tree: Parser.Tree, + resolvedPrefixes: ReadonlyMap, + ): Set => { + const ids = new Set(); + for (const match of runCompiledPatterns(SPRING_CONST_CLASS_PREFIX_PATTERNS, tree)) { + const argNode = match.captures.arg; + const classNode = match.captures.class; + if (!argNode || !classNode) continue; + if ((resolvedPrefixes.get(classNode.id) ?? []).length > 0) continue; + const expr = kotlinRouteArgumentExpression(argNode); + if (!expr || classifyPathArgument(expr) !== 'unresolvable') continue; + ids.add(classNode.id); + } + return ids; + }; + + /** + * Ids of `@FeignClient` interfaces whose `path` argument is present but not + * resolvable to a literal. + * + * `collectUnfoldablePrefixClassIds` cannot see these: it matches + * `@RequestMapping` only, so `@FeignClient(path = ApiPaths.BASE)` fell through + * to the `['']` prefix fallback and published the consumer at its unprefixed + * path — a call the service never makes. Kept as its own set rather than + * merged into the `@RequestMapping` one because `path` OUTRANKS + * `@RequestMapping` on a Feign client: an unresolvable `path` is fatal + * whatever the `@RequestMapping` says, and a resolvable `path` rescues a route + * whose `@RequestMapping` is a constant. The consumer lanes therefore consult + * the two in that same "path wins" order. + */ + const collectFeignUnfoldablePathClassIds = (tree: Parser.Tree): Set => { + const ids = new Set(); + for (const match of runCompiledPatterns(SPRING_CONST_FEIGN_PATH_PATTERNS, tree)) { + const argNode = match.captures.arg; + const classNode = match.captures.class; + if (!argNode || !classNode) continue; + const expr = kotlinFeignPathArgumentExpression(argNode); + if (!expr || classifyPathArgument(expr) !== 'unresolvable') continue; + ids.add(classNode.id); + } + return ids; + }; + // ─── Consumer: Spring RestTemplate ──────────────────────────────────── // Kotlin call-site shape mirrors the Java plugin's // `REST_TEMPLATE_PATTERNS`, but goes through tree-sitter-kotlin's @@ -875,11 +1225,24 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { const prefixNode = match.captures.prefix; const classNode = match.captures.class; if (!prefixNode || !classNode) continue; + // An INTERPOLATED literal (`"${ApiPaths.BASE}"`) is not a path — unquoting + // its raw text would carry the source spelling into the shared type view + // as a served prefix. Refusing it here is also what lets the unfoldable + // analysis below mark such a class (it skips classes with a resolved + // prefix), so the two stay one decision rather than two. + if (!isPlainStringLiteral(prefixNode)) continue; const prefix = unquoteLiteral(prefixNode.text); if (prefix !== null) pushPrefix(prefixByClassId, classNode.id, prefix); } // Method @(Get|...)Mapping routes keyed by the function_declaration node id. + // + // Only LITERAL paths land here. A constant-valued path is folded in `scan` + // against the repo constant map, which this inheritance-view collector has + // no access to; publishing it as an empty path would put `POST /`-shaped + // noise into the shared type view, so it is left out — the same skip floor + // `java.ts`'s `collectSpringTypes` keeps. const routesByMethodId = new Map>(); + const unfoldablePrefixClassIds = collectUnfoldablePrefixClassIds(tree, prefixByClassId); for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const pathNode = match.captures.path; @@ -889,6 +1252,10 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { if (!httpMethod) continue; const rawPath = unquoteLiteral(pathNode.text); if (rawPath === null) continue; + // A constant class prefix leaves no single prefix string for the + // inheritance view to carry, so this route would be published unprefixed. + const owner = findEnclosingClass(methodNode); + if (owner && unfoldablePrefixClassIds.has(owner.id)) continue; const arr = routesByMethodId.get(methodNode.id) ?? []; arr.push({ method: httpMethod, path: rawPath }); routesByMethodId.set(methodNode.id, arr); @@ -931,8 +1298,90 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { return { name: 'kotlin-http', language, - scan(tree) { + prepareRepo(args) { + // Build the repo-wide Kotlin string-constant map and import index once per + // extract() run. The orchestrator hands over a bare Parser with no language + // bound; bind Kotlin explicitly or `parseSource` spins to its whole time + // budget on every file. + try { + args.parser.setLanguage(language); + } catch { + // A parser that rejects binding cannot produce a constant map; the + // per-file try/catch below then skips everything harmlessly. + } + const constants = new Map(); + for (const rel of args.files) { + if (!rel.endsWith('.kt') && !rel.endsWith('.kts')) continue; + try { + const src = args.readFile(rel); + // Cheap content gate: only constant-DEFINITION candidates are parsed + // here. Import-only files (every controller) are deliberately NOT + // parsed in this pass — `scan` extracts the importing file's own + // import table from the tree it already holds, on demand, for the + // rare file that actually references a constant. A gate that also + // matched `import …` would parse the entire repository here. + if (!src || !isKotlinConstantFile(src)) continue; + const tree = args.parseSource(args.parser, src); + if (!tree) continue; + const mc = extractKotlinModuleConstants(tree); + if ( + mc.literals.size > 0 || + mc.exprs.size > 0 || + mc.imports.size > 0 || + unfoldableDeclarationsOf(mc).size > 0 + ) { + // POSIX key (see `normalizeRel`); `readFile` above got the raw `rel`. + constants.set(normalizeRel(rel), mc); + } + } catch { + // Per-file resilience: one unreadable/oversized/ill-formed file must + // not forfeit the whole repo's constant map. + continue; + } + } + return { constants, index: buildKotlinConstantIndex(constants) }; + }, + scan(tree, repoContext, fileRel) { const out: HttpDetection[] = []; + const kotlinCtx = repoContext as + | { constants: RepoConstants; index: KotlinConstantIndex } + | undefined; + + // Read side of the POSIX keying (see `normalizeRel`): the map `prepareRepo` + // built is keyed by normalized path, so every lookup and every fold entry + // point below uses `fileKey`, never the raw `fileRel`. + const fileKey = fileRel === undefined ? undefined : normalizeRel(fileRel); + + // Lazy per-file constants/index view. `prepareRepo` only indexes constant- + // DEFINING files, so an importing controller is absent from that map. When + // a route references a constant, extract THIS file's import table from the + // tree `scan` already holds and overlay it. Import-only overlays reuse the + // prepared package projections; files whose routes are all literal never + // pay this cost. + let foldIndex: KotlinConstantIndex | undefined; + const getFoldIndex = (): KotlinConstantIndex | undefined => { + if (foldIndex !== undefined) return foldIndex; + foldIndex = kotlinCtx?.index; + if (!kotlinCtx || !fileKey) return foldIndex; + if (kotlinCtx.constants.has(fileKey)) return foldIndex; + try { + const mc = extractKotlinModuleConstants(tree); + // Same admission test the pre-pass applies above. Keeping the complete + // test here also makes this overlay correct if a future gate safely + // excludes another declaration shape. + if ( + mc.literals.size > 0 || + mc.exprs.size > 0 || + mc.imports.size > 0 || + unfoldableDeclarationsOf(mc).size > 0 + ) { + foldIndex = overlayKotlinConstantIndex(kotlinCtx.index, fileKey, mc); + } + } catch { + // fold falls back to the repo-wide map (imports stay unresolved) + } + return foldIndex; + }; // ─── Class prefixes ───────────────────────────────────────────── const prefixByClassId = new Map(); @@ -940,10 +1389,17 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { const prefixNode = match.captures.prefix; const classNode = match.captures.class; if (!prefixNode || !classNode) continue; + // An INTERPOLATED literal (`"${ApiPaths.BASE}"`) is not a path — see + // `isPlainStringLiteral`. Refusing it here also lets the unfoldable + // analysis below mark such a class, since that skips classes whose + // prefix already resolved. + if (!isPlainStringLiteral(prefixNode)) continue; const prefix = unquoteLiteral(prefixNode.text); if (prefix !== null) pushPrefix(prefixByClassId, classNode.id, prefix); } + const classesWithUnfoldablePrefix = collectUnfoldablePrefixClassIds(tree, prefixByClassId); + // ─── OpenFeign client interfaces + HTTP Interface type prefixes ── // In tree-sitter-kotlin an `interface` is a `class_declaration`, so a // `@FeignClient` interface's @(Get|...)Mapping methods would otherwise be @@ -956,11 +1412,12 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { if (!classNode) continue; feignClassIds.add(classNode.id); const prefixNode = match.captures.prefix; - if (prefixNode) { + if (prefixNode && isPlainStringLiteral(prefixNode)) { const prefix = unquoteLiteral(prefixNode.text); if (prefix !== null) pushPrefix(feignPrefixByClassId, classNode.id, prefix); } } + const feignClassesWithUnfoldablePath = collectFeignUnfoldablePathClassIds(tree); const httpExchangePrefixByClassId = new Map(); for (const match of runCompiledPatterns(SPRING_HTTP_EXCHANGE_CLASS_PATTERNS, tree)) { const classNode = match.captures.class; @@ -971,24 +1428,91 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { } // ─── Method routes (Spring providers) + OpenFeign consumers ───── + // Literal and constant-valued paths are normalized into one candidate list + // so both reach the same Feign/interface/prefix classification below. + const methodRoutes: Array<{ + httpMethod: string; + rawPath: string; + nameNode: Parser.SyntaxNode | undefined; + methodNode: Parser.SyntaxNode; + }> = []; for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const pathNode = match.captures.path; - const nameNode = match.captures.method_name; const methodNode = match.captures.method; if (!annNode || !pathNode || !methodNode) continue; const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; if (!httpMethod) continue; const rawPath = unquoteLiteral(pathNode.text); if (rawPath === null) continue; + methodRoutes.push({ + httpMethod, + rawPath, + nameNode: match.captures.method_name, + methodNode, + }); + } + for (const match of runCompiledPatterns(SPRING_CONST_METHOD_ROUTE_PATTERNS, tree)) { + const annNode = match.captures.ann; + const argNode = match.captures.arg; + const methodNode = match.captures.method; + if (!annNode || !argNode || !methodNode) continue; + const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; + if (!httpMethod) continue; + const expr = kotlinRouteArgumentExpression(argNode); + if (!expr || !FOLDABLE_PATH_EXPRESSIONS.has(expr.type)) continue; + // No repo context (context-less fallback scanning) means no constant map + // and therefore no honest answer — skip rather than guess a path. + if (!fileKey) continue; + const index = getFoldIndex(); + if (!index) continue; + const operands = parseKotlinConstOperands(expr); + if (operands === null) continue; + // A bare reference means whatever the ENCLOSING types bind it to before + // it means anything at file level — Kotlin's rule for a companion + // member, which is in scope unqualified only inside its own class body. + const rawPath = foldKotlinOperands( + fileKey, + operands, + index.repo, + kotlinEnclosingTypeNames(methodNode), + index, + ); + if (rawPath === null) continue; + methodRoutes.push({ + httpMethod, + rawPath, + nameNode: match.captures.method_name, + methodNode, + }); + } + + for (const { httpMethod, rawPath, nameNode, methodNode } of methodRoutes) { const enclosingClass = findEnclosingClass(methodNode); // A @(Get|...)Mapping inside a @FeignClient interface is an OpenFeign // consumer (a remote call), not a route this service serves. if (enclosingClass && feignClassIds.has(enclosingClass.id)) { + // Whichever prefix GOVERNS must be resolvable, or the remote URL is + // unknowable and an unprefixed consumer would be a call this service + // never makes. Checked in the same "path wins" order the fallback + // below resolves in, so an unresolvable `@RequestMapping` does not + // suppress a client whose literal `@FeignClient(path)` outranks it, + // and an unresolvable `path` is fatal even when `@RequestMapping` is + // a literal. + // + // This reaches a Feign INTERFACE at all because tree-sitter-kotlin + // models `interface` as a `class_declaration`, and it should: Spring + // Cloud prepends the governing prefix to every method of the client. + // Java diverges only by accident of its grammar — `findEnclosingClass` + // skips `interface_declaration`, so `java.ts` still emits such a + // consumer at its unprefixed path. Aligning Java is a change to Java's + // behavior and belongs in its own follow-up, not in the Kotlin binding. + if (feignClassesWithUnfoldablePath.has(enclosingClass.id)) continue; + const feignPrefixes = feignPrefixByClassId.get(enclosingClass.id); + if (!feignPrefixes && classesWithUnfoldablePrefix.has(enclosingClass.id)) continue; // @FeignClient(path) wins over @RequestMapping; a multi-element prefix // yields one consumer per (prefix × this route). - const prefixes = feignPrefixByClassId.get(enclosingClass.id) ?? - prefixByClassId.get(enclosingClass.id) ?? ['']; + const prefixes = feignPrefixes ?? prefixByClassId.get(enclosingClass.id) ?? ['']; for (const prefix of prefixes) { out.push({ role: 'consumer', @@ -1002,6 +1526,10 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { } continue; } + // An unresolvable class prefix leaves no path this service serves, so + // every route under such a class is dropped rather than emitted at a + // wrong (unprefixed) one — the rule `java.ts` applies for Java. + if (enclosingClass && classesWithUnfoldablePrefix.has(enclosingClass.id)) continue; // A @(Get|...)Mapping on a (non-Feign) interface declares a route // *contract*, not a route this service serves — the implementing // @RestController is the provider, emitted via scanProject's interface @@ -1171,13 +1699,21 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { if (!parsed) continue; const enclosingClass = findEnclosingClass(methodNode); if (!enclosingClass || !isKotlinInterface(enclosingClass)) continue; + // The same governing-prefix resolvability guard the @(Get|...)Mapping-in-Feign + // lane applies, in the same "path wins" order — this loop resolves through + // the identical fallback chain, so an unresolvable governing prefix leaves + // the remote URL just as unknowable here. Without it a single interface + // could suppress its @(Get|...)Mapping routes and publish its @RequestLine + // routes under the very same unresolvable prefix. + if (feignClassesWithUnfoldablePath.has(enclosingClass.id)) continue; + const feignPrefixes = feignPrefixByClassId.get(enclosingClass.id); + if (!feignPrefixes && classesWithUnfoldablePrefix.has(enclosingClass.id)) continue; // Mirror java.ts (which pre-merges the @RequestMapping fallback into // feignPrefixByInterfaceId, "path wins"): @FeignClient(path) wins, else // the interface's class-level @RequestMapping prefix, else none. Without // the prefixByClassId fallback Kotlin dropped the class prefix that Java // applies — the same fallback chain the @GetMapping-in-Feign path uses above. - const prefixes = feignPrefixByClassId.get(enclosingClass.id) ?? - prefixByClassId.get(enclosingClass.id) ?? ['']; + const prefixes = feignPrefixes ?? prefixByClassId.get(enclosingClass.id) ?? ['']; for (const prefix of prefixes) { out.push({ role: 'consumer', diff --git a/gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts b/gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts new file mode 100644 index 000000000..ceb1f92e1 --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts @@ -0,0 +1,1400 @@ +/** + * Kotlin binding for the language-agnostic constant resolver (#2391 core). + * + * Supplies the two Kotlin-specific pieces — {@link resolveKotlinImport} (import + * specifier → file, honoring JVM package rules) and + * {@link extractKotlinModuleConstants} (tree → {@link ModuleConstants}) — plus + * the folding entry points {@link resolveKotlinConstant} and + * {@link foldKotlinOperands}, so callers stay language-oblivious. + * + * WHAT IS ACTUALLY SHARED WITH THE AGNOSTIC CORE. One value — + * {@link MAX_FOLD_LENGTH} — and five types. The core's own `resolveConstant` / + * `resolveOperands` are NOT called: the fold state machine below (cycle guard, + * success memo, depth caps, operand concatenation — roughly 200 of this file's + * lines) is a local fork, close enough to `java-const-resolver.ts`'s already + * forked copy that the two read as the same code with the language name + * swapped. + * + * That fork is a consequence, not an oversight. The core keys its maps by + * SIMPLE name, and a Kotlin operand can be a QUALIFIED reference at any + * position (`X = ApiPaths.Y + "/tail"`); handed to the core, `ApiPaths.Y` misses + * every map and floors the whole chain to null — see {@link computeKotlinFold}, + * which resolves operands through the qualified-aware walk for exactly this + * reason. The import chase is Kotlin-specific too: a member import is spelled + * identically to a type import, so {@link resolveImportedName} has to try both + * readings, and the core exposes no hook for that. Java forked first on the same + * grounds. Teaching the core qualified names, and retiring both copies against + * it, is the standing follow-up; until then the honest description of this file + * is "a second fork", not "a binding over a shared fold". + * + * Kotlin shares the JVM package/import model with Java, so this binding mirrors + * `java-const-resolver.ts` in structure, naming and skip-floor discipline. The + * four places Kotlin genuinely differs are handled explicitly, not translated: + * + * 1. **Where a constant can live.** Java has one carrier (`static final` on a + * type). Kotlin has three: a top-level `const val`/`val`, a member of an + * `object`, and a member of a `companion object` — the last is referenced + * through its ENCLOSING class (`Holder.NAME`), not through `Companion`. + * 2. **No `String` type gate.** Kotlin infers property types, so + * `const val ORDERS = "/orders"` carries no type node to check. The + * initializer decides: anything {@link parseKotlinConstOperands} cannot fold + * to a string (a number, a call, a template) drops the constant. + * 3. **File names and directories are free.** `object ApiPaths` may live in + * `Constants.kt`, and a file's `package` need not match the directory it + * sits in, so a `/.kt` PATH lookup is a convention and not a + * rule. The authority is each file's DECLARED `package`, which + * {@link extractKotlinModuleConstants} records and {@link resolveKotlinImport} + * requires an exact match on; the path is only a tie-break among files that + * already declare the right package. + * 4. **Member imports are unmarked.** Java spells them `import static a.b.C.F`; + * Kotlin writes `import a.b.C.F`, which is byte-identical to a type import + * of a class `F` in package `a.b.C`. Nothing in the syntax says which, so + * the fold tries both readings (see `resolveImportedName`) instead of + * guessing from casing. + * 5. **Any identifier may be backtick-quoted.** `` package com.example.`api` `` + * and `package com.example.api` are the SAME package to the compiler, and a + * keyword segment (`` com.example.`fun` ``) can only be spelled the quoted + * way. The grammar keeps the backticks in the node text, so every identifier is + * read through {@link unquoteKotlinIdentifier} before it becomes a map key + * or a lookup name — see that function for what a verbatim comparison cost. + * + * THREE PLACES THIS BINDING NO LONGER MIRRORS JAVA, each because the mirrored + * behavior was wrong rather than merely different, and each open as a Java + * follow-up rather than fixed here: + * + * * `java-const-resolver.ts` flattens nested types into one file-level + * namespace and argues the collision away — "qualified refs carry the class + * name, so nesting only matters for same-name fields, which flatten + * last-wins". The argument does not hold: the collision is one level BELOW + * the qualification, in the initializer, so a fully qualified `A.ROUTE` whose + * initializer names a bare sibling `BASE` still resolves through whichever + * same-named sibling was walked last. {@link extractKotlinModuleConstants} + * keys by Kotlin's own visibility instead. + * * Java's import resolution can lean on the `/.java` layout the + * language enforces. Kotlin's cannot, and inferring the package from the path + * lets a path-suffix twin outrank the real declaration — so + * {@link resolveKotlinImport} reads the declared `package` instead. + * * Java's fold entry points take a file and a name, because a Java `static + * final` reachable by simple name is reachable that way from anywhere in the + * file. A Kotlin COMPANION member is not: it is bound unqualified only inside + * its enclosing class body. {@link foldKotlinOperands} therefore also takes + * the enclosing type chain of the reference site, which is what lets the + * binding answer a bare reference by Kotlin's scoping rather than by "whoever + * was walked last" — see {@link qualifyKotlinRefInEnclosingTypes}. + * + * Constant shapes this binding harvests: + * + * const val TOP_LEVEL = "/api/v1" // file top level + * object ApiPaths { // object member + * const val BASE = "/api/v1" + * val ORDERS = BASE + "/orders" + * } + * class Holder { companion object { const val H = "/h" } } // → Holder.H + * + * Reference shapes at annotation sites this binding resolves: + * @PostMapping(ApiPaths.ORDERS) // qualified + * @PostMapping(com.example.app.api.ApiPaths.ORDERS) // FQN-qualified + * @PostMapping(ORDERS) // single-name import + * @PostMapping(ApiPaths.BASE + "/orders") // inline concat + * + * Which ANNOTATIONS count as routes is a separate question this module has no + * say in — `spring-shared.ts` owns that map. Folding and annotation recognition + * compose; neither implies the other. + * + * Keying (parity with the Java and Python bindings): the repo map is keyed by + * unique POSIX file path, and an import that cannot be pinned to exactly one + * file returns null (skip floor), never a wrong path. A missing route is a + * missing fact; a wrongly folded one is a false edge in the graph. "Exactly one + * file" is decided from the DECLARED package, not from the path: a path is a + * repository-layout accident that any decoy directory can imitate, whereas the + * `package` header is the declaration the compiler itself resolves against. + * + * POSIX keys are a PRECONDITION this module cannot check cheaply, so it is + * enforced at the one boundary that produces them: `http-patterns/kotlin.ts` + * normalizes separators on both the write side (the `prepareRepo` map keys) and + * the read side (`scan`'s `fileRel`). It has to, because the orchestrator's file + * list comes from glob v13, which has no `posix: true` and joins with the + * platform separator — so on Windows the keys arrive backslashed and every + * `/.kt` test in {@link resolveKotlinImport} would miss, silently + * disabling cross-file folding on that platform alone. Normalizing INSIDE this + * module instead cannot work: the resolver returns the key it matched, and a + * normalized return value would then miss in a map that was never normalized. + * + * WHERE THIS IS WIRED. Java reaches its binding from BOTH layers: the group + * extractor (`group/extractors/http-patterns/java.ts`) and the ingestion + * provider (`languages/java.ts`, via `extractModuleConstants` + + * `foldRoutePathOperands`). Kotlin is wired into the GROUP layer only, because + * the ingestion fold in `pipeline-phases/parse-impl.ts` runs exclusively over + * `decoratorRoutes` — and `languages/kotlin.ts` declares no + * `extractDecoratorRoutes`, since the ingestion Spring extractor (`spring.ts`) + * is bound to `tree-sitter-java` and its node types. Declaring the constant + * hooks on the Kotlin provider today would harvest a map on every Kotlin file + * that nothing consumes. An ingestion-side Kotlin route extractor is the + * prerequisite; when it lands, this binding is what its provider hooks should + * point at, and no change here is needed. + */ + +import type Parser from 'tree-sitter'; +import { unquoteSpringLiteral } from './spring-shared.js'; +import { + MAX_FOLD_LENGTH, + type ImportBinding, + type ModuleConstants, + type Operand, + type RepoConstants, +} from './constant-resolver.js'; + +export type { + ImportBinding, + ModuleConstants, + Operand, + RepoConstants, +} from './constant-resolver.js'; + +/** + * What {@link extractKotlinModuleConstants} returns: the agnostic + * {@link ModuleConstants} plus the one piece of per-file metadata JVM import + * resolution cannot be honest without — the file's declared `package`. + * + * Deliberately a KOTLIN-LOCAL widening rather than a field on the shared type. + * `ModuleConstants` is consumed by the Java, JS and Python bindings too, and + * none of them needs this: Python resolves imports from the module path, and + * Java's `package` is already pinned by the `/.java` rule the + * language enforces. Adding a required field there would force three unrelated + * bindings to fill it in; adding an optional one would put a Kotlin-shaped hole + * in a type whose whole point is language neutrality. + * + * Read the metadata through {@link declaredPackageOf} and + * {@link unfoldableDeclarationsOf}, never by field access: a + * {@link RepoConstants} is typed over the agnostic shape, so an entry that some + * other producer put there carries no package and must be REJECTED as a + * candidate rather than silently treated as the default package. Missing + * unfoldable-declaration metadata instead means "none known", preserving the + * agnostic entry's existing behavior. + */ +export interface KotlinModuleConstants extends ModuleConstants { + /** The file's declared `package`, or `''` for the default package. */ + readonly packageName: string; + /** Declaration keys whose initializer cannot be folded. */ + readonly unfoldableDeclarations: ReadonlySet; +} + +/** + * The declared `package` of the file `mc` describes, or null when the entry did + * not come from {@link extractKotlinModuleConstants} and therefore cannot be + * matched against an import specifier. + */ +function declaredPackageOf(mc: ModuleConstants | undefined): string | null { + const declared = (mc as KotlinModuleConstants | undefined)?.packageName; + return typeof declared === 'string' ? declared : null; +} + +const NO_UNFOLDABLE_DECLARATIONS: ReadonlySet = new Set(); + +/** + * Kotlin declaration keys known to exist but not fold, or an empty set when + * `mc` came from another language binding. + */ +export function unfoldableDeclarationsOf(mc: ModuleConstants | undefined): ReadonlySet { + const declarations = (mc as KotlinModuleConstants | undefined)?.unfoldableDeclarations; + return declarations instanceof Set ? declarations : NO_UNFOLDABLE_DECLARATIONS; +} + +/** Source extensions a Kotlin declaration can live in. */ +const KOTLIN_EXTENSIONS = ['.kt', '.kts'] as const; + +/** + * The name a backtick-quoted Kotlin identifier denotes: `` `api` `` → `api`. + * + * Quotes are spelling, not part of the name. tree-sitter-kotlin keeps them in + * node text, so every identifier that becomes a map key or lookup is read + * through here. Applied per dot-separated segment — a quoted identifier cannot + * contain `.`. Both the declaration side ({@link declaredPackage}) and the + * import side ({@link resolveKotlinImport}) are normalized, because either may + * carry the quotes while the other spells the same name plainly. + */ +export function unquoteKotlinIdentifier(text: string): string { + return text.length >= 2 && text.startsWith('`') && text.endsWith('`') ? text.slice(1, -1) : text; +} + +/** {@link unquoteKotlinIdentifier} applied to every segment of a dotted name. */ +function unquoteKotlinDottedName(text: string): string { + return text.includes('`') ? text.split('.').map(unquoteKotlinIdentifier).join('.') : text; +} + +/** + * Recursion ceiling for {@link parseKotlinConstOperands}, counted in `+` links. + * + * This bounds SYNTAX depth, not resolution: `A + B + C` nests one + * `additive_expression` per link, so the cap is really "how long a concatenation + * may one initializer be". Deliberately loose — generated route tables do + * concatenate a dozen fragments, and overrunning costs a skipped route, so the + * cap is a guard against pathological input rather than a statement about + * reasonable code. + */ +const MAX_OPERAND_PARSE_DEPTH = 64; + +/** + * Recursion ceiling for the fold, counted in REFERENCE hops (`A = B`, `B = C`). + * + * Larger than the agnostic core's own `MAX_RESOLVE_DEPTH` (8), which is + * module-private in `constant-resolver.ts` and therefore cannot simply be + * reused, and equal to the value the Java binding spells inline. It backstops + * the cycle guard, which terminates loops but not a long acyclic chain; the + * memo makes reaching it cheap. Both caps floor to null, i.e. to a skipped + * route. + */ +const MAX_FOLD_DEPTH = 32; + +/** + * Cheap content gate: can this Kotlin file DEFINE a string constant that a route + * annotation might reference? + * + * Exported so every caller uses the same predicate and none can disagree with + * {@link extractKotlinModuleConstants} about which files carry constants — the + * defect class the Java binding's shared `isJavaConstantFile` exists to prevent. + * + * Arms, all intended to be WIDER than the extractor (a gate may over-admit — it + * only costs a parse — while rejecting a file the extractor would accept costs a + * fact): + * - `const val NAME [: T] =`. `const` is legal only at a file's top level or in + * an `object`/`companion object`, i.e. exactly the carriers the extractor + * harvests, so this arm needs no scope check. + * - an `object` (or `companion object`) declaration together with a `val NAME =` + * binding. A non-`const` `val` is the other half of the extractor's input and + * carries no keyword of its own; requiring an `object` nearby keeps a file + * whose only `val`s are function locals from costing a parse. It still admits + * a top-level `val` in a file that happens to declare an object elsewhere, + * which is the harmless direction. + * - a `val NAME =` binding at file scope. A small lexical walk tracks braces + * and parentheses while skipping comments and literals, admitting the + * top-level non-`const` property the extractor harvests without turning every + * function-local `val` or constructor property into an extra parse. + * + * Both name arms accept a BACKTICK-QUOTED identifier as well as a bare one, + * because the extractor does: `unquoteKotlinIdentifier` strips the quoting + * everywhere a name becomes a key, so `const val \`ORDERS\` = "/orders"` is a + * constant this module resolves. A gate that matched only `\w+` rejected the + * file outright and the reference floored to skip — a gate narrower than the + * extractor, which is the one direction the arms above are meant to exclude. + */ +const KOTLIN_NAME = String.raw`(?:\w+|\`[^\`\n]+\`)`; +const CONST_VAL_AT = new RegExp(String.raw`const\s+val\s+${KOTLIN_NAME}(?=\s|:|=|$)`, 'y'); +const VAL_DECLARATION_AT = new RegExp(String.raw`val\s+${KOTLIN_NAME}(?=\s|:|=|by\b|$)`, 'y'); + +/** Is `source[index...]` the keyword `word`, rather than part of an identifier? */ +function keywordAt(source: string, index: number, word: string): boolean { + if (!source.startsWith(word, index)) return false; + const before = index === 0 ? '' : source[index - 1]; + const after = source[index + word.length] ?? ''; + return !/[\w$]/.test(before) && !/[\w$]/.test(after); +} + +/** Test a sticky declaration pattern at one source offset without slicing. */ +function declarationAt(pattern: RegExp, source: string, index: number): boolean { + pattern.lastIndex = index; + return pattern.test(source); +} + +export function isKotlinConstantFile(source: string): boolean { + let braces = 0; + let parens = 0; + let blockCommentDepth = 0; + let sawObject = false; + + for (let i = 0; i < source.length; i++) { + if (blockCommentDepth > 0) { + if (source.startsWith('/*', i)) { + blockCommentDepth++; + i++; + } else if (source.startsWith('*/', i)) { + blockCommentDepth--; + i++; + } + continue; + } + + if (source.startsWith('//', i)) { + const newline = source.indexOf('\n', i + 2); + if (newline < 0) break; + i = newline; + continue; + } + if (source.startsWith('/*', i)) { + blockCommentDepth = 1; + i++; + continue; + } + + const quote = source[i]; + if (source.startsWith('"""', i)) { + const end = source.indexOf('"""', i + 3); + if (end < 0) break; + i = end + 2; + continue; + } + if (quote === '"' || quote === "'") { + for (i++; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === quote) break; + } + continue; + } + if (quote === '`') { + const end = source.indexOf('`', i + 1); + if (end < 0) break; + i = end; + continue; + } + + if (quote === '{') { + braces++; + continue; + } + if (quote === '}') { + braces = Math.max(0, braces - 1); + continue; + } + if (quote === '(') { + parens++; + continue; + } + if (quote === ')') { + parens = Math.max(0, parens - 1); + continue; + } + + if (keywordAt(source, i, 'object')) { + sawObject = true; + i += 'object'.length - 1; + continue; + } + if (keywordAt(source, i, 'const') && declarationAt(CONST_VAL_AT, source, i)) return true; + if (keywordAt(source, i, 'val') && declarationAt(VAL_DECLARATION_AT, source, i)) { + if (sawObject || (braces === 0 && parens === 0)) return true; + i += 'val'.length - 1; + } + } + return false; +} + +/** Does `key` name the file `.kt` / `.kts`? */ +function isFileNamedAfterDeclaration(key: string, asPath: string): boolean { + for (const ext of KOTLIN_EXTENSIONS) { + const candidate = `${asPath}${ext}`; + if (key === candidate || key.endsWith(`/${candidate}`)) return true; + } + return false; +} + +/** + * Does the file `mc` describes declare a top-level entity called `name` — an + * `object`/companion carrier whose members are keyed `name.`, or a + * top-level constant keyed `name` outright? + * + * A true result is strong enough to select a unique declaring file before path + * fallbacks: the tested key set is a superset of every local key the fold may + * subsequently read for that imported name. Two matching files are therefore + * ambiguous; one is authoritative. A miss falls back to the conservative path + * heuristics below, whose result is still verified by the actual map lookup. + */ +function declaresTopLevelName(mc: ModuleConstants, name: string): boolean { + const prefix = `${name}.`; + for (const map of [mc.literals, mc.exprs]) { + if (map.has(name)) return true; + for (const key of map.keys()) if (key.startsWith(prefix)) return true; + } + for (const key of unfoldableDeclarationsOf(mc)) { + if (key === name || key.startsWith(prefix)) return true; + } + return false; +} + +/** Files and declaration ownership for one exact Kotlin package. */ +export interface KotlinPackageConstants { + readonly files: readonly string[]; + /** Unique declaring file, or null when the package declares the name twice. */ + readonly declarers: ReadonlyMap; +} + +/** One exact package-qualified declaration and its in-file lookup key. */ +interface KotlinImportTarget { + readonly fileKey: string; + readonly localName: string; +} + +/** + * Repo-wide projections reused by every fold in one extraction run. + * + * `repo` includes importing files overlaid by `scan`; `constantKeys` and + * `byPackage` include only files that define a foldable or explicitly + * unfoldable declaration, preserving import ambiguity semantics. + */ +export interface KotlinConstantIndex { + readonly repo: RepoConstants; + readonly constantKeys: ReadonlySet; + readonly byPackage: ReadonlyMap; + /** Exact FQN → unique file/local key, or null when the FQN is duplicated. */ + readonly byFqn: ReadonlyMap; +} + +/** Does this file contribute declarations to Kotlin import ambiguity? */ +function contributesKotlinConstants(mc: ModuleConstants): boolean { + return mc.literals.size > 0 || mc.exprs.size > 0 || unfoldableDeclarationsOf(mc).size > 0; +} + +/** Top-level names declared by one file (`Outer.X` contributes `Outer`). */ +function topLevelDeclarationNames(mc: ModuleConstants): Set { + const names = new Set(); + for (const map of [mc.literals, mc.exprs]) { + for (const key of map.keys()) { + const dot = key.indexOf('.'); + names.add(dot < 0 ? key : key.slice(0, dot)); + } + } + for (const key of unfoldableDeclarationsOf(mc)) { + const dot = key.indexOf('.'); + names.add(dot < 0 ? key : key.slice(0, dot)); + } + return names; +} + +/** Every declaration key recorded for a file, foldable or not. */ +function declarationKeys(mc: ModuleConstants): Set { + return new Set([...mc.literals.keys(), ...mc.exprs.keys(), ...unfoldableDeclarationsOf(mc)]); +} + +/** Build the immutable import projections once for a repo constant map. */ +export function buildKotlinConstantIndex(repo: RepoConstants): KotlinConstantIndex { + const constantKeys = new Set(); + const byFqn = new Map(); + const mutablePackages = new Map< + string, + { files: string[]; declarers: Map } + >(); + + for (const [key, mc] of repo) { + if (!contributesKotlinConstants(mc)) continue; + constantKeys.add(key); + const packageName = declaredPackageOf(mc); + if (packageName === null) continue; + let bucket = mutablePackages.get(packageName); + if (!bucket) { + bucket = { files: [], declarers: new Map() }; + mutablePackages.set(packageName, bucket); + } + bucket.files.push(key); + for (const name of topLevelDeclarationNames(mc)) { + if (!bucket.declarers.has(name)) bucket.declarers.set(name, key); + else if (bucket.declarers.get(name) !== key) bucket.declarers.set(name, null); + } + for (const declaration of declarationKeys(mc)) { + const parts = declaration.split('.'); + // A member key `Outer.Inner.Q` proves the file declares both owner paths + // as well as the member itself. This lets imports of nested objects and + // their members retain the complete in-file lookup path. + for (let length = 1; length <= parts.length; length++) { + const localName = parts.slice(0, length).join('.'); + const fqn = packageName === '' ? localName : `${packageName}.${localName}`; + const existing = byFqn.get(fqn); + if (existing === undefined) byFqn.set(fqn, { fileKey: key, localName }); + else if (existing !== null && existing.fileKey !== key) byFqn.set(fqn, null); + } + } + } + + return { repo, constantKeys, byPackage: mutablePackages, byFqn }; +} + +/** + * Add one scan-time file without rebuilding the base index when it only imports + * constants. A newly discovered declaration is rare and rebuilds once for that + * file's scan, never once per route. + */ +export function overlayKotlinConstantIndex( + index: KotlinConstantIndex, + fileKey: string, + mc: ModuleConstants, +): KotlinConstantIndex { + const repo = new Map(index.repo); + const replacing = repo.has(fileKey); + repo.set(fileKey, mc); + if (!replacing && !contributesKotlinConstants(mc)) return { ...index, repo }; + return buildKotlinConstantIndex(repo); +} + +/** + * Map a fully-qualified import specifier to the unique file key it refers to, or + * null when it cannot be pinned to exactly one file. + * + * A specifier is split at its last dot into the package it names and the + * declaration inside it (`com.example.app.api` + `ApiPaths`). Resolution then + * runs in three steps, all of them "unique or nothing": + * + * 0. **Declared package** — only files whose `package` header is EXACTLY the + * sought package can carry the declaration (compared after + * {@link unquoteKotlinIdentifier}, since backtick quoting is spelling and + * not identity). This is the authority, and it + * is checked first. Kotlin does not require a file's directory to match its + * package, so the reverse test — "does this path end with the package?" — + * answers a different question, one any decoy directory can satisfy: a file + * at `src/x/com/example/api/ApiPaths.kt` declaring `package x.com.example.api` + * is not `com.example.api.ApiPaths` and must never be folded as it, and a + * path-suffix test also lets a root-level `package data` be impersonated by + * `…/com/example/data/`. An entry with no recorded package is rejected, not + * assumed to be the default package. + * 1. **Declared name** — when exactly one file declares the sought name, use + * it. When two do, the FQN itself is duplicated in the repository and names + * no single declaration, so return null. This is the general form of the + * same-FQN check step 2 could only make for files that happen to follow the + * file-name convention, and it is what stops a `src/test/…` copy of a + * production constant from being folded into a production route. + * 2. **File named after the declaration** — when declaration metadata found no + * owner, try the package-matching file ending + * `com/example/app/api/ApiPaths.kt`. Kotlin does not require this (`object + * ApiPaths` may live in `Constants.kt`), so it is only a fallback candidate; + * the subsequent map lookup must still prove that it carries the value. + * 3. **Sole file in the package** — when declaration metadata cannot identify + * the name, use the unique package-matching candidate. The set passed in + * contains files with foldable or explicitly unfoldable declarations, so + * unrelated files cannot create ambiguity once step 1 identifies a unique + * declarer. With 2+ unidentified candidates it returns null. + * + * Steps 2 and 3 can still hand back a file that does not declare the wanted name + * (its package is right and it is the only candidate, but the name lives + * elsewhere or nowhere). That remains safe by construction: the fold looks the + * name up in that file's map, misses, and returns null. + * + * A "nearest shared directory" tie-break is deliberately NOT applied when a step + * has several candidates, for the reason the Java binding records: the JVM + * resolves duplicate FQNs by classpath order, not directory proximity, so a test + * fixture copy sitting closer in the tree can outrank the real dependency and + * yield a silently wrong literal. In a resolver whose whole contract is + * skip-or-correct, a plausible guess is the one answer that cannot be allowed. + * + * This can no longer be typed as the agnostic {@link ModuleConstants} consumer's + * `ImportResolver`, whose signature carries only file KEYS: deciding a candidate + * on its declared package needs the map those keys index. Nothing is lost — the + * core's own fold is not used here either (see the module header), and the + * alternative is a resolver that must guess from a path. + */ +export function resolveKotlinImport( + _importingFileKey: string, + rawModuleSpec: string, + candidateKeys: ReadonlySet, + repo: RepoConstants, +): string | null { + // Normalized here as well as at extraction, so the function answers the same + // question however a caller spells the specifier. + const moduleSpec = unquoteKotlinDottedName(rawModuleSpec); + const lastDot = moduleSpec.lastIndexOf('.'); + const packageName = lastDot < 0 ? '' : moduleSpec.slice(0, lastDot); + const simpleName = lastDot < 0 ? moduleSpec : moduleSpec.slice(lastDot + 1); + + // Step 0 + step 1 in one pass over the candidates. + const inPackage: string[] = []; + let declaring: string | null = null; + for (const key of candidateKeys) { + const mc = repo.get(key); + if (!mc || declaredPackageOf(mc) !== packageName) continue; + inPackage.push(key); + if (declaresTopLevelName(mc, simpleName)) { + if (declaring !== null) return null; // 2+ files declare this FQN + declaring = key; + } + } + if (inPackage.length === 0) return null; + if (declaring !== null) return declaring; + if (inPackage.length === 1) return inPackage[0]; // steps 2 and 3 agree + + // Step 2: the file-name convention, as a tie-break among valid candidates. + const asPath = moduleSpec.replace(/\./g, '/'); + let named: string | null = null; + for (const key of inPackage) { + if (!isFileNamedAfterDeclaration(key, asPath)) continue; + if (named !== null) return null; // 2+ files spell the convention + named = key; + } + // Step 3 is "the sole candidate", already returned above. + return named; +} + +/** Indexed equivalent of {@link resolveKotlinImport}, with identical fallbacks. */ +export function resolveKotlinImportWithIndex( + rawModuleSpec: string, + index: KotlinConstantIndex, +): string | null { + return resolveKotlinImportTarget(rawModuleSpec, index)?.fileKey ?? null; +} + +/** Resolve an import to both its file and complete in-file declaration path. */ +function resolveKotlinImportTarget( + rawModuleSpec: string, + index: KotlinConstantIndex, +): KotlinImportTarget | null { + const moduleSpec = unquoteKotlinDottedName(rawModuleSpec); + const lastDot = moduleSpec.lastIndexOf('.'); + const packageName = lastDot < 0 ? '' : moduleSpec.slice(0, lastDot); + const simpleName = lastDot < 0 ? moduleSpec : moduleSpec.slice(lastDot + 1); + const bucket = index.byPackage.get(packageName); + // Preserve the top-level interpretation whenever the exact declared package + // exists. A parent package may legally contain nested objects whose joined + // path spells the same FQN; letting that projection win would make a nested + // decoy override the real top-level declaration. + if (!bucket) return index.byFqn.get(moduleSpec) ?? null; + + if (bucket.declarers.has(simpleName)) { + const fileKey = bucket.declarers.get(simpleName); + return fileKey === null || fileKey === undefined ? null : { fileKey, localName: simpleName }; + } + if (bucket.files.length === 1) return { fileKey: bucket.files[0], localName: simpleName }; + + const asPath = moduleSpec.replace(/\./g, '/'); + let named: string | null = null; + for (const key of bucket.files) { + if (!isFileNamedAfterDeclaration(key, asPath)) continue; + if (named !== null) return null; + named = key; + } + return named === null ? null : { fileKey: named, localName: simpleName }; +} + +/** + * Is `node` a Kotlin string literal, and if so what value does the route layer + * give it? + * + * Two rejections, both floors rather than guesses: + * - **String templates.** `"$base/orders"` parses as a `string_literal` whose + * children include an interpolation alongside the `string_content` runs. + * Joining the content runs would silently DELETE the interpolated part and + * publish `/orders` — a path the application does not serve. Any named child + * that is not `string_content` means the value is not statically knowable, so + * the literal is refused. (The same test makes the function safe against a + * grammar that splits escape sequences into their own nodes: it would floor + * to skip, never to a de-escaped path.) + * - **Multi-line raw strings.** A single-line `"""/api"""` is exact — unlike a + * Java text block, a Kotlin raw string performs no escape processing and no + * incidental-indentation stripping, so it folds to precisely its content. A + * multi-line one carries newlines (and usually a `.trimIndent()` call this + * layer cannot fold), so it is refused. + * + * Otherwise the quotes are sliced off the RAW TEXT via + * {@link unquoteSpringLiteral} — the same function the literal path uses — so + * `@GetMapping(ApiPaths.USER_REGEX)` and `@GetMapping("/user/{id:\\d+}")` emit + * the same path for the same Kotlin source. + */ +function stringLiteralValue(node: Parser.SyntaxNode): string | null { + if (node.type !== 'string_literal') return null; + for (const child of node.namedChildren) { + if (child.type !== 'string_content') return null; + } + const raw = node.text; + if (raw.startsWith('"""') && raw.includes('\n')) return null; + return unquoteSpringLiteral(raw); +} + +/** + * Flatten a navigation expression (`ApiPaths`, `com.example.app.ApiPaths`) to + * its dotted text, or null when any segment is not a plain identifier (calls, + * `this`, indexing, safe navigation — not a constant shape). + */ +function flattenNavigation(node: Parser.SyntaxNode): string | null { + if (node.type === 'simple_identifier') return unquoteKotlinIdentifier(node.text); + if (node.type === 'navigation_expression') { + const target = node.namedChild(0); + const suffix = node.namedChildren.find((c) => c.type === 'navigation_suffix'); + const field = suffix?.namedChildren.find((c) => c.type === 'simple_identifier'); + if (target && field) { + const head = flattenNavigation(target); + return head === null ? null : `${head}.${unquoteKotlinIdentifier(field.text)}`; + } + } + return null; +} + +/** + * Parse a Kotlin constant initializer (or an inline annotation argument) into an + * operand list, or null when it is not a foldable string expression. Handles a + * bare string literal, a bare identifier (`X = Y`), a qualified reference + * (`X = ApiPaths.Y` — recorded as ONE ref named `ApiPaths.Y`), and + * left-associative `+` chains of the three. Everything else — numbers, calls, + * `when`/`if` expressions, templates, `buildString` — returns null, which makes + * the constant unresolvable (→ skip floor), never a wrong value. + * + * A chain nests: tree-sitter-kotlin parses `A + B + C` as + * `additive_expression(additive_expression(A, B), C)`, so every node here has + * exactly two operands and arbitrary-length chains fold by recursion. The same + * node type also carries `-`, which is not a string operation, so a `+` token + * must be present. + * + * A PARENTHESIZED operand (`(A + B) + "/c"`) is deliberately NOT unwrapped, + * matching `parseJavaConstOperands`, which has no parenthesis arm either. The + * shape is vanishingly rare in a route annotation and the cost of omitting it is + * a skipped route, not a wrong one; adding it to both bindings at once is the + * only way to keep them in parity, so it is left to a follow-up. + */ +export function parseKotlinConstOperands( + node: Parser.SyntaxNode | null | undefined, + depth = 0, +): Operand[] | null { + if (!node) return null; + if (depth > MAX_OPERAND_PARSE_DEPTH) return null; + if (node.type === 'string_literal') { + const value = stringLiteralValue(node); + return value === null ? null : [{ kind: 'literal', value }]; + } + if (node.type === 'simple_identifier') { + return [{ kind: 'ref', name: unquoteKotlinIdentifier(node.text) }]; + } + if (node.type === 'navigation_expression') { + const name = flattenNavigation(node); + return name === null ? null : [{ kind: 'ref', name }]; + } + // `additive_expression` covers both `+` and `-` in tree-sitter-kotlin; only a + // `+` chain concatenates strings. + if (node.type === 'additive_expression') { + if (!(node.children ?? []).some((c) => c.type === '+')) return null; + const operandNodes = node.namedChildren; + if (operandNodes.length !== 2) return null; + const left = parseKotlinConstOperands(operandNodes[0], depth + 1); + const right = parseKotlinConstOperands(operandNodes[1], depth + 1); + if (left === null || right === null) return null; + return [...left, ...right]; + } + return null; +} + +/** The `val`/`var` keyword a property declaration binds with, or null. */ +function bindingKind(property: Parser.SyntaxNode): string | null { + return property.children.find((c) => c.type === 'binding_pattern_kind')?.text ?? null; +} + +/** + * The initializer expression of a property declaration, or null when it has + * none. + * + * Reads the `=` that is a DIRECT child of the `property_declaration`, so a + * custom getter (`val X: String get() = "/g"`, whose `=` lives under `getter`) + * and a delegate (`val X by lazy { … }`, which has no `=` at all) both yield + * null. Both are computed at access time and are not constants. + */ +function initializerOf(property: Parser.SyntaxNode): Parser.SyntaxNode | null { + let equalsIndex = -1; + for (let i = 0; i < property.childCount; i++) { + if (property.child(i)?.type === '=') { + equalsIndex = i; + break; + } + } + if (equalsIndex < 0) return null; + for (let i = equalsIndex + 1; i < property.childCount; i++) { + const child = property.child(i); + if (child?.isNamed) return child; + } + return null; +} + +/** + * One `val` declaration, captured before anything is written to the file's + * namespace so that the DECLARING SCOPE of every initializer is known regardless + * of the order the declarations appear in. + */ +interface KotlinConstDeclaration { + /** The declaration's simple name. */ + readonly name: string; + /** `.`, or null for a top-level declaration. */ + readonly qualified: string | null; + /** + * The qualified-key prefixes in LEXICAL scope for this declaration's + * initializer, innermost first (`['Outer.Inner', 'Outer']`). Empty at file + * level. + */ + readonly scopes: readonly string[]; + /** + * Is the simple name a FILE-LEVEL binding — one any reference in the file can + * use unqualified? True only for a top-level `val`. FALSE for a member of a + * named `object` (which every caller outside that object's body must qualify) + * and FALSE for a companion member, whose unqualified binding exists only + * inside its enclosing class body and is reached through + * {@link qualifyKotlinRefInEnclosingTypes} instead. + */ + readonly fileLevelName: boolean; + /** + * Does an unfoldable initializer here take a same-named IMPORT down with it? + * + * True wherever the declaration binds the simple name for at least some of the + * file — a top-level `val` (everywhere) or a companion member (inside its + * class). Deliberately wider than {@link fileLevelName}: a companion's shadow + * is scoped, but this map is not, and over-deleting an import can only cost a + * route, whereas under-deleting one publishes the imported value at a + * reference the compiler resolves to the unfoldable member. An `object` member + * shadows nothing and is false. + */ + readonly shadowsImport: boolean; + /** The parsed initializer, or null when it is not a foldable string. */ + readonly operands: readonly Operand[] | null; +} + +/** + * The file's declared `package`, or `''` when it declares none (default + * package). Shaped exactly like the import walk below: `package_header` holds + * one `identifier` whose `simple_identifier` children are the dotted segments. + * + * Each segment is unquoted (see {@link unquoteKotlinIdentifier}), so a package + * declared `` com.example.`api` `` is recorded — and therefore matched — as the + * same package an import spells `com.example.api`. + */ +function declaredPackage(root: Parser.SyntaxNode): string { + const header = root.children.find((c) => c.type === 'package_header'); + const identifier = header?.children.find((c) => c.type === 'identifier'); + if (!identifier) return ''; + return identifier.namedChildren + .filter((c) => c.type === 'simple_identifier') + .map((c) => unquoteKotlinIdentifier(c.text)) + .join('.'); +} + +/** + * Extract the declared package, file-level string constants and import bindings + * of one parsed Kotlin file into the {@link KotlinModuleConstants} shape the + * resolver consumes. + * + * Constants come from the three carriers Kotlin allows a caller to reach without + * an instance: file top level, `object` members, and `companion object` members. + * A `val` in a plain class or interface body is per-instance or abstract and is + * NOT collected — the Kotlin analogue of Java's `static final` requirement. `var` + * is rejected outright. + * + * KEYS FOLLOW KOTLIN'S OWN VISIBILITY, not a flattened namespace. Every constant + * is recorded under `.`, the spelling a qualified reference + * uses, with a companion member keyed under its ENCLOSING CLASS (`Holder.NAME`) + * because that is how Kotlin source refers to it — `Companion` never appears in + * a reference. The SIMPLE name is recorded only for a TOP-LEVEL `val`, the one + * carrier whose bare binding really does span the file. A member of a named + * `object` gets no bare key, because `BASE` alone does not name `A.BASE` from + * anywhere outside `object A`'s own body. Writing one anyway (as this binding + * and the Java one both used to) fabricates a binding the language does not + * have, and a fabricated key outranks the genuine `import com.example.api.Paths.ORDERS` + * that {@link computeKotlinFold} consults only after literals and expressions. + * + * A COMPANION member gets no bare key either: it is bound unqualified inside + * its enclosing class body and nowhere else. {@link qualifyKotlinRefInEnclosingTypes} + * rewrites a bare name to `.` when an enclosing type + * declares it, so the companion wins inside its own class and loses everywhere + * else. + * + * An initializer that names a SIBLING is resolved the same way, against its own + * scope chain, innermost first, before the file level: inside + * `object A { const val BASE = "/right"; const val ROUTE = BASE + "/m" }` the + * operand `BASE` is rewritten to `A.BASE`. Collecting every declaration before + * recording any keeps that independent of declaration order. + * + * A TOP-LEVEL initializer has an EMPTY scope chain, so its bare operands stay + * bare and resolve at file level — they must not pick up a companion key. + * + * A non-foldable rebind (`X = compute()`) DROPS X to unresolvable rather than + * leaving a stale literal — and drops a same-named import with it whenever the + * declaration shadows that import ANYWHERE (top level, or a companion inside its + * class). The import map has no scopes, so a companion's shadow is applied + * file-wide: the conservative direction, costing a route rather than publishing + * the imported value at a reference the compiler binds to the unfoldable member. + * An `object` member shadows nothing and must leave the import alone. + */ +export function extractKotlinModuleConstants(tree: Parser.Tree): KotlinModuleConstants { + const literals = new Map(); + const exprs = new Map(); + const imports = new Map(); + const unfoldableDeclarations = new Set(); + + // Pass 1: imports. + const walkImports = (node: Parser.SyntaxNode): void => { + if (node.type === 'import_header') { + // `import a.b.*` binds no single name — nothing to key the fold on, and + // guessing which package member a bare reference came from is exactly the + // wrong answer. Skipped, so such a reference floors to skip. + const isWildcard = node.children.some((c) => c.type === 'wildcard_import'); + const identifier = node.children.find((c) => c.type === 'identifier'); + if (!isWildcard && identifier) { + const segments = identifier.namedChildren + .filter((c) => c.type === 'simple_identifier') + .map((c) => unquoteKotlinIdentifier(c.text)); + if (segments.length >= 2) { + const spec = segments.join('.'); + const originalName = segments[segments.length - 1]; + const aliasNode = node.children + .find((c) => c.type === 'import_alias') + ?.namedChildren.find((c) => c.type === 'type_identifier'); + const alias = aliasNode ? unquoteKotlinIdentifier(aliasNode.text) : undefined; + // `module` is the specifier AS WRITTEN, complete. Kotlin does not mark + // member imports, so the fold — not the extractor — decides whether the + // trailing segment is a declaration or one of its members. + imports.set(alias ?? originalName, { module: spec, originalName }); + } + } + return; + } + for (const child of node.children ?? []) walkImports(child); + }; + walkImports(tree.rootNode); + + // Pass 2a: collect every declaration, writing nothing yet. Which member each + // unqualified operand means depends on the whole file, so no key can be + // written — and no operand rewritten — until the last declaration is in. + const declarations: KotlinConstDeclaration[] = []; + /** Declaring scope → the simple names it declares, foldable or not. */ + const membersByScope = new Map>(); + + const collectProperties = ( + body: Parser.SyntaxNode, + declaringType: string | null, + scopes: readonly string[], + fileLevelName: boolean, + shadowsImport: boolean, + ): void => { + for (const member of body.children ?? []) { + if (member.type !== 'property_declaration') continue; + if (bindingKind(member) !== 'val') continue; + const declaration = member.children.find((c) => c.type === 'variable_declaration'); + const nameNode = declaration?.namedChildren.find((c) => c.type === 'simple_identifier'); + if (!nameNode) continue; + const name = unquoteKotlinIdentifier(nameNode.text); + if (declaringType !== null) { + let members = membersByScope.get(declaringType); + if (!members) membersByScope.set(declaringType, (members = new Set())); + // Recorded even when the initializer does not fold: a sibling reference + // to an unfoldable member must resolve to that member and then MISS, + // not fall through to a same-named constant at file level. + members.add(name); + } + declarations.push({ + name, + qualified: declaringType === null ? null : `${declaringType}.${name}`, + scopes, + fileLevelName, + shadowsImport, + operands: parseKotlinConstOperands(initializerOf(member)), + }); + } + }; + + const bodyOf = (node: Parser.SyntaxNode): Parser.SyntaxNode | undefined => + node.children.find((c) => c.type === 'class_body'); + + /** The declared name of an `object_declaration` / `class_declaration`. */ + const typeNameOf = (node: Parser.SyntaxNode): string | null => { + const ident = node.children.find((c) => c.type === 'type_identifier'); + return ident ? unquoteKotlinIdentifier(ident.text) : null; + }; + + /** Append one simple type name to its enclosing qualified type path. */ + const nestedTypeName = (enclosingType: string | null, name: string | null): string | null => { + if (name === null) return enclosingType; + return enclosingType === null ? name : `${enclosingType}.${name}`; + }; + + /** Prepend a qualified scope unless it is already the innermost scope. */ + const withScope = (scope: string | null, scopes: readonly string[]): readonly string[] => + scope === null || scopes[0] === scope ? scopes : [scope, ...scopes]; + + const walkDeclarations = ( + node: Parser.SyntaxNode, + enclosingType: string | null, + scopes: readonly string[], + ): void => { + for (const child of node.children ?? []) { + if (child.type === 'object_declaration') { + const name = typeNameOf(child); + const body = bodyOf(child); + if (!body) continue; + // Carry the full path: a nested object member is `Outer.Inner.NAME`, not + // `Inner.NAME`. Inside the body a bare name searches that qualified + // scope first, then each enclosing type. + const declaredType = nestedTypeName(enclosingType, name); + const inner = withScope(declaredType, scopes); + collectProperties(body, declaredType, inner, false, false); + walkDeclarations(body, declaredType, inner); + continue; + } + if (child.type === 'companion_object') { + const body = bodyOf(child); + if (!body) continue; + // Referenced through the enclosing class (`Holder.NAME`), never through + // `Companion` — so the qualified alias is keyed on `enclosingType`. The + // simple name is bound inside that class body only, which is a SCOPE and + // not a file-level key: it is reached from the reference site by + // `qualifyKotlinRefInEnclosingTypes`, through this same `Holder.NAME`. + const inner = withScope(enclosingType, scopes); + collectProperties(body, enclosingType, inner, false, true); + walkDeclarations(body, enclosingType, inner); + continue; + } + if (child.type === 'class_declaration') { + // A class/interface body's own `val`s are per-instance or abstract, so + // only its nested objects and companion contribute constants. + const name = typeNameOf(child); + const body = bodyOf(child); + const declaredType = nestedTypeName(enclosingType, name); + if (body) walkDeclarations(body, declaredType, withScope(declaredType, scopes)); + continue; + } + walkDeclarations(child, enclosingType, scopes); + } + }; + + collectProperties(tree.rootNode, null, [], true, true); + walkDeclarations(tree.rootNode, null, []); + + // Pass 2b: rewrite each initializer's unqualified operands against the scope + // chain that encloses it, then record. Only a top-level declaration writes a + // bare key, so nothing here can collide across scopes; a companion's + // unqualified binding is applied at the reference site instead. + // A PARTIALLY qualified reference is resolved here too, not just a bare one: + // inside `object Outer`, the initializer `Inner.Q + "/m"` names `Outer.Inner.Q`, + // and taking a dotted name as already complete looked up a key nothing + // declares. Split at the last dot and prefix the scope onto the OWNER, so the + // bare case (`ownerSuffix === null`) stays exactly what it was. + const qualifyRef = (refName: string, scopes: readonly string[]): string => { + const lastDot = refName.lastIndexOf('.'); + const ownerSuffix = lastDot < 0 ? null : refName.slice(0, lastDot); + const member = lastDot < 0 ? refName : refName.slice(lastDot + 1); + for (const scope of scopes) { + const declaringType = ownerSuffix === null ? scope : `${scope}.${ownerSuffix}`; + if (membersByScope.get(declaringType)?.has(member)) return `${declaringType}.${member}`; + } + return refName; // file level, or unresolvable — the fold decides + }; + + for (const decl of declarations) { + const keys: string[] = []; + if (decl.fileLevelName) keys.push(decl.name); + if (decl.qualified !== null) keys.push(decl.qualified); + + if (decl.operands === null) { + for (const key of keys) { + literals.delete(key); + exprs.delete(key); + unfoldableDeclarations.add(key); + } + if (decl.shadowsImport) imports.delete(decl.name); + continue; + } + + const operands = decl.operands.map((op) => + op.kind === 'ref' ? { kind: 'ref' as const, name: qualifyRef(op.name, decl.scopes) } : op, + ); + const literalValue = + operands.length === 1 && operands[0].kind === 'literal' ? operands[0].value : null; + for (const key of keys) { + unfoldableDeclarations.delete(key); + if (literalValue !== null) { + literals.set(key, literalValue); + exprs.delete(key); + } else { + exprs.set(key, operands); + literals.delete(key); + } + } + } + + return { + literals, + exprs, + imports, + packageName: declaredPackage(tree.rootNode), + unfoldableDeclarations, + }; +} + +/** + * Per-fold state. Mirrors {@link resolveJavaConstant}'s, for the same reasons: + * + * - `memo` caches SUCCESSES only and is never popped, so a shared-descendant + * DAG (`X_k = X_{k+1} + X_{k+1}`) folds in O(nodes) instead of O(2^depth). + * A `null` may be transient — a name that cycles on one branch can resolve on + * another — so caching it would be unsound. + * - `visited` is the ACTIVE resolution stack, popped on unwind, so diamonds fold + * instead of false-cycling while true cycles still terminate. + * - `index` carries the constant-defining key set and exact-package declaration + * buckets built by `prepareRepo`. A public one-shot call can still build it + * lazily, while production folds reuse it across every route in the scan. + * Import-only files stay out of the candidate set so they cannot manufacture + * ambiguity. + */ +interface KotlinFoldState { + readonly index: KotlinConstantIndex; + readonly visited: Set; + readonly memo: Map; +} + +function newFoldState(repo: RepoConstants, index?: KotlinConstantIndex): KotlinFoldState { + return { + index: index ?? buildKotlinConstantIndex(repo), + visited: new Set(), + memo: new Map(), + }; +} + +/** + * Resolve a single Kotlin constant referenced in `fileKey` to its literal string + * value, folding `+` concatenation and following import chains via + * {@link resolveKotlinImport}, or null when it cannot be fully folded. + * + * `name` may be simple (`ORDERS`, resolved via a single-name import or a + * same-file constant) or qualified (`ApiPaths.ORDERS`, resolved via the type + * import plus the target file's qualified alias). + */ +export function resolveKotlinConstant( + fileKey: string, + name: string, + repo: RepoConstants, + depth = 0, + index?: KotlinConstantIndex, +): string | null { + return resolveWithState(fileKey, name, newFoldState(repo, index), depth); +} + +function resolveWithState( + fileKey: string, + name: string, + state: KotlinFoldState, + depth: number, +): string | null { + if (depth > MAX_FOLD_DEPTH) return null; + const guard = `${fileKey}::${name}`; + const memoized = state.memo.get(guard); + if (memoized !== undefined) return memoized; + if (state.visited.has(guard)) return null; // cycle: `name` is on the active stack + state.visited.add(guard); + try { + const result = computeKotlinFold(fileKey, name, state, depth); + if (result !== null) state.memo.set(guard, result); + return result; + } finally { + state.visited.delete(guard); + } +} + +/** + * Resolve a name bound by an import, trying both readings of the specifier. + * + * Kotlin writes a member import exactly like a type import, so + * `import com.example.app.api.ApiPaths.ORDERS` is syntactically + * indistinguishable from a type import of `ORDERS` in package + * `com.example.app.api.ApiPaths`. Rather than guess from casing — a convention, + * not a rule, and one that quietly breaks on `object apiPaths` or `const val + * Orders` — both readings are attempted and the first that actually RESOLVES + * wins. A reading that resolves to no constant simply falls through. + */ +function resolveImportedName( + fileKey: string, + imp: ImportBinding, + state: KotlinFoldState, + depth: number, +): string | null { + // Reading A: the specifier names the declaration itself (a top-level + // `const val`, or a type whose file we then search). + const direct = resolveKotlinImportTarget(imp.module, state.index); + if (direct !== null) { + const value = resolveWithState(direct.fileKey, direct.localName, state, depth); + if (value !== null) return value; + } + // Reading B: the specifier names a MEMBER of the declaration one segment up + // (`…ApiPaths.ORDERS` → member `ORDERS` of `ApiPaths`). + const dot = imp.module.lastIndexOf('.'); + if (dot <= 0) return null; + const ownerSpec = imp.module.slice(0, dot); + const owner = resolveKotlinImportTarget(ownerSpec, state.index); + if (owner === null) return null; + return resolveWithState(owner.fileKey, `${owner.localName}.${imp.originalName}`, state, depth); +} + +function computeKotlinFold( + fileKey: string, + name: string, + state: KotlinFoldState, + depth: number, +): string | null { + const { repo } = state.index; + // Qualified reference (`ApiPaths.ORDERS`): constants and imports are keyed by + // their IN-FILE name, so a dotted name never hits directly. Split head.tail, + // resolve the head through the importing file's type import, then look the + // member up in the target file under its declaring name. + // + // Unlike the Java binding there is NO bare-`tail` fallback: in Kotlin + // `Head.TAIL` means TAIL is a member of the object or companion `Head`, so a + // top-level `TAIL` in the target file is a different declaration and matching + // it would fabricate a value. + const dot = name.indexOf('.'); + if (dot > 0) { + const head = name.slice(0, dot); + const tail = name.slice(dot + 1); + const imp = repo.get(fileKey)?.imports.get(head); + if (imp) { + const target = resolveKotlinImportTarget(imp.module, state.index); + if (target === null) return null; + // `originalName` un-aliases `import … .ApiPaths as Paths`, so the lookup + // uses the declaring type's real name. + return resolveWithState(target.fileKey, `${target.localName}.${tail}`, state, depth + 1); + } + // Un-imported qualified name (FQN form `com.example.app.api.ApiPaths.ORDERS`): + // try the longest dotted prefix that resolves to a file. + const parts = name.split('.'); + for (let cut = parts.length - 2; cut >= 1; cut--) { + const fqn = parts.slice(0, cut + 1).join('.'); + const target = resolveKotlinImportTarget(fqn, state.index); + if (target !== null) { + const member = parts.slice(cut + 1).join('.'); + return resolveWithState(target.fileKey, `${target.localName}.${member}`, state, depth + 1); + } + } + // No import bound the head and no FQN prefix resolved — fall through. A + // dotted name is ALSO a valid key in this file's own maps, so a same-file + // qualified reference (`ApiPaths.ORDERS` inside the file declaring + // `object ApiPaths`) resolves below. + } + + // Name lookup: literals, then same-file expressions, then the import chase. + // Expressions are folded HERE rather than handed to the agnostic core because + // an operand may itself be a QUALIFIED reference (`X = ApiPaths.Y + "/tail"`) + // and the core only knows bare names: it would look `ApiPaths.Y` up in maps + // keyed by simple name, miss, and floor the whole chain to null. + const mc = repo.get(fileKey); + if (!mc) return null; + const literal = mc.literals.get(name); + if (literal !== undefined) return literal; + const expr = mc.exprs.get(name); + if (expr !== undefined) return foldOperands(fileKey, expr, state, depth + 1); + const imp = mc.imports.get(name); + if (imp !== undefined) return resolveImportedName(fileKey, imp, state, depth + 1); + return null; +} + +/** + * Concatenate an operand list, resolving each `ref` through the qualified-aware + * walk so `ApiPaths.BASE` works at every position, not just at the entry point. + * + * Bounded by {@link MAX_FOLD_LENGTH}: the depth cap bounds RECURSION but not + * OUTPUT, which grows multiplicatively (`X = A + A; A = B + B; …`), so a + * pathological chain would build a gigabyte-scale string before any cap fired. + * Overrun floors to null. + */ +function foldOperands( + fileKey: string, + operands: readonly Operand[], + state: KotlinFoldState, + depth: number, +): string | null { + let out = ''; + for (const op of operands) { + if (op.kind === 'literal') { + out += op.value; + } else { + const piece = resolveWithState(fileKey, op.name, state, depth); + if (piece === null) return null; + out += piece; + } + if (out.length > MAX_FOLD_LENGTH) return null; + } + return out; +} + +/** + * Rewrite one BARE reference to the enclosing type that binds it, or leave it + * bare when none does — the reference-site twin of the `qualifyRef` that + * {@link extractKotlinModuleConstants} applies to sibling initializers. + * + * `enclosingTypes` is the chain of qualified type paths the reference sits + * inside, INNERMOST FIRST (`['Outer.Inner', 'Outer']`). A companion member is + * keyed `.` and is bound unqualified exactly within that + * class body — including its nested types, which is why the whole chain is + * walked and not just the innermost link. An `object`'s own members are in scope + * inside its body under the same `.` key, so the same walk covers + * both. + * + * Innermost-first, and BEFORE the file-level maps the fold consults next, is + * Kotlin's own order: a companion member shadows a same-named top-level + * declaration and a same-named import throughout its class. Outside that class + * the bare name never means the companion at all, which is precisely what an + * empty chain expresses. + */ +function qualifyKotlinRefInEnclosingTypes( + fileKey: string, + name: string, + repo: RepoConstants, + enclosingTypes: readonly string[], +): string { + // A dotted reference carries AN owner, not necessarily its OWN full one, so it + // is resolved against the enclosing scopes exactly like a bare name. Kotlin + // binds `Inner.Q` inside `object Outer` to `Outer.Inner.Q`, and returning it + // unchanged looked for a key nothing declares. Worse, when the partial owner + // also names a top-level declaration the unchanged form MATCHES it: with a + // top-level `object ApiPaths` beside a nested one, `@GetMapping(ApiPaths.ORDERS)` + // inside the class holding the nested object resolved to the top-level value — + // a path the application does not serve, where the compiler binds the nested + // one. The scopes are already qualified (`kotlinEnclosingTypeNames`), so + // prefixing them onto whatever the reference spells is the whole rule. + const mc = repo.get(fileKey); + if (!mc) return name; + const unfoldableDeclarations = unfoldableDeclarationsOf(mc); + for (const type of enclosingTypes) { + const key = `${type}.${name}`; + if (mc.literals.has(key) || mc.exprs.has(key) || unfoldableDeclarations.has(key)) { + return key; + } + } + return name; +} + +/** + * Fold an inline operand list (e.g. `ApiPaths.BASE + "/orders"`) against + * `fileKey`, or null when any piece is unresolvable (skip floor). + * + * `enclosingTypes` is the chain of type declarations the REFERENCE sits inside + * (innermost first), and it is applied to the entry operands only — everything + * deeper is either already qualified by + * {@link extractKotlinModuleConstants} against its own declaring scope, or lives + * in another file where this chain means nothing. Passing it empty answers + * "what does this name mean at file level", which is the right question for a + * reference outside any type and the only one a caller without position + * information can honestly ask. + * + * An empty result is a SUCCESS, not a skip. `const val ROOT = ""` folds to `""`, + * which `joinPath` then resolves against the class-level prefix exactly as it + * resolves the literal `@GetMapping("")` — both mean "the prefix itself", the + * Spring idiom for a collection root. Collapsing it into `null` would make a + * resolved-empty path indistinguishable from an unresolvable one — the skip + * floor is reserved for "could not fold", and nothing else in the resolver + * conflates the two: {@link resolveKotlinConstant} returns `''` for an empty + * constant, and `resolveOperands` in the shared core returns its fold + * unfiltered. Matches `foldJavaOperands`, so the two JVM bindings do not + * diverge on the same input. + */ +export function foldKotlinOperands( + fileKey: string, + operands: readonly Operand[], + repo: RepoConstants, + enclosingTypes: readonly string[] = [], + index?: KotlinConstantIndex, +): string | null { + // Allocation gate only: skip the map when there is nothing to qualify against + // or no reference to qualify. It must not restate the rule — a dotted operand + // is qualified too, so testing for a bare one here decided the result instead + // of merely avoiding an allocation, and did so per-OPERAND-LIST: the same + // `Inner.Q` folded or not depending on whether a SIBLING operand happened to + // be bare. + const needsQualify = enclosingTypes.length > 0 && operands.some((op) => op.kind === 'ref'); + const scoped = needsQualify + ? operands.map((op) => + op.kind === 'ref' + ? { + kind: 'ref' as const, + name: qualifyKotlinRefInEnclosingTypes(fileKey, op.name, repo, enclosingTypes), + } + : op, + ) + : operands; + return foldOperands(fileKey, scoped, newFoldState(repo, index), 0); +} diff --git a/gitnexus/test/unit/group/kotlin-const-route-fold.test.ts b/gitnexus/test/unit/group/kotlin-const-route-fold.test.ts new file mode 100644 index 000000000..93c44d6cf --- /dev/null +++ b/gitnexus/test/unit/group/kotlin-const-route-fold.test.ts @@ -0,0 +1,1265 @@ +/** + * Constant-valued Spring route paths on the Kotlin group plugin. + * + * Drives `KOTLIN_HTTP_PLUGIN.prepareRepo` + `scan(tree, ctx, rel)` with all + * three arguments, which is the shape the http-route-extractor orchestrator + * uses. The existing Kotlin guards call `scan(tree)` with ONE argument and are + * therefore structurally blind here: without a repo context the plugin has no + * constant map and drops every constant-valued route. + * + * Asserted: + * • the four reference forms fold to the right provider contract — qualified + * access, fully-qualified name, single-name import, `+`-concatenation; + * • a class prefix that resolves to NO literal suppresses every method route + * under that class, literal ones included (the prefix is not knowable here, + * and emitting the methods unprefixed would publish paths the application + * does not serve) — the rule `java.ts` already applies. Pinned across every + * spelling that reaches the suppression, because the analysis inverts a + * literalness test rather than listing node types: a bare constant, both + * argument spellings, `[…]`, `arrayOf(…)`, a call, an `if`, and an + * interpolated string; + * • a prefix that resolves only PARTLY still publishes its resolvable arm — + * Kotlin's vararg `@RequestMapping("/lit", ApiPaths.BASE)` keeps `/lit`, + * because suppression exists to avoid wrong routes, not to discard right + * ones; + * • a `@RequestMapping` with no path argument at all is not a prefix and does + * not suppress anything; + * • an OpenFeign consumer folds a constant method path, and both consumer + * lanes (`@(Get|…)Mapping` and `@RequestLine`) are suppressed by an + * unresolvable governing prefix for the same reason a provider is — + * resolved in "path wins" order, so a literal `@FeignClient(path)` rescues + * an interface whose `@RequestMapping` is a constant, and an unresolvable + * `path` is fatal on its own; + * • an unresolvable constant emits nothing rather than a guessed path; + * • a cross-file fold survives BACKSLASHED repository keys — the shape glob + * v13 hands the orchestrator on Windows, and the one every other fixture + * here misses by writing POSIX string literals; + * • a constant that folds to `""` publishes the class prefix, exactly as the + * literal `@GetMapping("")` beside it does — an empty fold is a success, + * not the skip floor; + * • without a repo context the plugin emits nothing (the documented skip + * floor, and the branch the 1-argument guards cannot reach); + * • literal routes are untouched and are not emitted twice. + */ + +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import { requireVendoredGrammar } from '../../../src/core/tree-sitter/vendored-grammars.js'; +import { KOTLIN_HTTP_PLUGIN } from '../../../src/core/group/extractors/http-patterns/kotlin.js'; +import type { HttpLanguagePlugin } from '../../../src/core/group/extractors/http-patterns/types.js'; + +// Vendored grammar — loaded from vendor/ by absolute path, never node_modules (#2111). +let Kotlin: unknown; +try { + Kotlin = requireVendoredGrammar('tree-sitter-kotlin'); +} catch { + // Optional grammar; the suite skips when its native binding is unavailable. +} + +const describeKotlin = Kotlin && KOTLIN_HTTP_PLUGIN ? describe : describe.skip; +// Non-null only inside `describeKotlin`, which is skipped when the plugin is null. +const plugin = KOTLIN_HTTP_PLUGIN as HttpLanguagePlugin; + +const parseSource = (p: Parser, src: string): Parser.Tree => { + p.setLanguage(Kotlin as Parser.Language); + return p.parse(src); +}; + +/** prepareRepo + a 3-argument scan over every file; contracts of one role. */ +function contracts(files: Record, role: 'provider' | 'consumer'): string[] { + const ctx = plugin.prepareRepo?.({ + repoPath: '/virtual', + files: Object.keys(files), + parser: new Parser(), + readFile: (rel: string) => files[rel] ?? null, + parseSource, + }); + const out: string[] = []; + for (const rel of Object.keys(files)) { + for (const d of plugin.scan(parseSource(new Parser(), files[rel]), ctx, rel)) { + if (d.role === role) out.push(`${d.method} ${d.path}`); + } + } + return out.sort(); +} + +const providers = (files: Record): string[] => contracts(files, 'provider'); +const consumers = (files: Record): string[] => contracts(files, 'consumer'); + +const CONSTS = 'src/main/kotlin/com/example/app/api/ApiPaths.kt'; +const CONTROLLER = 'src/main/kotlin/com/example/app/web/OrderController.kt'; +const CLIENT = 'src/main/kotlin/com/example/app/client/OrderClient.kt'; + +const CONSTS_SRC = `package com.example.app.api + +object ApiPaths { + const val BASE = "/api/v1" + const val ORDERS = BASE + "/orders" +} +`; + +describeKotlin('Kotlin constant-valued Spring routes (group plugin)', () => { + it('folds a qualified reference in a positional argument', () => { + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('folds a standalone top-level val from another file', () => { + expect( + providers({ + [CONSTS]: `package com.example.app.api + +val ORDERS = "/api/v1/orders" +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ORDERS + +@RestController +class OrderController { + @GetMapping(ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('folds a constant imported through a nested object', () => { + expect( + providers({ + [CONSTS]: `package com.example.app.api + +object Outer { + object Inner { + const val ORDERS = "/nested/orders" + } +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.Outer.Inner + +@RestController +class OrderController { + @GetMapping(Inner.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /nested/orders']); + }); + + it('folds a named `value =` / `path =` argument and an inline concatenation', () => { + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @PostMapping(value = ApiPaths.BASE + "/orders/create") + fun create() {} + + @DeleteMapping(path = ApiPaths.ORDERS) + fun remove() {} +} +`, + }), + ).toEqual(['DELETE /api/v1/orders', 'POST /api/v1/orders/create']); + }); + + it('folds a fully-qualified reference and a single-name import', () => { + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths.ORDERS + +@RestController +class OrderController { + @GetMapping(ORDERS) + fun list() {} + + @PutMapping(com.example.app.api.ApiPaths.ORDERS) + fun replace() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders', 'PUT /api/v1/orders']); + }); + + it('ignores a non-route named argument', () => { + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(path = ApiPaths.ORDERS, produces = [MediaType.APPLICATION_JSON_VALUE]) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('suppresses every method route under a CONSTANT class prefix', () => { + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +@RequestMapping(ApiPaths.BASE) +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} + + @GetMapping("/literal") + fun literal() {} +} +`, + }), + ).toEqual([]); + }); + + it('suppresses them just the same when the class prefix is a NAMED argument', () => { + // `@RequestMapping(value = ApiPaths.BASE)` takes the other branch of + // `kotlinRouteArgumentExpression` (read the key, then take `namedChild(1)`) + // than the positional case above. Both must reach the same verdict: a + // regression in the named branch would let the class escape suppression and + // publish every method under it unprefixed. + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +@RequestMapping(value = ApiPaths.BASE) +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} + + @GetMapping("/literal") + fun literal() {} +} +`, + }), + ).toEqual([]); + }); + + /** + * A controller carrying `prefix` as its class-level `@RequestMapping`, with + * one constant-valued and one literal route under it. `decls` holds any + * top-level declaration the prefix expression refers to. + */ + const controllerWithPrefix = (prefix: string, decls = ''): Record => ({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths +${decls} +@RestController +@RequestMapping(${prefix}) +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} + + @GetMapping("/literal") + fun literal() {} +} +`, + }); + + // Every prefix spelling that resolves to no literal, and so must suppress. + // This is a table rather than one representative case on purpose: the two + // tests above pin a BARE constant, which any node-type allow-list would also + // catch. These are the shapes such a list forgets — and forgetting one does + // not degrade to "no route", it publishes every method of the class at its + // UNPREFIXED path, which the application does not serve. The `if` and the + // interpolated string are the two that need no constant map at all to go + // wrong, and the `[…]` / `arrayOf(…)` pair matters because the literal + // prefix patterns DO reach inside both — so a naive "is it a literal + // container?" test would pass them straight through. + it.each([ + ['a collection literal holding a constant', '[ApiPaths.BASE]', ''], + ['an arrayOf(…) holding a constant', 'arrayOf(ApiPaths.BASE)', ''], + ['a named collection literal holding a constant', 'value = [ApiPaths.BASE]', ''], + ['a function call', 'buildPath()', '\nfun buildPath(): String = ApiPaths.BASE\n'], + ['an interpolated string', '"${ApiPaths.BASE}"', ''], + ['an if expression', 'if (USE_V2) "/api/v2" else "/api/v1"', '\nconst val USE_V2 = false\n'], + ])('suppresses every method route under a class prefix that is %s', (_label, prefix, decls) => { + expect(providers(controllerWithPrefix(prefix, decls))).toEqual([]); + }); + + it('keeps both routes when that same class prefix is a plain literal', () => { + // The control for the table above: same two methods, same helper, a prefix + // the extractor can resolve. Without it an empty result there would be + // indistinguishable from the fixture failing to produce routes at all. + expect(providers(controllerWithPrefix('"/api"'))).toEqual([ + 'GET /api/api/v1/orders', + 'GET /api/literal', + ]); + }); + + it('keeps the resolvable arm of a PARTLY resolvable class prefix', () => { + // Kotlin's vararg spelling. `/lit` is a real prefix the application really + // serves, so the routes under it are derivable and must survive; only the + // `ApiPaths.BASE` arm is missing from the result, exactly as it was before + // constant folding existed. Marking the class unfoldable here would trade a + // wrong route for a missing one, which is not the bargain suppression makes. + expect(providers(controllerWithPrefix('"/lit", ApiPaths.BASE'))).toEqual([ + 'GET /lit/api/v1/orders', + 'GET /lit/literal', + ]); + // Same shape spelled as one collection argument. + expect(providers(controllerWithPrefix('["/lit", ApiPaths.BASE]'))).toEqual([ + 'GET /lit/api/v1/orders', + 'GET /lit/literal', + ]); + }); + + it('does not treat a @RequestMapping without a path argument as a prefix', () => { + // `produces` is not a path, so this class has no prefix — not an + // unresolvable one. Suppressing here would drop routes that are correct and + // complete as written. + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +@RequestMapping(produces = [MediaType.APPLICATION_JSON_VALUE]) +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('does not treat an EMPTY class path array as an unresolvable prefix', () => { + // `@RequestMapping(arrayOf())` designates NO prefix — Spring maps the class + // at the application root — so `/literal` really is served at `/literal`. + // Treating it as an unresolvable prefix suppressed every route under the + // class, the literal one included, which no constant fold was ever involved + // in. The arithmetic behind that: an empty array has no elements, and "no + // element is a literal" is trivially true of an empty set, so the class read + // as unresolvable. "No prefix" is not "an unresolvable prefix". + expect(providers(controllerWithPrefix('arrayOf()'))).toEqual([ + 'GET /api/v1/orders', + 'GET /literal', + ]); + }); + + it('keeps an inherited route under an EMPTY interface path array', () => { + const files = { + 'src/OrderApi.kt': `package com.example.app + +@RequestMapping([]) +interface OrderApi { + @GetMapping("/orders") + fun list() +} +`, + 'src/OrderController.kt': `package com.example.app + +@RestController +class OrderController : OrderApi { + override fun list() {} +} +`, + }; + const detections = + plugin.scanProject?.( + Object.entries(files).map(([filePath, source]) => ({ + filePath, + tree: parseSource(new Parser(), source), + })), + ) ?? []; + expect( + detections.flatMap((file) => + file.detections + .filter((detection) => detection.role === 'provider') + .map((detection) => `${detection.method} ${detection.path}`), + ), + ).toEqual(['GET /orders']); + }); + + it('still suppresses a NON-empty array whose only element is a constant', () => { + // The control for the empty `arrayOf()` case: an array that DOES designate + // a prefix still suppresses when that prefix is unknowable. + expect(providers(controllerWithPrefix('arrayOf(ApiPaths.BASE)'))).toEqual([]); + }); + + it('does not treat an EMPTY @FeignClient(path) array as an unresolvable path', () => { + // Same distinction on the consumer side, where the governing-prefix guard is + // its own set: `path = arrayOf()` adds no prefix, so the remote URL is + // exactly the method's own path and the consumer is knowable. + expect( + consumers({ + [CLIENT]: `package com.example.app.client + +@FeignClient(name = "orders", path = arrayOf()) +interface OrderClient { + @GetMapping("/orders") + fun listOrders(): String +} +`, + }), + ).toEqual(['GET /orders']); + }); + + it('still applies a LITERAL class prefix to a folded method path', () => { + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +@RequestMapping("/api/v1") +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/api/v1/orders']); + }); + + it('emits nothing when the constant cannot be resolved', () => { + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual([]); + }); + + it('emits nothing for a constant route scanned without a repo context', () => { + // This is the branch the 1-argument guards cannot reach; pin it so it is + // not silently dead in the suite. + const tree = parseSource( + new Parser(), + `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + ); + expect(plugin.scan(tree).filter((d) => d.role === 'provider')).toEqual([]); + }); + + it('folds a constant method path on a @FeignClient interface', () => { + expect( + consumers({ + [CONSTS]: CONSTS_SRC, + [CLIENT]: `package com.example.app.client + +import com.example.app.api.ApiPaths + +@FeignClient(name = "orders") +interface OrderClient { + @GetMapping(ApiPaths.ORDERS) + fun list() +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('drops a @FeignClient consumer whose interface prefix is a CONSTANT', () => { + // In tree-sitter-kotlin an `interface` is a `class_declaration`, so the + // suppression rule reaches a Feign interface too — and it must, for the same + // reason it reaches a controller: the prefix is not knowable here, so the + // alternative is publishing the remote call at `/orders` when the service is + // really called at `/api/v1/orders`. A dropped consumer edge is a missing + // fact; a wrong URL is a false edge. + const files = { + [CONSTS]: CONSTS_SRC, + [CLIENT]: `package com.example.app.client + +import com.example.app.api.ApiPaths + +@FeignClient(name = "orders") +@RequestMapping(ApiPaths.BASE) +interface OrderClient { + @GetMapping("/orders") + fun list() +} +`, + }; + expect(consumers(files)).toEqual([]); + // Control: the identical interface with a LITERAL prefix is still detected, + // so the empty result above is the suppression rule and not a blind spot in + // Feign detection itself. + expect( + consumers({ + ...files, + [CLIENT]: files[CLIENT].replace( + '@RequestMapping(ApiPaths.BASE)', + '@RequestMapping("/api/v1")', + ), + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('drops a @FeignClient consumer whose `path` argument is a CONSTANT', () => { + // `path` is the Feign client's own prefix and is never a `@RequestMapping`, + // so the class-prefix analysis cannot see it. Left unchecked, this interface + // falls through to the no-prefix fallback and publishes a remote call to + // `/api/v1/orders` as a call to `/orders` — a consumer edge pointing at a + // route no service serves. + const files = { + [CONSTS]: CONSTS_SRC, + [CLIENT]: `package com.example.app.client + +import com.example.app.api.ApiPaths + +@FeignClient(name = "orders", path = ApiPaths.BASE) +interface OrderClient { + @GetMapping(ApiPaths.ORDERS) + fun list() +} +`, + }; + expect(consumers(files)).toEqual([]); + // Control: the same interface with a LITERAL `path` is still detected. + expect( + consumers({ + ...files, + [CLIENT]: files[CLIENT].replace('path = ApiPaths.BASE', 'path = "/svc"'), + }), + ).toEqual(['GET /svc/api/v1/orders']); + }); + + it('lets a literal @FeignClient(path) outrank a CONSTANT @RequestMapping', () => { + // `path` wins over `@RequestMapping` when the URL is assembled, so it has to + // win when resolvability is judged too — otherwise an interface whose real + // prefix is perfectly knowable loses its consumer to a `@RequestMapping` + // that never governed it. + expect( + consumers({ + [CONSTS]: CONSTS_SRC, + [CLIENT]: `package com.example.app.client + +import com.example.app.api.ApiPaths + +@FeignClient(name = "orders", path = "/svc") +@RequestMapping(ApiPaths.BASE) +interface OrderClient { + @GetMapping("/orders") + fun list() +} +`, + }), + ).toEqual(['GET /svc/orders']); + }); + + it('drops a @RequestLine consumer under an unresolvable interface prefix', () => { + // `@RequestLine` carries its own verb and path but is still prefixed by the + // interface, and it resolves through the same "path wins" fallback chain as + // the `@(Get|…)Mapping` lane — so an unresolvable governing prefix leaves + // the remote URL just as unknowable here. + const files = { + [CONSTS]: CONSTS_SRC, + [CLIENT]: `package com.example.app.client + +import com.example.app.api.ApiPaths + +@FeignClient(name = "orders") +@RequestMapping(ApiPaths.BASE) +interface OrderClient { + @RequestLine("GET /list") + fun list() +} +`, + }; + expect(consumers(files)).toEqual([]); + // Control: a literal interface prefix still yields the prefixed consumer. + expect( + consumers({ + ...files, + [CLIENT]: files[CLIENT].replace( + '@RequestMapping(ApiPaths.BASE)', + '@RequestMapping("/lit")', + ), + }), + ).toEqual(['GET /lit/list']); + }); + + it('judges @RequestLine and @(Get|…)Mapping alike on ONE interface', () => { + // Both lanes read the same prefix through the same fallback chain, so they + // must reach the same verdict on it. A guard on only one of them lets the + // interface suppress one route and publish the other under the very same + // unresolvable prefix — a self-inconsistency visible in a single scan. + expect( + consumers({ + [CONSTS]: CONSTS_SRC, + [CLIENT]: `package com.example.app.client + +import com.example.app.api.ApiPaths + +@FeignClient(name = "orders") +@RequestMapping(ApiPaths.BASE) +interface OrderClient { + @GetMapping(ApiPaths.ORDERS) + fun list() + + @RequestLine("GET /list") + fun listLegacy() +} +`, + }), + ).toEqual([]); + }); + + it('folds across files when repository keys use Windows separators', () => { + // The orchestrator's file list comes from glob v13, which has no + // `posix: true` and joins with the platform separator, so on Windows both + // `prepareRepo({files})` and `scan(tree, ctx, rel)` see + // `src\main\kotlin\…`. `resolveKotlinImport` asks whether a key ends with + // `com/example/app/api/ApiPaths.kt` — a test no backslashed key can pass — + // so EVERY cross-file fold returned null on Windows and on Windows only: + // the pre-pass still ran and the context was still built, the feature was + // just silently absent. Every other fixture in this file is a POSIX string + // literal, which is exactly why CI stayed green. + // + // The keys are backslashed HERE rather than derived from `path.sep`, so the + // regression is pinned on every runner instead of only on the Windows + // matrix — the plugin reads keys, not the host OS, so simulating the keys + // simulates the whole bug. + const winKey = (rel: string): string => rel.replace(/\//g, '\\'); + expect( + providers({ + [winKey(CONSTS)]: CONSTS_SRC, + [winKey(CONTROLLER)]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('treats a constant that folds to "" as the class prefix itself', () => { + // `const val ROOT = ""` is Spring's idiom for "the collection root", and + // `joinPath` resolves it against the class prefix exactly as it resolves the + // literal `@PostMapping("")` beside it. The fold used to collapse `''` into + // the skip floor, so the two annotations below — the same path, written two + // ways — disagreed: the literal published `POST /api`, the constant + // published nothing. Asserting BOTH in one class is the point; a test on the + // constant alone would pass against any chosen convention rather than + // pinning the two spellings together. + expect( + providers({ + [CONSTS]: `package com.example.app.api + +object ApiPaths { + const val ROOT = "" +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +@RequestMapping("/api") +class OrderController { + @GetMapping(ApiPaths.ROOT) + fun list() {} + + @PostMapping("") + fun create() {} +} +`, + }), + ).toEqual(['GET /api/', 'POST /api/']); + }); + + it('serves the same route whichever of two same-named objects is declared first', () => { + // `A.ROUTE = BASE + "/m"` means `A.BASE`. Recording every object member + // under its bare name too made that operand resolve through whichever + // same-named sibling was walked LAST, so reordering two objects — a change + // Kotlin does not even see — moved the published route from `/right/m` to + // `/wrong/m`. Both orders are asserted; either alone passes on a last-wins + // implementation. + const controllerWith = (objects: string): string => `package com.example.app.web + +${objects} + +@RestController +class OrderController { + @GetMapping(A.ROUTE) + fun get() {} +} +`; + const A = `object A { + const val BASE = "/right" + const val ROUTE = BASE + "/m" +}`; + const B = `object B { + const val BASE = "/wrong" +}`; + expect(providers({ [CONTROLLER]: controllerWith(`${A}\n\n${B}`) })).toEqual(['GET /right/m']); + expect(providers({ [CONTROLLER]: controllerWith(`${B}\n\n${A}`) })).toEqual(['GET /right/m']); + }); + + it('reads a bare route constant from the import, not from a local object member', () => { + // Bare `ORDERS` in this file is the IMPORT: `object Local` binds + // `Local.ORDERS` and nothing else. A bare key for the object member is a + // binding Kotlin does not have, and it outranked the import because the fold + // consults literals before imports — publishing a path the service does not + // serve. + expect( + providers({ + [CONSTS]: `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/api/v1/orders" +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths.ORDERS + +object Local { + const val ORDERS = "/local" +} + +@RestController +class OrderController { + @GetMapping(ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('keeps a companion constant readable under its bare name', () => { + // The control for the test above, and the reason object members and + // companion members are keyed differently: a companion's members ARE in + // scope unqualified throughout the enclosing class, which is precisely where + // route annotations sit. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +@RestController +class OrderController { + companion object { + const val ORDERS = "/api/v1/orders" + } + + @GetMapping(ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('resolves a bare companion constant inside a nested class', () => { + // Declaration extraction and reference-site qualification must agree on + // the full owner path. `ORDERS` here means `Outer.Inner.ORDERS`, not the + // nonexistent top-level `Inner.ORDERS`. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +@RestController +class Outer { + class Inner { + companion object { + const val ORDERS = "/nested/orders" + } + + @GetMapping(ORDERS) + fun list() {} + } +} +`, + }), + ).toEqual(['GET /nested/orders']); + }); + + it('folds through the file that declares the package, not one whose path imitates it', () => { + // The decoy's PATH ends with the imported FQN, but it declares + // `package x.com.example.app.api` — a different declaration. Choosing the + // candidate by path let it win, and because it declares the same member the + // fold did not skip: it published `/wrong`. The declared `package` is the + // authority; the path is only a tie-break among files that already declare + // the right one. + expect( + providers({ + 'src/generated/Constants.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/api/v1/orders" +} +`, + 'src/x/com/example/app/api/ApiPaths.kt': `package x.com.example.app.api + +object ApiPaths { + const val ORDERS = "/wrong" +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('emits nothing when a test-source copy duplicates a production constant', () => { + // Same package, same object, different value, and only the copy follows the + // `/.kt` convention — so a file-name tie-break folded a + // test-only path into a production route. Two declarations of one + // fully-qualified name identify no single declaration, so the honest answer + // is no route: preferring the production source set would be a guess about + // build configuration this layer cannot see. + expect( + providers({ + 'src/main/kotlin/generated/RoutePaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/api/v1/orders" +} +`, + 'src/test/kotlin/com/example/app/api/ApiPaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/test-only" +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual([]); + }); + + it('keeps an unfoldable production twin in duplicate-FQN detection', () => { + expect( + providers({ + 'src/main/kotlin/generated/RoutePaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = ("/production") +} +`, + 'src/test/kotlin/com/example/app/api/ApiPaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/test-only" +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual([]); + }); + + it('resolves a bare reference by the class it sits in, not by declaration order', () => { + // A companion member is bound unqualified inside its enclosing class BODY + // and nowhere else. Recorded under a file-level bare key it landed in the + // same namespace as top-level declarations and, because companions are + // walked last, won every unqualified reference in the file — so the + // reference in `OrderController` below, which is not `Holder`'s body, + // published `/companion` where the application serves `/top`. + // + // Both classes are asserted from ONE file: a fixture with only the wrong + // one would also pass on an implementation that simply dropped companions, + // and a fixture with only the right one would pass on the old file-wide key. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +const val ORDERS = "/top" + +class Holder { + companion object { + const val ORDERS = "/companion" + } + + @GetMapping(ORDERS) + fun inside() {} +} + +@RestController +class OrderController { + @GetMapping(ORDERS) + fun outside() {} +} +`, + }), + ).toEqual(['GET /companion', 'GET /top']); + }); + + it('does not publish a top-level route for an unfoldable companion binding', () => { + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +const val ORDERS = "/top" + +@RestController +class Holder { + companion object { + const val ORDERS = ("/companion") + } + + @GetMapping(ORDERS) + fun inside() {} +} + +@RestController +class OtherController { + @GetMapping(ORDERS) + fun outside() {} +} +`, + }), + ).toEqual(['GET /top']); + }); + + it('gives two colliding companions each their own class', () => { + // Kotlin scopes each companion's members to its own class, so the two + // references below mean different constants even though they are spelled + // identically. One file-level namespace could only answer last-wins: both + // read `/h2`, and swapping the two classes flipped both to `/h1`. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +@RestController +class FirstController { + companion object { + const val ORDERS = "/h1" + } + + @GetMapping(ORDERS) + fun list() {} +} + +@RestController +class SecondController { + companion object { + const val ORDERS = "/h2" + } + + @GetMapping(ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /h1', 'GET /h2']); + }); + + it('folds a top-level initializer at file level even when a companion shares the name', () => { + // `ROUTE`'s initializer is TOP-LEVEL, so its scope chain is empty and its + // operand `BASE` means the top-level `BASE`. With a file-wide companion key + // the empty chain left the operand bare and the companion answered it, + // publishing `/comp/m` where the application serves `/top/m`. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +const val BASE = "/top" +const val ROUTE = BASE + "/m" + +class Holder { + companion object { + const val BASE = "/comp" + } +} + +@RestController +class OrderController { + @GetMapping(ROUTE) + fun list() {} +} +`, + }), + ).toEqual(['GET /top/m']); + }); + + it('folds through a package whose segment is backtick-quoted on one side only', () => { + // `` package com.example.app.`api` `` and `package com.example.app.api` are + // the same package to the compiler — the quotes are lexical syntax, not part + // of the name. Comparing the two verbatim rejected the one real candidate + // and dropped the route. Both directions are asserted because either side + // can carry the quotes. + const controller = (spec: string): string => `package com.example.app.web + +import ${spec} + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`; + const quotedDeclaration = `package com.example.app.\`api\` + +object ApiPaths { + const val ORDERS = "/api/v1/orders" +} +`; + // Declaration quoted, import plain. + expect( + providers({ + [CONSTS]: quotedDeclaration, + [CONTROLLER]: controller('com.example.app.api.ApiPaths'), + }), + ).toEqual(['GET /api/v1/orders']); + // Import quoted, declaration plain. + expect( + providers({ + [CONSTS]: CONSTS_SRC, + [CONTROLLER]: controller('com.example.app.`api`.ApiPaths'), + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('folds a same-file top-level `val` even when the file imports nothing', () => { + // Three conditions have to line up for this to break, which is why a + // realistic controller never hit it: the constant is a top-level non-`const` + // `val` (so `isKotlinConstantFile` rejects the file and the pre-pass never + // indexes it), the file declares no `object` (the gate's other arm), and it + // imports nothing. The on-demand overlay in `scan` is the file's only + // remaining chance, and it used to admit the extraction only when the file + // had imports — throwing away the very constants the route needs. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +val PATH = "/orders" + +@RestController +@RequestMapping("/api/v1") +class OrderController { + @GetMapping(PATH) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('folds a backtick-quoted constant declared in its own file', () => { + // The gate decides whether a file is parsed into the repo map at all, so a + // gate that rejects backticks silently drops a constant the resolver can + // fold — a cross-file reference to it then floors to skip. + expect( + providers({ + [CONSTS]: `package com.example.app.api + +object ApiPaths { + const val \`ORDERS\` = "/api/v1/orders" +} +`, + [CONTROLLER]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders']); + }); + + it('resolves a PARTIALLY qualified reference against the enclosing scopes', () => { + // `Inner.Q` carries an owner, but not its whole one: the key is + // `Outer.Inner.Q`. Treating any dotted reference as already complete looked + // for a key nothing declares and dropped the route. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +@RestController +object Outer { + object Inner { + const val Q = "/orders" + } + + @GetMapping(Inner.Q) + fun list() {} +} +`, + }), + ).toEqual(['GET /orders']); + }); + + it('binds a partially qualified reference to the NESTED owner, not a top-level twin', () => { + // The severe half of the same defect. Kotlin binds `ApiPaths` to the nested + // object, so the route is `/inner`. Left unqualified, `ApiPaths.ORDERS` + // matched the TOP-LEVEL object instead and published `/orders` — a path the + // application does not serve, which is worse than the dropped route above. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +object ApiPaths { + const val ORDERS = "/orders" +} + +@RestController +class OrderController { + object ApiPaths { + const val ORDERS = "/inner" + } + + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }), + ).toEqual(['GET /inner']); + }); + + it('resolves a partially qualified reference inside an INITIALIZER too', () => { + // The same rule on the other side. `ROUTE = Inner.Q + "/m"` sits in + // `Outer`'s body, so `Inner.Q` means `Outer.Inner.Q` there. Fixing only the + // reference site would leave the two halves disagreeing about what a dotted + // name means — the asymmetry that produced the earlier defects here. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +object Outer { + object Inner { + const val Q = "/orders" + } + + const val ROUTE = Inner.Q + "/m" +} + +@RestController +class OrderController { + @GetMapping(Outer.ROUTE) + fun list() {} +} +`, + }), + ).toEqual(['GET /orders/m']); + }); + + it('folds a partially qualified reference regardless of its sibling operands', () => { + // Control for the allocation gate: it must not decide the result. Gating on + // "some operand is bare" made this same `Inner.Q` fold only because `SUFFIX` + // sits beside it, while `Inner.Q` alone did not — the same reference + // resolving differently by the company it keeps. + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +@RestController +object Outer { + object Inner { + const val Q = "/orders" + } + + const val SUFFIX = "/list" + + @GetMapping(Inner.Q + SUFFIX) + fun list() {} +} +`, + }), + ).toEqual(['GET /orders/list']); + }); + + it('leaves literal routes unchanged and emits each exactly once', () => { + expect( + providers({ + [CONTROLLER]: `package com.example.app.web + +@RestController +@RequestMapping("/api/v1") +class OrderController { + @GetMapping("/orders") + fun list() {} + + @PostMapping(value = "/orders") + fun create() {} +} +`, + }), + ).toEqual(['GET /api/v1/orders', 'POST /api/v1/orders']); + }); +}); diff --git a/gitnexus/test/unit/kotlin-route-const-resolver.test.ts b/gitnexus/test/unit/kotlin-route-const-resolver.test.ts new file mode 100644 index 000000000..5f9a193f8 --- /dev/null +++ b/gitnexus/test/unit/kotlin-route-const-resolver.test.ts @@ -0,0 +1,1448 @@ +/** + * Kotlin route-path constant resolution (the Kotlin binding of the #2391 core). + * + * Covers the four reference forms the Java binding already handles — qualified + * access, fully-qualified name, single-name import, `+`-concatenation — plus the + * places Kotlin genuinely differs from Java and therefore needs its own + * behavior rather than a translation: + * + * • `object` / `companion object` / top-level carriers, where Java has only + * `static final` on a type (a companion member is referenced through its + * ENCLOSING class, never through `Companion`); + * • no `String` type gate — Kotlin infers property types, so the initializer + * decides whether a constant is foldable; + * • a file name need not match the declaration it holds, so import resolution + * falls back to the package directory; + * • member imports are unmarked (`import a.b.C.F` is spelled exactly like a + * type import), so both readings are tried; + * • string templates (`"$base/orders"`) are refused rather than silently + * folded with the interpolation deleted. + * + * Every unresolvable case asserts `null` — an ambiguous import must never + * produce a guessed path, because a wrong route is a false edge in the graph + * while a missing one is only a missing fact. + */ + +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js'; +import { MAX_FOLD_LENGTH } from '../../src/core/ingestion/route-extractors/constant-resolver.js'; +import { + buildKotlinConstantIndex, + extractKotlinModuleConstants, + foldKotlinOperands, + isKotlinConstantFile, + overlayKotlinConstantIndex, + parseKotlinConstOperands, + resolveKotlinConstant, + resolveKotlinImport, + resolveKotlinImportWithIndex, + type ModuleConstants, + type RepoConstants, +} from '../../src/core/ingestion/route-extractors/kotlin-const-resolver.js'; +import { unquoteSpringLiteral } from '../../src/core/ingestion/route-extractors/spring-shared.js'; + +// Vendored grammar — loaded from vendor/ by absolute path, never node_modules (#2111). +let Kotlin: unknown; +try { + Kotlin = requireVendoredGrammar('tree-sitter-kotlin'); +} catch { + // Optional grammar; the suite skips when its native binding is unavailable. +} + +const parser = new Parser(); +if (Kotlin) parser.setLanguage(Kotlin as Parser.Language); + +const parse = (src: string): Parser.Tree => parser.parse(src); + +/** Build a RepoConstants map from virtual files: { 'a/b/C.kt': source }. */ +function repoOf(files: Record): RepoConstants { + const map = new Map(); + for (const [key, src] of Object.entries(files)) { + map.set(key, extractKotlinModuleConstants(parse(src))); + } + return map; +} + +/** The initializer expression of the first `property_declaration` in `src`. */ +function firstInitializer(src: string): Parser.SyntaxNode { + const property = parse(src).rootNode.descendantsOfType('property_declaration')[0]; + expect(property, 'expected a property_declaration').toBeDefined(); + const eq = property.children.findIndex((c) => c.type === '='); + expect(eq, 'expected an initializer').toBeGreaterThan(-1); + const init = property.children.slice(eq + 1).find((c) => c.isNamed); + expect(init, 'expected an initializer expression').toBeDefined(); + return init as Parser.SyntaxNode; +} + +const CONSTS_KEY = 'src/main/kotlin/com/example/app/api/ApiPaths.kt'; +const CONTROLLER_KEY = 'src/main/kotlin/com/example/app/web/OrderController.kt'; + +const CONSTS_SRC = `package com.example.app.api + +object ApiPaths { + const val BASE = "/api/v1" + const val ORDERS = BASE + "/orders" + val LEGACY: String = "/legacy/orders" +} +`; + +const describeKotlin = Kotlin ? describe : describe.skip; + +describeKotlin('Kotlin route-path constant resolution', () => { + describe('reference forms shared with the Java binding', () => { + it('resolves a qualified reference through a type import', () => { + const repo = repoOf({ + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @GetMapping(ApiPaths.ORDERS) + fun list() {} +} +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('resolves a fully-qualified reference with no import at all', () => { + const repo = repoOf({ + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +@RestController +class OrderController { + @GetMapping(com.example.app.api.ApiPaths.ORDERS) + fun list() {} +} +`, + }); + expect( + resolveKotlinConstant(CONTROLLER_KEY, 'com.example.app.api.ApiPaths.ORDERS', repo), + ).toBe('/api/v1/orders'); + }); + + it('resolves a single-name import of an object member', () => { + const repo = repoOf({ + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths.ORDERS + +@RestController +class OrderController { + @GetMapping(ORDERS) + fun list() {} +} +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('folds an inline `+`-concatenation at the annotation site', () => { + const repo = repoOf({ + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths + +@RestController +class OrderController { + @PostMapping(value = ApiPaths.BASE + "/orders/create") + fun create() {} +} +`, + }); + const operands = parseKotlinConstOperands( + firstInitializer('val X = ApiPaths.BASE + "/orders/create"'), + ); + if (operands === null) throw new Error('expected a foldable operand list'); + expect(operands).toEqual([ + { kind: 'ref', name: 'ApiPaths.BASE' }, + { kind: 'literal', value: '/orders/create' }, + ]); + expect(foldKotlinOperands(CONTROLLER_KEY, operands, repo)).toBe('/api/v1/orders/create'); + }); + + it('folds a constant defined by concatenating another constant', () => { + // `ORDERS = BASE + "/orders"` inside the same object. + const repo = repoOf({ [CONSTS_KEY]: CONSTS_SRC }); + expect(resolveKotlinConstant(CONSTS_KEY, 'ApiPaths.ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('folds a chain of three or more `+` operands', () => { + // tree-sitter-kotlin nests `A + B + C` left-associatively, so every + // `additive_expression` has exactly two operands and the chain folds by + // recursion. Pinned because the two-operand case cannot detect a + // regression to a flat-node reading. + const key = 'src/main/kotlin/com/example/app/api/Chained.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Chained { + const val BASE = "/api" + const val VERSION = "/v1" + const val ORDERS = BASE + VERSION + "/orders" + const val ORDER_ITEMS = BASE + VERSION + "/orders" + "/items" +} +`, + }); + expect(resolveKotlinConstant(key, 'Chained.ORDERS', repo)).toBe('/api/v1/orders'); + expect(resolveKotlinConstant(key, 'Chained.ORDER_ITEMS', repo)).toBe('/api/v1/orders/items'); + }); + + it('rejects a `-` expression, which shares one node type with `+`', () => { + // tree-sitter-kotlin gives `A + B` and `A - B` the same + // `additive_expression` type, so only the presence of a `+` token + // distinguishes a concatenation. Subtraction is not a string operation; + // folding it as one would fabricate a path. + const key = 'src/main/kotlin/com/example/app/api/Minus.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Minus { + const val BASE = "/api" + const val VERSION = "/v1" + val BROKEN = BASE - VERSION +} +`, + }); + expect(resolveKotlinConstant(key, 'Minus.BROKEN', repo)).toBeNull(); + }); + + it('folds escapes to exactly what the literal path would produce', () => { + const key = 'src/main/kotlin/com/example/app/api/Regexes.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Regexes { + const val USER = "/user/{id:\\\\d+}" +} +`, + }); + expect(resolveKotlinConstant(key, 'Regexes.USER', repo)).toBe( + unquoteSpringLiteral('"/user/{id:\\\\d+}"'), + ); + }); + }); + + describe('prepared constant index', () => { + it('preserves fold results while narrowing lookup to the declared package', () => { + const files: Record = { + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }; + for (let i = 0; i < 256; i++) { + files[`src/main/kotlin/com/noise/p${i}/Noise.kt`] = `package com.noise.p${i} + +object Noise${i} { + const val PATH = "/noise/${i}" +} +`; + } + const repo = repoOf(files); + const index = buildKotlinConstantIndex(repo); + + expect(index.constantKeys.size).toBe(257); + expect(index.byPackage.get('com.example.app.api')?.files).toEqual([CONSTS_KEY]); + expect(resolveKotlinImportWithIndex('com.example.app.api.ApiPaths', index)).toBe(CONSTS_KEY); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo, 0, index)).toBe( + resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo), + ); + }); + + it('reuses package projections for an import-only scan overlay', () => { + const repo = repoOf({ [CONSTS_KEY]: CONSTS_SRC }); + const index = buildKotlinConstantIndex(repo); + const controller = extractKotlinModuleConstants( + parse(`package com.example.app.web + +import com.example.app.api.ApiPaths +`), + ); + const overlaid = overlayKotlinConstantIndex(index, CONTROLLER_KEY, controller); + + expect(overlaid.repo.get(CONTROLLER_KEY)).toBe(controller); + expect(overlaid.constantKeys).toBe(index.constantKeys); + expect(overlaid.byPackage).toBe(index.byPackage); + expect(resolveKotlinImportWithIndex('com.example.app.api.ApiPaths', overlaid)).toBe( + CONSTS_KEY, + ); + expect( + resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', overlaid.repo, 0, overlaid), + ).toBe('/api/v1/orders'); + }); + + it('keeps duplicate declarations ambiguous after an overlay', () => { + const repo = repoOf({ [CONSTS_KEY]: CONSTS_SRC }); + const index = buildKotlinConstantIndex(repo); + const duplicateKey = 'src/test/kotlin/com/example/app/api/ApiPaths.kt'; + const duplicate = extractKotlinModuleConstants( + parse(`package com.example.app.api + +object ApiPaths { + const val ORDERS = "/test-only" +} +`), + ); + const overlaid = overlayKotlinConstantIndex(index, duplicateKey, duplicate); + + expect(overlaid.constantKeys.size).toBe(2); + expect(resolveKotlinImportWithIndex('com.example.app.api.ApiPaths', overlaid)).toBeNull(); + }); + + it('prefers an exact declared package over a nested path with the same FQN', () => { + const parentKey = 'src/main/kotlin/com/example/app/Parent.kt'; + const childKey = 'src/main/kotlin/com/example/app/api/ApiPaths.kt'; + const repo = repoOf({ + [parentKey]: `package com.example.app + +object api { + object ApiPaths { + const val ORDERS = "/wrong" + } +} +`, + [childKey]: `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/right" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }); + const index = buildKotlinConstantIndex(repo); + + expect(resolveKotlinImportWithIndex('com.example.app.api.ApiPaths', index)).toBe(childKey); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo, 0, index)).toBe( + '/right', + ); + expect( + resolveKotlinConstant( + CONTROLLER_KEY, + 'com.example.app.api.ApiPaths.ORDERS', + repo, + 0, + index, + ), + ).toBe('/right'); + }); + }); + + describe('ambiguity floors to skip, never to a guess', () => { + it('returns null when two modules carry the same fully-qualified name', () => { + const files = { + 'service-a/src/main/kotlin/com/example/app/api/ApiPaths.kt': CONSTS_SRC, + 'service-b/src/main/kotlin/com/example/app/api/ApiPaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/legacy/orders" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }; + const repo = repoOf(files); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBeNull(); + // Same verdict at the resolver layer the fold delegates to. It reads the + // candidates' DECLARED packages, so it takes the repo map as well as the + // key set. + expect( + resolveKotlinImport( + CONTROLLER_KEY, + 'com.example.app.api.ApiPaths', + new Set(Object.keys(files)), + repo, + ), + ).toBeNull(); + }); + + it('keeps an unfoldable duplicate in the fully-qualified-name candidate set', () => { + const repo = repoOf({ + 'src/main/kotlin/generated/RoutePaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = ("/production") +} +`, + 'src/test/kotlin/com/example/app/api/ApiPaths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/test-only" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBeNull(); + }); + + it('uses the unique declaring file before package filename fallbacks', () => { + // An unrelated constant file in the same package is not ambiguity when + // exactly one candidate declares the imported type. + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/Paths.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/api/v1/orders" +} +`, + 'src/main/kotlin/com/example/app/api/More.kt': `package com.example.app.api + +object MorePaths { + const val ITEMS = ("/api/v1/items") +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('returns null for a wildcard import', () => { + // `import com.example.app.api.*` binds no single name, so there is nothing + // to key the fold on and no honest way to pick a package member. + const repo = repoOf({ + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.* +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ORDERS', repo)).toBeNull(); + }); + + it('returns null for an unknown reference rather than an empty path', () => { + const repo = repoOf({ [CONSTS_KEY]: CONSTS_SRC }); + expect(resolveKotlinConstant(CONSTS_KEY, 'ApiPaths.MISSING', repo)).toBeNull(); + expect(foldKotlinOperands(CONSTS_KEY, [{ kind: 'ref', name: 'MISSING' }], repo)).toBeNull(); + }); + + it('folds to the empty string as a SUCCESS, not a skip', () => { + // The counterpart of the test above, and the distinction it depends on: + // `null` means "could not fold", `''` means "folded, and the answer is + // empty". `const val ROOT = ""` is Spring's spelling for "the class prefix + // itself", so collapsing it into null loses a route the literal + // `@GetMapping("")` publishes from the same class. `resolveKotlinConstant` + // already returned `''` here; `foldKotlinOperands` did not, which made the + // two entry points disagree about the same constant. + const key = 'src/main/kotlin/com/example/app/api/Root.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object ApiPaths { + const val ROOT = "" +} +`, + }); + expect(resolveKotlinConstant(key, 'ApiPaths.ROOT', repo)).toBe(''); + expect(foldKotlinOperands(key, [{ kind: 'ref', name: 'ApiPaths.ROOT' }], repo)).toBe(''); + expect(foldKotlinOperands(key, [{ kind: 'literal', value: '' }], repo)).toBe(''); + }); + + it('terminates on a self-referential constant', () => { + const key = 'src/main/kotlin/com/example/app/api/Cycle.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Cycle { + val A = B + "/a" + val B = A + "/b" +} +`, + }); + expect(resolveKotlinConstant(key, 'Cycle.A', repo)).toBeNull(); + }); + }); + + describe('Kotlin-specific declaration forms', () => { + it('reads a companion object member through its enclosing class', () => { + const key = 'src/main/kotlin/com/example/app/api/OrderApi.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +class OrderApi { + companion object { + const val ORDERS = "/api/v1/orders" + } +} +`, + }); + // Kotlin source says `OrderApi.ORDERS`; `Companion` never appears. + expect(resolveKotlinConstant(key, 'OrderApi.ORDERS', repo)).toBe('/api/v1/orders'); + expect(resolveKotlinConstant(key, 'Companion.ORDERS', repo)).toBeNull(); + }); + + it('reads a top-level `const val` through a single-name import', () => { + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/TopLevel.kt': `package com.example.app.api + +const val ORDERS = "/api/v1/orders" +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ORDERS +`, + }); + // The declaration's file is named `TopLevel.kt`, so this only resolves via + // the package-directory fallback tier. + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('reads an object whose file is not named after it', () => { + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/Constants.kt': `package com.example.app.api + +object ApiPaths { + const val ORDERS = "/api/v1/orders" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('un-aliases an aliased import', () => { + const repo = repoOf({ + [CONSTS_KEY]: CONSTS_SRC, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths as Paths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'Paths.ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('accepts a non-`const` `val` in an object but rejects `var`', () => { + const key = 'src/main/kotlin/com/example/app/api/Mixed.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Mixed { + val STABLE = "/api/v1/stable" + var MUTABLE = "/api/v1/mutable" +} +`, + }); + expect(resolveKotlinConstant(key, 'Mixed.STABLE', repo)).toBe('/api/v1/stable'); + expect(resolveKotlinConstant(key, 'Mixed.MUTABLE', repo)).toBeNull(); + }); + + it('rejects a computed property (custom getter or delegate)', () => { + const key = 'src/main/kotlin/com/example/app/api/Computed.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Computed { + val VIA_GETTER: String get() = "/api/v1/getter" + val VIA_DELEGATE: String by lazy { "/api/v1/delegate" } +} +`, + }); + expect(resolveKotlinConstant(key, 'Computed.VIA_GETTER', repo)).toBeNull(); + expect(resolveKotlinConstant(key, 'Computed.VIA_DELEGATE', repo)).toBeNull(); + }); + + it('does not harvest an instance property of a plain class', () => { + // `val` in a class body is per-instance; `Holder.ORDERS` does not compile. + const key = 'src/main/kotlin/com/example/app/api/Holder.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +class Holder { + val ORDERS = "/api/v1/orders" +} +`, + }); + expect(resolveKotlinConstant(key, 'Holder.ORDERS', repo)).toBeNull(); + expect(resolveKotlinConstant(key, 'ORDERS', repo)).toBeNull(); + }); + + it('refuses a string template instead of dropping the interpolation', () => { + // Joining the literal runs of `"$BASE/orders"` would publish `/orders` — + // a path the application does not serve. + const key = 'src/main/kotlin/com/example/app/api/Templated.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Templated { + const val BASE = "/api/v1" + val ORDERS = "\${BASE}/orders" + val ITEMS = "$BASE/items" +} +`, + }); + expect(resolveKotlinConstant(key, 'Templated.BASE', repo)).toBe('/api/v1'); + expect(resolveKotlinConstant(key, 'Templated.ORDERS', repo)).toBeNull(); + expect(resolveKotlinConstant(key, 'Templated.ITEMS', repo)).toBeNull(); + }); + + it('folds a single-line raw string, which Kotlin leaves byte-exact', () => { + const key = 'src/main/kotlin/com/example/app/api/Raw.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Raw { + const val ORDERS = """/api/v1/orders""" +} +`, + }); + expect(resolveKotlinConstant(key, 'Raw.ORDERS', repo)).toBe('/api/v1/orders'); + }); + + it('drops a constant whose initializer is not a string expression', () => { + // Kotlin infers property types, so there is no `String` type node to gate + // on — the initializer is what decides. + const key = 'src/main/kotlin/com/example/app/api/NonString.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object NonString { + const val PORT = 8080 + val COMPUTED = buildPath() +} +`, + }); + expect(resolveKotlinConstant(key, 'NonString.PORT', repo)).toBeNull(); + expect(resolveKotlinConstant(key, 'NonString.COMPUTED', repo)).toBeNull(); + }); + + it('does not answer `Owner.NAME` with a same-named top-level constant', () => { + // In Kotlin `Owner.NAME` means NAME is a member of the object/companion + // `Owner`; a top-level `NAME` in the same file is a different declaration, + // so matching it would fabricate a value. (The Java binding's bare-name + // fallback is sound there only because the file name pins the class.) + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/ApiPaths.kt': `package com.example.app.api + +const val ORDERS = "/top-level/orders" + +object Unrelated { + const val ITEMS = "/api/v1/items" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBeNull(); + }); + }); + + describe('member names resolve in their declaring scope, not a flat namespace', () => { + /** Two objects declaring `BASE`; only `A` is referenced. Order is the axis. */ + const siblingShadow = (first: 'A' | 'B'): string => { + const a = `object A { + const val BASE = "/right" + const val ROUTE = BASE + "/m" +}`; + const b = `object B { + const val BASE = "/wrong" +}`; + return `package com.example.app.api\n\n${first === 'A' ? `${a}\n\n${b}` : `${b}\n\n${a}`}\n`; + }; + const SIBLING_KEY = 'src/main/kotlin/com/example/app/api/Siblings.kt'; + + it('answers a sibling initializer identically whichever object is declared first', () => { + // `A.ROUTE = BASE + "/m"` means `A.BASE`, so the answer is `/right/m` in + // both spellings. Recording every member under its BARE name too made the + // operand resolve through whichever object was walked last, so moving + // `object B` above `object A` changed the emitted route for source that + // had not changed — the same file, merely reordered, served a different + // path. Both orders are asserted because either one alone passes on a + // last-wins implementation. + for (const first of ['A', 'B'] as const) { + const repo = repoOf({ [SIBLING_KEY]: siblingShadow(first) }); + expect(resolveKotlinConstant(SIBLING_KEY, 'A.ROUTE', repo), `${first} first`).toBe( + '/right/m', + ); + expect(resolveKotlinConstant(SIBLING_KEY, 'B.BASE', repo), `${first} first`).toBe('/wrong'); + } + }); + + it('does not bind an `object` member to its bare name, so an import still wins', () => { + // `object Local { const val ORDERS }` binds `Local.ORDERS` and nothing + // else — bare `ORDERS` in this file is the IMPORT. A bare key for the + // object member is a binding Kotlin does not have, and it outranks the + // import because the fold consults literals before imports. + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/Paths.kt': `package com.example.app.api + +object Paths { + const val ORDERS = "/imported" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.Paths.ORDERS + +object Local { + const val ORDERS = "/local-member" +} +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ORDERS', repo)).toBe('/imported'); + // The qualified spelling still reaches the object member. + expect(resolveKotlinConstant(CONTROLLER_KEY, 'Local.ORDERS', repo)).toBe('/local-member'); + }); + + it('keeps a top-level `const val` shadowing a same-named import', () => { + // The control for the test above: a top-level declaration IS the bare + // binding, so it must keep winning over the import. + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/Paths.kt': `package com.example.app.api + +object Paths { + const val ORDERS = "/imported" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.Paths.ORDERS + +const val ORDERS = "/local" +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ORDERS', repo)).toBe('/local'); + }); + + it('keeps a companion member visible under its bare name INSIDE its class', () => { + // The other control: a companion's members ARE in scope unqualified + // throughout the enclosing class, which is where route annotations sit — + // but ONLY there. The binding is reached from the reference site through + // the enclosing type chain, not from a file-level key, so all three + // spellings below are asserted together: the same name answers one way + // inside `OrderApi` and does not answer at all outside it. + const key = 'src/main/kotlin/com/example/app/web/OrderApi.kt'; + const repo = repoOf({ + [key]: `package com.example.app.web + +class OrderApi { + companion object { + const val ORDERS = "/companion/orders" + } +} +`, + }); + const bare = [{ kind: 'ref', name: 'ORDERS' } as const]; + expect(foldKotlinOperands(key, bare, repo, ['OrderApi'])).toBe('/companion/orders'); + expect(foldKotlinOperands(key, bare, repo)).toBeNull(); + expect(resolveKotlinConstant(key, 'OrderApi.ORDERS', repo)).toBe('/companion/orders'); + }); + + it('does not let a companion outrank a top-level constant outside its class', () => { + // Recording the companion's simple name at FILE level put it in the same + // namespace as the top-level declaration, and companions are recorded + // last, so the companion won every unqualified reference in the file — + // including from a class that is not its own. Kotlin binds the top-level + // `ORDERS` there. Both scopes are asserted from one file; either alone + // passes on an implementation that gets the other wrong. + const key = 'src/main/kotlin/com/example/app/web/Routes.kt'; + const repo = repoOf({ + [key]: `package com.example.app.web + +const val ORDERS = "/top" + +class Holder { + companion object { + const val ORDERS = "/companion" + } +} + +class OrderController +`, + }); + const bare = [{ kind: 'ref', name: 'ORDERS' } as const]; + expect(foldKotlinOperands(key, bare, repo, ['OrderController'])).toBe('/top'); + expect(foldKotlinOperands(key, bare, repo, ['Holder'])).toBe('/companion'); + expect(foldKotlinOperands(key, bare, repo)).toBe('/top'); + }); + + it('does not fall through an unfoldable companion to a top-level constant', () => { + const key = 'src/main/kotlin/com/example/app/web/Routes.kt'; + const repo = repoOf({ + [key]: `package com.example.app.web + +const val ORDERS = "/top" + +class Holder { + companion object { + const val ORDERS = ("/companion") + } +} + +class Other +`, + }); + const bare = [{ kind: 'ref', name: 'ORDERS' } as const]; + expect(foldKotlinOperands(key, bare, repo, ['Holder'])).toBeNull(); + expect(foldKotlinOperands(key, bare, repo, ['Other'])).toBe('/top'); + expect(foldKotlinOperands(key, bare, repo)).toBe('/top'); + }); + + it('scopes each of two colliding companions to its own class', () => { + // Kotlin scopes the member name to its enclosing class, so the same + // spelling means a different constant in each body. One file-level + // namespace could only answer last-wins — both reads returned `/h2`, and + // reordering the two classes flipped both to `/h1`. Both orders are + // asserted, because either alone passes on a last-wins implementation. + const holders = (first: 'A' | 'B'): string => { + const a = `class HolderA { + companion object { + const val ORDERS = "/h1" + } +}`; + const b = `class HolderB { + companion object { + const val ORDERS = "/h2" + } +}`; + return `package com.example.app.web\n\n${first === 'A' ? `${a}\n\n${b}` : `${b}\n\n${a}`}\n`; + }; + const key = 'src/main/kotlin/com/example/app/web/Holders.kt'; + const bare = [{ kind: 'ref', name: 'ORDERS' } as const]; + for (const first of ['A', 'B'] as const) { + const repo = repoOf({ [key]: holders(first) }); + expect(foldKotlinOperands(key, bare, repo, ['HolderA']), `${first} first`).toBe('/h1'); + expect(foldKotlinOperands(key, bare, repo, ['HolderB']), `${first} first`).toBe('/h2'); + } + }); + + it('folds a TOP-LEVEL initializer at file level despite a same-named companion', () => { + // A top-level initializer has an EMPTY scope chain, so `qualifyRef` leaves + // its operand bare and the file-level maps answer it. A file-wide + // companion key WAS one of those maps, so `ROUTE` folded to `/comp/m` + // where Kotlin serves `/top/m` — the case an earlier note in the resolver + // claimed could not arise because sibling initializers "go through the + // scope chain". An empty chain is exactly what that missed. + const key = 'src/main/kotlin/com/example/app/web/Routes.kt'; + const repo = repoOf({ + [key]: `package com.example.app.web + +const val BASE = "/top" +const val ROUTE = BASE + "/m" + +class Holder { + companion object { + const val BASE = "/comp" + } +} +`, + }); + expect(resolveKotlinConstant(key, 'ROUTE', repo)).toBe('/top/m'); + // The companion's own binding is intact, reached the way Kotlin reaches it. + expect(resolveKotlinConstant(key, 'Holder.BASE', repo)).toBe('/comp'); + }); + + it('lets a single-name import win over a companion outside that companion class', () => { + // The fold consults literals before imports, so a file-level companion key + // outranked a genuine `import …Paths.ORDERS` everywhere in the file. The + // import is what Kotlin binds outside `Holder`; inside `Holder`, the + // companion shadows it. + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/Paths.kt': `package com.example.app.api + +object Paths { + const val ORDERS = "/imported" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.Paths.ORDERS + +class Holder { + companion object { + const val ORDERS = "/companion" + } +} +`, + }); + const bare = [{ kind: 'ref', name: 'ORDERS' } as const]; + expect(foldKotlinOperands(CONTROLLER_KEY, bare, repo)).toBe('/imported'); + expect(foldKotlinOperands(CONTROLLER_KEY, bare, repo, ['Other'])).toBe('/imported'); + expect(foldKotlinOperands(CONTROLLER_KEY, bare, repo, ['Holder'])).toBe('/companion'); + }); + + it('reaches an enclosing class companion from a NESTED type', () => { + // Kotlin keeps the companion's members in scope through the nested types + // of its class, so the whole enclosing chain is walked, innermost first — + // and the inner link still wins where both declare the name. + const key = 'src/main/kotlin/com/example/app/web/Nested.kt'; + const repo = repoOf({ + [key]: `package com.example.app.web + +class Outer { + companion object { + const val ORDERS = "/outer" + const val ONLY_OUTER = "/only-outer" + } + + class Inner { + companion object { + const val ORDERS = "/inner" + } + } +} +`, + }); + expect( + foldKotlinOperands(key, [{ kind: 'ref', name: 'ORDERS' }], repo, ['Outer.Inner', 'Outer']), + ).toBe('/inner'); + expect( + foldKotlinOperands(key, [{ kind: 'ref', name: 'ONLY_OUTER' }], repo, [ + 'Outer.Inner', + 'Outer', + ]), + ).toBe('/only-outer'); + }); + + it('keys a nested object by its full enclosing type path', () => { + // `Inner`'s initializer names `P`, which `Inner` does not declare and + // `Outer` does; the scope chain is walked innermost-first, so it means + // `Outer.P` — not the same-named member of the unrelated `Other`. The + // declaration itself is reachable as `Outer.Inner.Q`, never `Inner.Q`. + const key = 'src/main/kotlin/com/example/app/api/Nested.kt'; + const controllerKey = 'src/main/kotlin/com/example/app/web/Controller.kt'; + const nestedImportKey = 'src/main/kotlin/com/example/app/web/NestedImport.kt'; + const memberImportKey = 'src/main/kotlin/com/example/app/web/MemberImport.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +object Other { + const val P = "/wrong" +} + +object Outer { + const val P = "/right" + object Inner { + const val Q = P + "/q" + } +} +`, + [controllerKey]: `package com.example.app.web + +import com.example.app.api.Outer +`, + [nestedImportKey]: `package com.example.app.web + +import com.example.app.api.Outer.Inner +`, + [memberImportKey]: `package com.example.app.web + +import com.example.app.api.Outer.Inner.Q +`, + }); + expect(resolveKotlinConstant(key, 'Outer.Inner.Q', repo)).toBe('/right/q'); + expect(resolveKotlinConstant(controllerKey, 'Outer.Inner.Q', repo)).toBe('/right/q'); + expect(resolveKotlinConstant(controllerKey, 'com.example.app.api.Outer.Inner.Q', repo)).toBe( + '/right/q', + ); + expect(resolveKotlinConstant(nestedImportKey, 'Inner.Q', repo)).toBe('/right/q'); + expect(resolveKotlinConstant(memberImportKey, 'Q', repo)).toBe('/right/q'); + expect(resolveKotlinConstant(key, 'Inner.Q', repo)).toBeNull(); + }); + + it('does not fall through to a file-level constant for an unfoldable sibling', () => { + // `A.R` names `A.BASE`, which does not fold. The answer is the skip floor, + // not the top-level `BASE` that happens to share the simple name. + const key = 'src/main/kotlin/com/example/app/api/Unfoldable.kt'; + const repo = repoOf({ + [key]: `package com.example.app.api + +const val BASE = "/top-level" + +object A { + val BASE = buildBase() + val R = BASE + "/r" +} +`, + }); + expect(resolveKotlinConstant(key, 'A.R', repo)).toBeNull(); + expect(resolveKotlinConstant(key, 'BASE', repo)).toBe('/top-level'); + }); + + it('lets an unfoldable object member leave a same-named import alone', () => { + // A local declaration drops a same-named import only when it SHADOWS it. + // An object member shadows nothing, so dropping the import here would + // floor a reference the language resolves perfectly well. + const repo = repoOf({ + 'src/main/kotlin/com/example/app/api/Paths.kt': `package com.example.app.api + +object Paths { + const val ORDERS = "/imported" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.app.api.Paths.ORDERS + +object Local { + val ORDERS = buildOrders() +} +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ORDERS', repo)).toBe('/imported'); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'Local.ORDERS', repo)).toBeNull(); + }); + }); + + describe('imports resolve on the declared package, not on the path', () => { + it('folds through the file that DECLARES the package, not one whose path imitates it', () => { + // `src/x/com/example/api/ApiPaths.kt` ends with the imported FQN but + // declares `package x.com.example.api`, so it is a different declaration + // entirely. Selecting candidates by path made it beat the real file — and + // because the decoy declares the same member, the fold did not skip, it + // published `/wrong`. + const repo = repoOf({ + 'src/generated/Constants.kt': `package com.example.api + +object ApiPaths { + const val ORDERS = "/right" +} +`, + 'src/x/com/example/api/ApiPaths.kt': `package x.com.example.api + +object ApiPaths { + const val ORDERS = "/wrong" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/right'); + }); + + it('does not let a deep directory impersonate a root-level package', () => { + // `package data` lives at the repository root, which the old + // package-DIRECTORY fallback could not see at all, while + // `src/main/kotlin/com/example/data/` matched `data` by path suffix. Both + // halves are gone: the declared package is the whole test. + const repo = repoOf({ + 'Constants.kt': `package data + +object Constants { + const val ORDERS = "/right" +} +`, + 'src/main/kotlin/com/example/data/AppPaths.kt': `package com.example.data + +object Constants { + const val ORDERS = "/wrong" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import data.Constants +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'Constants.ORDERS', repo)).toBe('/right'); + }); + + it('skips rather than guesses when no file declares the imported package', () => { + // The same import with the real declaration absent. A path-suffix match + // answered `/wrong` here; the honest answer is that the constant is not + // in this repository. + const repo = repoOf({ + 'src/main/kotlin/com/example/data/AppPaths.kt': `package com.example.data + +object Constants { + const val ORDERS = "/wrong" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import data.Constants +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'Constants.ORDERS', repo)).toBeNull(); + }); + + it('skips when two files declare the same fully-qualified name', () => { + // A test-source copy of a production constant: same package, same object, + // different value. Only the copy follows the `/.kt` + // convention, so a file-name tie-break picked it and folded a test-only + // path into a production route. Two declarations of one FQN name no single + // declaration, whichever paths they sit at. + const repo = repoOf({ + 'src/main/kotlin/generated/RoutePaths.kt': `package com.example.api + +object ApiPaths { + const val ORDERS = "/right" +} +`, + 'src/test/kotlin/com/example/api/ApiPaths.kt': `package com.example.api + +object ApiPaths { + const val ORDERS = "/test-only" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBeNull(); + }); + + it('prefers a unique declarer among same-package candidates', () => { + // The declaration itself is stronger evidence than either filename or + // package-only fallback, even when its file also follows the convention. + const repo = repoOf({ + 'src/main/kotlin/com/example/api/ApiPaths.kt': `package com.example.api + +object ApiPaths { + const val ORDERS = "/right" +} +`, + 'src/main/kotlin/com/example/api/Other.kt': `package com.example.api + +object OtherPaths { + const val ITEMS = "/items" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/right'); + }); + + it('reaches the sole file of a package whose name matches nothing', () => { + // The decoy declares a DIFFERENT package, so it is not a candidate at all + // and the unconventionally named `Constants.kt` is the only one left. + // This used to be the one shape the old resolver's safety argument + // covered, and it covered it by emitting nothing. + const repo = repoOf({ + 'src/generated/Constants.kt': `package com.example.api + +object ApiPaths { + const val ORDERS = "/right" +} +`, + 'src/x/com/example/api/ApiPaths.kt': `package x.com.example.api + +object ApiPaths { + const val OTHER = "/other" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/right'); + }); + + it('rejects a candidate carrying no recorded package', () => { + // `RepoConstants` is typed over the agnostic shape, so an entry some other + // producer put there has no `packageName`. Unknown is not "the default + // package": the candidate is rejected, and the fold floors to skip. + const key = 'src/main/kotlin/com/example/api/ApiPaths.kt'; + const foreign = extractKotlinModuleConstants( + parse(`package com.example.api + +object ApiPaths { + const val ORDERS = "/right" +} +`), + ); + const repo = new Map(); + // Stripped to the agnostic shape: same maps, no `packageName`. + repo.set(key, { + literals: foreign.literals, + exprs: foreign.exprs, + imports: foreign.imports, + }); + repo.set( + CONTROLLER_KEY, + extractKotlinModuleConstants( + parse(`package com.example.app.web + +import com.example.api.ApiPaths +`), + ), + ); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBeNull(); + }); + + it('matches a package whose segment is backtick-quoted on one side only', () => { + // `` package com.example.`api` `` and `package com.example.api` name the + // SAME package: the quotes are lexical syntax, not part of the name. The + // declared package was recorded with the backticks and the import required + // an exact string match, so the one real candidate was rejected and the + // route lost. All three spellings are asserted, because either side can + // carry the quotes — a declaration may quote a segment an import spells + // plainly, and an import may quote one the declaration does not. + const declaration = (pkg: string): string => `package ${pkg} + +object ApiPaths { + const val ORDERS = "/right" +} +`; + const importing = (spec: string): string => `package com.example.app.web + +import ${spec} +`; + const CONSTS = 'src/main/kotlin/com/example/api/ApiPaths.kt'; + for (const [declared, spec] of [ + ['com.example.`api`', 'com.example.api.ApiPaths'], + ['com.example.api', 'com.example.`api`.ApiPaths'], + ['com.example.`api`', 'com.example.`api`.ApiPaths'], + ] as const) { + const repo = repoOf({ + [CONSTS]: declaration(declared), + [CONTROLLER_KEY]: importing(spec), + }); + expect( + resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo), + `${declared} <- ${spec}`, + ).toBe('/right'); + } + }); + + it('still refuses a package that merely resembles the quoted one', () => { + // The control for the test above: unquoting compares NAMES, it does not + // widen the match. `com.example.other` is a different package however + // either side spells it, so the fold floors to skip rather than reaching + // for the only file it can see. + const repo = repoOf({ + 'src/main/kotlin/com/example/other/ApiPaths.kt': `package com.example.\`other\` + +object ApiPaths { + const val ORDERS = "/wrong" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.api.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBeNull(); + }); + + it('folds through a KEYWORD package segment, which Kotlin can only spell quoted', () => { + // `package com.example.fun` does not compile — the segment must be + // `` `fun` `` on both sides. Unquoting must not break the case that only + // works BECAUSE it is quoted, so this is the control for the pair above. + const repo = repoOf({ + 'src/main/kotlin/com/example/fun/ApiPaths.kt': `package com.example.\`fun\` + +object ApiPaths { + const val ORDERS = "/right" +} +`, + [CONTROLLER_KEY]: `package com.example.app.web + +import com.example.\`fun\`.ApiPaths +`, + }); + expect(resolveKotlinConstant(CONTROLLER_KEY, 'ApiPaths.ORDERS', repo)).toBe('/right'); + }); + + it('matches a backtick-quoted declaration name and reference', () => { + // Quoting reaches declarations too, and the two sides need not agree: + // `` object `ApiPaths` `` is keyed `ApiPaths.ORDERS`, and a reference + // written `` `ApiPaths`.`ORDERS` `` parses to that same name. + const key = 'src/main/kotlin/com/example/api/ApiPaths.kt'; + const repo = repoOf({ + [key]: `package com.example.api + +object \`ApiPaths\` { + const val \`ORDERS\` = "/right" +} +`, + }); + expect(resolveKotlinConstant(key, 'ApiPaths.ORDERS', repo)).toBe('/right'); + expect(parseKotlinConstOperands(firstInitializer('val X = `ApiPaths`.`ORDERS`\n'))).toEqual([ + { kind: 'ref', name: 'ApiPaths.ORDERS' }, + ]); + }); + }); + + describe('the fold is bounded in output, depth and time', () => { + /** `object Doubling { const val X = ; val X = X + X … }`. */ + const doublingChain = (levels: number, leaf: string): string => { + const lines = [` const val X${levels} = "${leaf}"`]; + for (let i = levels - 1; i >= 0; i--) lines.push(` val X${i} = X${i + 1} + X${i + 1}`); + return `package com.example.app.api\n\nobject Doubling {\n${lines.join('\n')}\n}\n`; + }; + const DOUBLING_KEY = 'src/main/kotlin/com/example/app/api/Doubling.kt'; + + it('folds a 30-level shared-descendant DAG instead of exploring 2^30 paths', () => { + // Every intermediate value here is the EMPTY string, so MAX_FOLD_LENGTH + // never fires and only the success memo keeps this from re-folding each + // child once per reference — O(2^depth). The assertion is the explicit + // timeout: a regression does not fail this test slowly, it fails it. + const repo = repoOf({ [DOUBLING_KEY]: doublingChain(30, '') }); + expect(resolveKotlinConstant(DOUBLING_KEY, 'Doubling.X0', repo)).toBe(''); + }, 5_000); + + it('caps output at MAX_FOLD_LENGTH, which the depth cap cannot bound', () => { + // Same shape with a one-character leaf: output doubles per level while + // depth only increments, so 13 levels land exactly on MAX_FOLD_LENGTH and + // 14 overrun it. Pinned from both sides — a chain deep enough to matter in + // practice (30 levels, a gigabyte of string) is the same code path. + const foldOf = (levels: number): string | null => + resolveKotlinConstant( + DOUBLING_KEY, + 'Doubling.X0', + repoOf({ [DOUBLING_KEY]: doublingChain(levels, 'a') }), + ); + expect(foldOf(13)).toHaveLength(MAX_FOLD_LENGTH); + expect(foldOf(14)).toBeNull(); + }); + + /** `object Link { const val X = "/end"; val X = X … }`. */ + const referenceChain = (links: number): string => { + const lines = [` const val X${links} = "/end"`]; + for (let i = links - 1; i >= 0; i--) lines.push(` val X${i} = X${i + 1}`); + return `package com.example.app.api\n\nobject Link {\n${lines.join('\n')}\n}\n`; + }; + const LINK_KEY = 'src/main/kotlin/com/example/app/api/Link.kt'; + + it('resolves a chain inside the cross-file depth cap but stops past it', () => { + // Each link costs one level of `resolveWithState`, so a 30-link chain + // resolves and a 40-link one runs into the cap. Asserted from both sides: + // a bare `toBeNull()` would also pass if the fold had stopped working. + expect( + resolveKotlinConstant(LINK_KEY, 'Link.X0', repoOf({ [LINK_KEY]: referenceChain(30) })), + ).toBe('/end'); + expect( + resolveKotlinConstant(LINK_KEY, 'Link.X0', repoOf({ [LINK_KEY]: referenceChain(40) })), + ).toBeNull(); + }); + + it('caps operand parsing on a pathologically long `+` chain', () => { + // `A + B + C` nests left-associatively, so an n-term concatenation is n-1 + // levels deep and a long enough one would recurse without the parse cap. + const chainOf = (terms: number): string => + `val X = ${Array.from({ length: terms }, (_, i) => `"/${i}"`).join(' + ')}`; + expect(parseKotlinConstOperands(firstInitializer(chainOf(60)))).toHaveLength(60); + expect(parseKotlinConstOperands(firstInitializer(chainOf(80)))).toBeNull(); + }); + }); + + describe('isKotlinConstantFile gate', () => { + it('admits every shape the extractor harvests', () => { + expect(isKotlinConstantFile(CONSTS_SRC)).toBe(true); + expect(isKotlinConstantFile('const val ORDERS = "/api/v1/orders"')).toBe(true); + expect(isKotlinConstantFile('val ORDERS = "/api/v1/orders"')).toBe(true); + expect(isKotlinConstantFile('object O { val ORDERS: String = "/api/v1/orders" }')).toBe(true); + expect( + isKotlinConstantFile('class C { companion object { const val O = "/api/v1/orders" } }'), + ).toBe(true); + }); + + it('rejects a file with no constant carrier at all', () => { + expect( + isKotlinConstantFile(`package com.example.app.web + +class OrderService { + fun list(): List = emptyList() +} +`), + ).toBe(false); + }); + + it('admits top-level vals without admitting locals or constructor properties', () => { + expect( + isKotlinConstantFile(`package com.example.app.api + +@JvmField +val ORDERS: String = "/api/v1/orders" +`), + ).toBe(true); + expect( + isKotlinConstantFile(`val ORDERS: + String + = "/api/v1/orders" +`), + ).toBe(true); + expect(isKotlinConstantFile('val ORDERS: String get() = "/computed"')).toBe(true); + expect(isKotlinConstantFile('val ORDERS by lazy { "/computed" }')).toBe(true); + expect(isKotlinConstantFile('val `ORDER PATH` = "/api/v1/orders"')).toBe(true); + expect( + isKotlinConstantFile(`fun route(): String { + val ORDERS = "/local" + return ORDERS +} +`), + ).toBe(false); + expect(isKotlinConstantFile('class Holder { val ORDERS = "/instance" }')).toBe(false); + expect(isKotlinConstantFile('data class Route(val path: String = "/constructor")')).toBe( + false, + ); + }); + + it('ignores declaration-shaped text in comments and literals', () => { + expect(isKotlinConstantFile('// const val ORDERS = "/comment"')).toBe(false); + expect( + isKotlinConstantFile('/* outer /* const val ORDERS = "/nested-comment" */ end */'), + ).toBe(false); + expect(isKotlinConstantFile('val text() = "const val ORDERS = \\"/string\\""')).toBe(false); + expect(isKotlinConstantFile('val text() = """const val ORDERS = "/raw-string\""""')).toBe( + false, + ); + expect( + isKotlinConstantFile(`fun route(): String { + val open = '{' + val close = '}' + return "$open$close" +} +`), + ).toBe(false); + }); + + it('admits a backtick-quoted name, because the extractor resolves one', () => { + // A gate NARROWER than the extractor costs a fact: the file is never + // parsed into the repo map, so a cross-file reference to the constant + // floors to skip. `unquoteKotlinIdentifier` strips the quoting everywhere + // a name becomes a key, so these declarations are ones this module folds. + expect(isKotlinConstantFile('const val `ORDERS` = "/api/v1/orders"')).toBe(true); + expect(isKotlinConstantFile('object O { val `ORDERS`: String = "/api/v1/orders" }')).toBe( + true, + ); + expect( + isKotlinConstantFile('class C { companion object { const val `O` = "/orders" } }'), + ).toBe(true); + }); + + it('does not let the backtick arm widen into a file with no carrier', () => { + // The control for the arm above: accepting backticks must not turn the + // gate into "any file mentioning val", which is the whole repository. + expect( + isKotlinConstantFile(`package com.example.app.web + +class OrderService { + fun list(): List { + val \`local name\` = "not a constant" + return listOf(\`local name\`) + } +} +`), + ).toBe(false); + }); + }); +}); From b059ab3541ea68c2ce292955fc367a5de04b39ea Mon Sep 17 00:00:00 2001 From: Przemek Poppe <139065333+p-poppe@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:47:41 +0200 Subject: [PATCH 13/28] PHP: detect generated-client Request(method, host . resourcePath) consumer shape (#3079) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(group/php): detect generated-client Request(method, host . resourcePath) openapi-generator-php / swagger-codegen PHP clients build every operation as `$resourcePath = '/foo/bar'; ...; new Request($method, $host . $resourcePath);` — a shape the PHP consumer patterns didn't cover (only `$client->verb('/path')` literal calls were matched), documented in the module's own docblock as a follow-up ("constant-folding the surrounding scope"). Adds a pattern for `new [Qualified\]Request(...)` constructor calls, with a conservative, single-scope backward constant fold: the last variable in the path argument's concatenation chain (generated clients build ` . `) is resolved to a `$var = '';` assignment in the same enclosing function/method body (or file scope) if one exists earlier in the same scope. No interprocedural resolution — a miss just leaves the endpoint undetected, never a wrong one. The HTTP verb is often itself a parameter in these generated clients (not a literal at the call site), so when it can't be resolved to a literal the detection reports a wildcard method (`'*'`), consistent with this project's existing manifest-link convention for a contract whose verb isn't pinned. `hasConsumerSignals` is widened to stay a proven superset of what `scan()` now detects (required by its own contract, checked by `http-consumer-signals.test.ts`). The docblock notes this is a deliberately narrow, single-scope fallback, not this language's entry into the shared cross-file constant-fold the other languages use (`constant-resolver.ts`, wired in via `java-const-resolver.ts` / `python-const-resolver.ts` / `js-const-resolver.ts`) — PHP has no such binding yet; adding one is a separate, larger project (this repo's PHP import resolution for `use`-statements is its own multi-file subsystem built for symbol/scope resolution, not constant extraction) and is out of scope here. Tests: 7 scan()-level cases (resolution across a member-access host, fully-qualified class name, purely literal call, negative — different function scope, negative — non-Request constructor, negative — non-HTTP literal, picking the LAST var in a 3-part concatenation), plus 2 new hasConsumerSignals cases and a negative. `tsc --noEmit` clean, `eslint` clean, full test/unit/group green (1035+/1037; 2 pre-existing native EBUSY failures on a `.lbug` file in bridge-meta-swap-window.test.ts, unrelated subsystem, reproduces in isolation on unchanged upstream too). * fix(group/php): fix two code-review findings in guzzle-request-ctor 1. lastConcatVariable silently returned the WRONG variable for a parenthesized right operand: `$host . ($resourcePath . $suffix)` fell through to the left operand (unhandled parenthesized_expression) and returned $host instead of looking inside the parens. Restored parenthesis unwrapping (present in an earlier draft, dropped during a simplification pass that didn't account for this fallthrough). 2. resolveLocalStringLiteral stopped at the nearest compound_statement, so a `new Request(...)` call nested in `if`/`try`/`foreach` inside the same function couldn't see an assignment made just above that block — despite the docblock's claim of covering the "enclosing function/method body". Now widens level by level (search the immediate block's preceding statements, then its own enclosing block, and so on), stopping at `program` so it still never crosses into a different function or the containing class body — verified by a regression test asserting exactly that boundary. Also documents the line-number choice (path argument, not the `new Request(` call site — the two differ for this pattern's characteristically multi-line calls) inline, matching the other three consumer patterns' convention in this file. 4 new regression tests (35 total in this file's suite): parenthesized right operand no longer mismatches, enclosing-block resolution across an `if`, and a negative case proving the widened search still respects the function boundary. tsc --noEmit clean, eslint clean. * fix(group/php): address gitnexus-check bot review on PR #3079 1. resolveLocalStringLiteral fell through an intervening non-literal reassignment: `$v = '/old'; $v = buildPath(); new Request(..., $v)` resolved to '/old' even though $v never holds that literal at the call site. The NEAREST assignment to the target variable now decides the outcome unconditionally — a non-string RHS stops the search (returns null) instead of letting the scan continue past it to an older, shadowed literal. This was a real "wrong answer", not a miss, directly contradicting the function's own documented invariant. 2. lastConcatVariable recursed into every binary_expression regardless of operator, so `$host && $resourcePath`, `$host + $resourcePath`, and `$host ?? $resourcePath` were walked exactly like `.` concatenation. Now checks operator === '.' before recursing. 3. hasConsumerSignals matches case-insensitively (`/i`), correctly, since PHP class names are case-insensitive at the language level — but scan() compared the resolved class name to 'Request' case-sensitively, so a valid `new request(...)` / `new \NS\REQUEST(...)` call would pass the parse-skip gate as a signal and then be silently dropped by scan() itself. Both sides now agree (case-insensitive compare in scan() too). 4. The first test's own PHP source assigned `$method = 'POST';` as a local variable but asserted `method: '*'` with a comment calling it "a parameter" — it wasn't; it was exactly the same locally-resolvable shape as $resourcePath. Fixed by (a) rewriting that test's source to show $method as a genuine function parameter (the shape generated clients actually use — the verb is fixed by the caller of the builder method), which is what the test intended to demonstrate, and (b) actually implementing symmetric resolution: method now resolves through the same resolveLocalStringLiteral fold as path when it IS a local variable, with a new test proving that case resolves to a literal method instead of a wildcard. 5 new regression tests (39 total in this file's suite, up from 35): non-literal-reassignment shadowing, non-concatenation operator rejected, case-insensitive class name match, and local-variable method resolution. tsc --noEmit clean, eslint clean. * fix(group/php): address second round of gitnexus-check bot review 1. Backward fold missed reassignments nested inside a preceding if/foreach/ try/switch: the scan only recognized direct expression_statement siblings as candidate assignments, so `if ($cond) { $v = '/new'; }` right before the call was invisible, and an OLDER, now-shadowed literal outside that block was returned instead — a real wrong answer whenever that branch runs. Any non-assignment sibling that contains an assignment to the target ANYWHERE inside it now stops the search (miss) rather than being skipped over, since whether that branch ran is unknown. 2. Level-by-level scope widening crossed anonymous-function boundaries without checking PHP's actual capture rule: closures capture NOTHING automatically, only variables listed in `use (...)` are visible inside — unlike arrow functions, which auto-capture everything and have no `compound_statement` body (never seen as a scope by this walk at all). Widening past a closure's body now checks its `use (...)` clause first; real PHP would throw "Undefined variable" for anything not captured, not resolve to a value from the enclosing scope. 3. lastConcatVariable still fell through to the LEFT operand whenever the right one wasn't a variable-or-nestable-expression — `new Request($m, $host . '/users')` (a trailing string literal, not a variable) resolved to $host instead of recognizing there's simply nothing to resolve at that position. Removed the left-operand fallback entirely: the rightmost position decides, full stop, matching the function's own "single lookup, not a fallback list" docblock (which the previous round already stated but the code didn't yet fully honor for this case). Also strengthened a test that the bot correctly flagged as non-diagnostic: "ignores an unrelated constructor" used an unresolvable $resourcePath, so it would have passed even with the class-name filter deleted. Now uses a fully resolvable path so the class-name filter is what the assertion actually exercises. 4 new regression tests (43 total, up from 39): shadowed-by-conditional- reassignment, closure boundary without use()-capture (negative), closure boundary WITH use()-capture (positive control), and trailing-literal concatenation no longer mistaken for the host variable. tsc --noEmit clean, eslint clean. * chore: trigger re-review (previous gitnexus-check report cited stale line numbers) * fix(group/php): stop scope widening at a function/method boundary The digest posted on PR #3079 (verified against the current file, not the stale HEAD it was generated from — three of its four findings were already fixed in prior commits) reproduced a real, still-present fourth issue: after exhausting a method's own body, widening continued straight to `program` (file/script scope) and could resolve a top-level variable into a class method — but PHP methods (and plain functions) have NO access to file-level variables without an explicit `global $v;`, which this resolver intentionally never adds support for. A file-level `$resourcePath = '/x';` could therefore leak into an unrelated method's `new Request(...)` as a real, wrong answer. Widening now stops unconditionally at a `function_definition` or `method_declaration` boundary — these get no automatic capture and no implicit global in PHP, unlike closures (already handled: an `anonymous_function` boundary stops unless `$target` is `use()`-captured). The call-site-at-file-scope case still resolves correctly, since `program` is reached directly there with no boundary to cross. 3 new regression tests (46 total): file-scope variable does not leak into a class method, does not leak into a plain top-level function either, and a positive control confirming file-scope-to-file-scope resolution still works when there's no function boundary at all. tsc --noEmit clean, eslint clean. --------- Co-authored-by: Gergő Magyar --- .../group/extractors/http-patterns/php.ts | 296 ++++++++++++++- .../unit/group/http-consumer-signals.test.ts | 6 + .../group/php-guzzle-request-ctor.test.ts | 352 ++++++++++++++++++ 3 files changed, 643 insertions(+), 11 deletions(-) create mode 100644 gitnexus/test/unit/group/php-guzzle-request-ctor.test.ts diff --git a/gitnexus/src/core/group/extractors/http-patterns/php.ts b/gitnexus/src/core/group/extractors/http-patterns/php.ts index bf2eb7aba..65a1896c9 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/php.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/php.ts @@ -15,20 +15,36 @@ import type { HttpDetection, HttpLanguagePlugin } from './types.js'; * Providers: * - Laravel `Route::get/post/...` * - * Consumers (string-literal URLs only): + * Consumers (string-literal URLs only, unless noted): * - Laravel HTTP client: `Http::get/post/put/delete/patch($url)` * - Guzzle / generic object method: `$client->get/post/...($url)` * - `file_get_contents($url)` + * - `new Request($method, $host . $resourcePath)` — the openapi-generator-php + * / swagger-codegen client shape. `$resourcePath` is resolved via a + * single-scope backward constant fold (see `resolveLocalStringLiteral`), + * not a string literal at the call site itself. * * The pipeline already uses `PHP.php_only` for ingesting plain `.php` * files (see `core/tree-sitter/parser-loader.ts`), and we do the same * here so Laravel route files are parsed with the right grammar dialect. * - * Scope notes: consumer patterns match string literals only. URLs built - * via binary concatenation (`$base . '/path'`), `sprintf`, or config - * lookup (`config('services.foo.base').'/path'`) are intentionally left - * for a follow-up — they require constant-folding the surrounding - * scope to be meaningful. + * Scope notes: consumer patterns match string literals only, with one + * narrow exception (above). URLs built via `sprintf`, config lookup + * (`config('services.foo.base').'/path'`), or a variable resolved from + * outside its own function/method body are intentionally left for a + * follow-up — they require constant-folding beyond one local scope to + * be meaningful. + * + * That narrow exception (`resolveLocalStringLiteral`) is a temporary, + * single-scope fallback, not this language's entry into the shared + * cross-file constant-fold used by the other languages in this plugin + * (`constant-resolver.ts`, wired in via `java-const-resolver.ts` / + * `python-const-resolver.ts` / `js-const-resolver.ts`). PHP has no such + * binding yet — adding one is a real, separate project (this repo's PHP + * import resolution for `use`-statements is its own multi-file subsystem + * under `ingestion/import-resolvers/php.ts`, built for symbol/scope + * resolution, not constant extraction) and is intentionally out of scope + * here. Tracked as a follow-up, not silently punted. */ const LARAVEL_ROUTE_SPEC: PatternSpec> = { @@ -71,11 +87,31 @@ const FILE_GET_CONTENTS_SPEC: PatternSpec> = { `, }; +/** + * `new Request($method, $host . $resourcePath)` — the shape swagger-codegen / + * openapi-generator-php emit for every operation of a generated API client + * (Guzzle's `\GuzzleHttp\Psr7\Request`, or a bare `Request` behind a `use` + * import). Matches both `(name)` and `(qualified_name)` class references; + * `scan()` below filters to the last path segment being exactly `Request` + * and resolves the concatenated path argument (see `resolveLocalStringLiteral`). + */ +const GUZZLE_REQUEST_CTOR_SPEC: PatternSpec> = { + meta: {}, + query: ` + (object_creation_expression + [(name) (qualified_name)] @class + (arguments + . (argument (_) @methodArg) + . (argument (_) @pathArg))) + `, +}; + interface PhpPatternBundle { laravelRoute: CompiledPatterns>; httpFacade: CompiledPatterns>; guzzleMember: CompiledPatterns>; fileGetContents: CompiledPatterns>; + guzzleRequestCtor: CompiledPatterns>; } const mk = (spec: PatternSpec>, suffix: string) => @@ -90,6 +126,7 @@ const PHP_PATTERNS: PhpPatternBundle = { httpFacade: mk(HTTP_FACADE_SPEC, 'http-facade'), guzzleMember: mk(GUZZLE_MEMBER_SPEC, 'guzzle-member'), fileGetContents: mk(FILE_GET_CONTENTS_SPEC, 'file-get-contents'), + guzzleRequestCtor: mk(GUZZLE_REQUEST_CTOR_SPEC, 'guzzle-request-ctor'), }; /** @@ -129,6 +166,183 @@ function isHttpUrlLiteral(path: string): boolean { return path.startsWith('http://') || path.startsWith('https://'); } +/** + * Last identifier segment of a class-name reference: `(name)` returns its + * own text, `(qualified_name)` returns the text of its last child (the + * unqualified class name — `\GuzzleHttp\Psr7\Request` → `Request`). + */ +function lastNameSegment(node: import('tree-sitter').SyntaxNode): string { + if (node.type === 'qualified_name') { + const last = node.child(node.childCount - 1); + return last ? last.text : node.text; + } + return node.text; +} + +/** + * Return the variable at the LAST position of a `.`-concatenation + * expression, if (and only if) that position is a plain variable — + * generated clients build ` . `, so the path segment + * is the one closest to the end. + * + * No fallback to an earlier operand: if the rightmost position is anything + * other than a variable, a parenthesized sub-expression, or a nested `.` + * concatenation (a literal, a function call, ...), that position is a real + * value we simply can't resolve — falling back to an EARLIER operand would + * silently substitute a different value (e.g. the host) for the one that's + * actually there. `null` here is a miss, not a signal to keep looking. + */ +function lastConcatVariable( + node: import('tree-sitter').SyntaxNode, +): import('tree-sitter').SyntaxNode | null { + if (node.type === 'variable_name') return node; + if (node.type === 'parenthesized_expression') { + const inner = node.namedChild(0); + return inner ? lastConcatVariable(inner) : null; + } + if (node.type === 'binary_expression') { + const operator = node.childForFieldName('operator'); + if (!operator || operator.text !== '.') return null; // not concatenation + const right = node.childForFieldName('right'); + return right ? lastConcatVariable(right) : null; + } + return null; +} + +/** + * True if `node`'s subtree assigns to `$target` ANYWHERE inside it, at any + * depth (including inside nested functions — deliberately over-broad: a + * false positive here only costs a miss in the caller, never a wrong + * answer, so there's no need to be precise about scoping inside the probe + * itself). + */ +function containsAssignmentTo(node: import('tree-sitter').SyntaxNode, target: string): boolean { + if (node.type === 'assignment_expression') { + const lhs = node.childForFieldName('left'); + if (lhs && lhs.type === 'variable_name' && lhs.text === target) return true; + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child && containsAssignmentTo(child, target)) return true; + } + return false; +} + +/** + * True if an `anonymous_function` node's `use (...)` clause lists + * `$target`. PHP closures capture NOTHING automatically — only variables + * named in `use (...)` are visible inside — unlike arrow functions + * (`fn() => ...`), which auto-capture everything by value and have no + * `compound_statement` body of their own, so they're never seen as a + * `scope` by the walk below in the first place. + */ +function anonymousFunctionCaptures( + anonFn: import('tree-sitter').SyntaxNode, + target: string, +): boolean { + for (let i = 0; i < anonFn.namedChildCount; i++) { + const child = anonFn.namedChild(i); + if (!child || child.type !== 'anonymous_function_use_clause') continue; + for (let j = 0; j < child.namedChildCount; j++) { + const v = child.namedChild(j); + if (v && v.type === 'variable_name' && v.text === target) return true; + } + return false; // has a use(...) clause, but $target isn't in it + } + return false; // no use(...) clause at all — nothing is captured +} + +/** + * Best-effort, single-scope constant fold: given a `variable_name` node + * referenced inside a `new Request(...)` argument, walk BACKWARD through + * the preceding statements of its immediately enclosing function/method + * body (or file scope, for top-level script code) looking for the nearest + * `$var = '';` assignment. + * + * "Enclosing body" is resolved level by level, not just the nearest + * `compound_statement` — a call site nested in `if`/`foreach`/`try` inside + * that function is still within the same function/method body, and a + * preceding assignment above that conditional must still be found. Each + * level searches only its own preceding siblings, then the search + * continues from the enclosing block itself one level up, UNLESS that + * block IS the body of a function/method/closure: + * - a regular `function_definition` or `method_declaration` boundary + * always stops the search — PHP gives a function or method no access + * to anything outside its own body (no automatic capture, no implicit + * global), so widening past one into the containing class or + * file-level scope would resolve a variable the call site could never + * actually see at runtime; + * - an `anonymous_function` boundary stops UNLESS `$target` is + * explicitly captured via `use (...)` — closures capture nothing + * automatically either. + * It stops at `program` regardless, for the case where the call site was + * at file/script scope all along. + * + * A preceding sibling that ISN'T a plain assignment but might reassign the + * target somewhere inside itself (an `if`/`foreach`/`try`/`switch`, ...) + * stops the search rather than being skipped over: whether that branch ran + * is unknown, so an older literal further back can't be trusted either. + * + * Deliberately conservative and bounded — no interprocedural resolution, + * no constant/property lookups. A miss just means the endpoint stays + * undetected, never a wrong one: this is exactly the class of case the + * module docblock flags as in-scope only for one local scope. + */ +function resolveLocalStringLiteral(varNode: import('tree-sitter').SyntaxNode): string | null { + const target = varNode.text; // includes the `$` sigil, e.g. "$resourcePath" + let cursor: import('tree-sitter').SyntaxNode = varNode; + + for (;;) { + let scope: import('tree-sitter').SyntaxNode | null = cursor.parent; + while (scope && scope.type !== 'compound_statement' && scope.type !== 'program') { + scope = scope.parent; + } + if (!scope) return null; + + let stmt: import('tree-sitter').SyntaxNode | null = cursor; + while (stmt && stmt.parent !== scope) stmt = stmt.parent; + if (!stmt) return null; + + let sibling = stmt.previousNamedSibling; + while (sibling) { + if (sibling.type === 'expression_statement') { + const inner = sibling.namedChild(0); + if (inner && inner.type === 'assignment_expression') { + const lhs = inner.childForFieldName('left'); + if (lhs && lhs.type === 'variable_name' && lhs.text === target) { + // The NEAREST assignment to this variable wins, full stop — an + // older literal further back is shadowed by this one even when + // this one isn't itself a resolvable string (`$v = f();`). + const rhs = inner.childForFieldName('right'); + return rhs && rhs.type === 'string' ? phpStringText(rhs) : null; + } + } + } else if (containsAssignmentTo(sibling, target)) { + return null; // reassigned somewhere inside a conditional/loop/try + } + sibling = sibling.previousNamedSibling; + } + + if (scope.type === 'program') return null; + const enclosing = scope.parent; + if (enclosing && enclosing.type === 'anonymous_function') { + // Closures capture nothing automatically — only what's use()'d. + if (!anonymousFunctionCaptures(enclosing, target)) return null; + } else if ( + enclosing && + (enclosing.type === 'function_definition' || enclosing.type === 'method_declaration') + ) { + // A regular function or method boundary — NOT a closure. PHP gives + // these no access to anything outside their own body (no automatic + // capture, no implicit global): widening past one into the + // containing class body or file-level scope would resolve a + // variable the call site could never actually see at runtime. + return null; + } + cursor = scope; // one block up: search resumes from this block's own position + } +} + export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'php-http', language: PHP.php_only, @@ -136,12 +350,16 @@ export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { // ingestion, so the graph is authoritative for PHP providers (#2138 Part 2). routeCoverage: 'complete', // Consumer signals scan() can detect: Laravel `Http::`, Guzzle client - // `->get/post/.../request(...)`, and `file_get_contents` of an HTTP URL. A - // provider-covered file with any of these must still be parsed (ingestion - // emits no FETCHES for PHP). Conservative — the `->verb(` shape over-matches - // ordinary method calls, which only costs a parse, never data. + // `->get/post/.../request(...)`, `file_get_contents` of an HTTP URL, and a + // generated-client `new ...Request(...)` constructor call. A provider-covered + // file with any of these must still be parsed (ingestion emits no FETCHES for + // PHP). Conservative — the `->verb(`/`new ...Request(` shapes over-match + // ordinary method calls and unrelated constructors, which only costs a + // parse, never data. hasConsumerSignals(content) { - return /Http::|file_get_contents|->\s*(get|post|put|delete|patch|request)\s*\(/i.test(content); + return /Http::|file_get_contents|->\s*(get|post|put|delete|patch|request)\s*\(|new\s+[\\\w]*Request\s*\(/i.test( + content, + ); }, scan(tree) { const out: HttpDetection[] = []; @@ -222,6 +440,62 @@ export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { }); } + for (const match of runCompiledPatterns(PHP_PATTERNS.guzzleRequestCtor, tree)) { + const classNode = match.captures.class; + const methodArg = match.captures.methodArg; + const pathArg = match.captures.pathArg; + if (!classNode || !methodArg || !pathArg) continue; + // PHP class names are case-insensitive at the language level, and + // `hasConsumerSignals` above matches case-insensitively (`/i`) for + // the same reason — this comparison must agree with it, or a valid + // `new request(...)` / `new \NS\REQUEST(...)` call would be waved + // through the parse-skip gate as a signal and then silently dropped + // here. + if (lastNameSegment(classNode).toLowerCase() !== 'request') continue; + + // Path: a direct string literal, or the last variable in a + // concatenation chain (see `lastConcatVariable`) resolved to a + // locally-assigned literal. + let path: string | null = null; + if (pathArg.type === 'string') { + path = phpStringText(pathArg); + } else { + const lastVar = lastConcatVariable(pathArg); + path = lastVar ? resolveLocalStringLiteral(lastVar) : null; + } + if (path === null || !isHttpClientPath(path)) continue; + + // The HTTP verb is a literal, a local variable resolved the same way + // as the path (see `resolveLocalStringLiteral` above), or — commonly + // in generated clients — a parameter of the enclosing builder method + // fixed by ITS caller, not by this call site. That last case needs + // the same interprocedural reach the module docblock rules out, so it + // falls through to a wildcard verb, matching this project's own + // convention for a contract whose verb isn't pinned (see manifest + // links, `http::*::`). + let method: string | null = null; + if (methodArg.type === 'string') { + method = phpStringText(methodArg); + } else if (methodArg.type === 'variable_name') { + method = resolveLocalStringLiteral(methodArg); + } + + out.push({ + role: 'consumer', + framework: 'guzzle-request-ctor', + method: method ? method.toUpperCase() : '*', + path, + name: null, + // Line of the path ARGUMENT, not the `new Request(` call — same + // choice the other three consumer patterns in this file make, but + // this is the one pattern where the two routinely differ (generated + // clients wrap the call across multiple lines). Line-span + // containment still resolves to the right symbol either way. + line: pathArg.startPosition.row + 1, + confidence: 0.6, + }); + } + return out; }, }; diff --git a/gitnexus/test/unit/group/http-consumer-signals.test.ts b/gitnexus/test/unit/group/http-consumer-signals.test.ts index 087524cfe..6d98f44c7 100644 --- a/gitnexus/test/unit/group/http-consumer-signals.test.ts +++ b/gitnexus/test/unit/group/http-consumer-signals.test.ts @@ -46,6 +46,8 @@ describe('PHP hasConsumerSignals — superset of scan() consumer idioms', () => ['Laravel Http facade', "Http::get('/api/x');"], ['Guzzle member call', "$client->post('/api/x', []);"], ['file_get_contents', "file_get_contents('https://x/api');"], + ['bare Request constructor', 'new Request($method, $host . $resourcePath);'], + ['namespaced Request constructor', "new Foo\\Bar\\Request('GET', $url);"], ])('detects %s', (_label, src) => { expect(has(PHP_HTTP_PLUGIN, src)).toBe(true); }); @@ -53,6 +55,10 @@ describe('PHP hasConsumerSignals — superset of scan() consumer idioms', () => it('returns false for a pure Laravel route file (provider only)', () => { expect(has(PHP_HTTP_PLUGIN, "Route::get('/api/a/list', 'AController@list');")).toBe(false); }); + + it('returns false for an unrelated constructor call (not *Request)', () => { + expect(has(PHP_HTTP_PLUGIN, 'new Response($body, 200);')).toBe(false); + }); }); describe('Python hasConsumerSignals — superset of scan() consumer idioms', () => { diff --git a/gitnexus/test/unit/group/php-guzzle-request-ctor.test.ts b/gitnexus/test/unit/group/php-guzzle-request-ctor.test.ts new file mode 100644 index 000000000..9e6f44b4a --- /dev/null +++ b/gitnexus/test/unit/group/php-guzzle-request-ctor.test.ts @@ -0,0 +1,352 @@ +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import PHP from 'tree-sitter-php'; +import { PHP_HTTP_PLUGIN } from '../../../src/core/group/extractors/http-patterns/php.js'; + +const parser = new Parser(); +parser.setLanguage(PHP.php_only); + +const scan = (src: string) => PHP_HTTP_PLUGIN.scan(parser.parse(src)); +const consumers = (src: string) => scan(src).filter((d) => d.role === 'consumer'); + +describe('PHP guzzle-request-ctor pattern', () => { + it('resolves a locally-assigned $resourcePath concatenated with a member-access host, method is a real parameter', () => { + // $method is a FUNCTION PARAMETER here (the shape openapi-generator-php + // actually emits — the verb is fixed by the caller of this builder + // method, not assigned inside its body), not a local variable that + // happens to share the resolver's single-scope shape. A local + // `$method = 'POST';` immediately before the call would in fact resolve + // via the same fold as `$resourcePath` — that's a different case, + // covered separately below. + const src = `operationHost . $resourcePath + ); + return $this->client->send($request); + } +} +`; + const found = consumers(src); + expect(found).toHaveLength(1); + expect(found[0]).toMatchObject({ + framework: 'guzzle-request-ctor', + method: '*', // $method is a parameter, not a literal at this call site + path: '/payments/pay', + }); + }); + + it('resolves a fully-qualified GuzzleHttp/Psr7/Request with a literal verb', () => { + // Built via join(), not a literal backslash in this source file: a + // template-literal backslash-escape is easy to mis-transcribe (dropped + // silently by the JS/TS escape rules for an unrecognized `\`), so + // this sidesteps that entirely and is robust regardless of how the file + // itself gets written to disk. + const bs = String.fromCharCode(92); + const qualified = ['', 'GuzzleHttp', 'Psr7', 'Request'].join(bs); + const src = [ + ' { + const src = ` { + const src = `host . $resourcePath); + return $request; +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('ignores an unrelated constructor whose class name does not end in "Request"', () => { + // $resourcePath IS resolvable here (unlike an earlier version of this + // test) — the class-name filter must be the reason this produces no + // detection, not an incidental miss elsewhere in the pipeline. Without + // a resolvable path, this test would pass even with the class-name + // filter deleted. + const src = `host . $resourcePath); + return $response; +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('rejects a resolved literal that is not an HTTP-looking path', () => { + const src = `host . $resourcePath); + return $request; +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('prefers the LAST variable in a 3-part concatenation (path, not an earlier segment)', () => { + const src = ` { + // Regression: an earlier version of lastConcatVariable treated an + // unhandled `parenthesized_expression` as "no variable here" and fell + // through to the LEFT operand — silently returning $host instead of + // failing to find anything inside the parens. + const src = ` { + // Regression: resolveLocalStringLiteral stopped at the nearest + // compound_statement (the `if` block), not the enclosing function body, + // so an assignment made just above the `if` — in the same function — + // was invisible to a `new Request(...)` call nested inside it. + const src = `isValid()) { + $request = new Request($method, $this->host . $resourcePath); + return $request; + } + return null; +} +`; + const found = consumers(src); + expect(found).toHaveLength(1); + expect(found[0].path).toBe('/payments/pay'); + }); + + it('still does not cross into a sibling function even when searching level by level', () => { + // The level-by-level widening must stop at `program` / the enclosing + // function boundary — it must not walk into a DIFFERENT function's body + // just because that function is a preceding sibling statement. + const src = `host . $resourcePath); + return $request; + } +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('does not resolve through an intervening non-literal reassignment (last write wins)', () => { + // Regression: the backward scan used to skip PAST an assignment whose + // RHS wasn't a string literal, landing on an older literal that the + // variable no longer holds at the call site — a wrong answer, not a + // miss. The nearest assignment must decide the outcome, full stop. + const src = `host . $resourcePath); + return $request; +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('does not treat a non-concatenation binary expression as a path candidate', () => { + // Regression: lastConcatVariable recursed into ANY binary_expression + // without checking the operator, so `$host ?? $resourcePath` (or `&&`, + // `+`, ...) was walked exactly like `.` concatenation. + const src = `host ?? $resourcePath); + return $request; +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('matches a lowercase "request" constructor — PHP class names are case-insensitive', () => { + const src = ` { + const src = `host . $resourcePath); + return $request; +} +`; + const found = consumers(src); + expect(found).toHaveLength(1); + expect(found[0]).toMatchObject({ method: 'POST', path: '/payments/pay' }); + }); + + it('does not use a stale literal shadowed by a reassignment inside a preceding if block', () => { + // Regression: the backward scan only inspected direct expression_statement + // siblings, so `$resourcePath = '/new';` nested inside an `if` right + // before the call was invisible, and the OLDER `/old` (outside the `if`) + // was returned instead — an unconditional wrong answer whenever that + // branch runs, not a miss. + const src = `host . $resourcePath); + return $request; +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('does not resolve a variable across an anonymous-function boundary it was not use()-captured into', () => { + // Regression: level-by-level widening climbed straight from the + // closure's body to the enclosing method's body without checking PHP's + // actual capture rule (closures capture NOTHING unless listed in + // `use (...)`), resolving a variable the closure can't actually see — + // real PHP would throw "Undefined variable" here, not build this path. + const src = `host . $resourcePath); + }; + return $build(); +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('DOES resolve a variable across an anonymous-function boundary it WAS use()-captured into', () => { + const src = `host . $resourcePath); + }; + return $build(); +} +`; + const found = consumers(src); + expect(found).toHaveLength(1); + expect(found[0].path).toBe('/payments/pay'); + }); + + it('does not fall back to the host when the concatenation ends in a string literal, not a variable', () => { + // Regression: lastConcatVariable fell through to the LEFT operand when + // the right one wasn't a variable, so `$host . '/users'` resolved to + // $host instead of recognizing the trailing literal isn't a variable at + // all — if $host happened to be an HTTP URL locally, that URL would be + // emitted as the path instead of a miss. + const src = ` { + // Regression: after exhausting a method's own body, widening went + // straight to `program` (file scope) and found the top-level literal — + // but PHP methods have NO access to file-level variables without an + // explicit `global $v;`, which this resolver deliberately never adds + // support for. This produced a wrong contract, not a miss. + const src = `host . $resourcePath); + return $request; + } +} +`; + expect(consumers(src)).toHaveLength(0); + }); + + it('does not resolve a variable from file scope into a plain top-level function either', () => { + const src = ` { + const src = ` Date: Sat, 29 Aug 2026 08:38:20 +0100 Subject: [PATCH 14/28] fix(impact): report scope extraction omissions (#3071) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(impact): surface scope extraction omissions * fix(impact): preserve complete index fixtures * fix(impact): preserve scope completeness evidence * test(analyze): model successful scope extraction in harnesses --------- Co-authored-by: Gergő Magyar --- GUARDRAILS.md | 6 ++ RUNBOOK.md | 11 ++ gitnexus/src/core/index-freshness.ts | 19 +++- .../src/core/ingestion/parsing-processor.ts | 7 ++ .../ingestion/pipeline-phases/parse-impl.ts | 13 +++ .../core/ingestion/pipeline-phases/parse.ts | 4 + gitnexus/src/core/ingestion/pipeline.ts | 6 +- .../core/ingestion/scope-extractor-bridge.ts | 12 ++- .../scope-resolution/pipeline/phase.ts | 22 +++- .../scope-resolution/pipeline/run.ts | 13 ++- .../scope-extraction-failures.ts | 49 +++++++++ .../core/ingestion/workers/parse-worker.ts | 13 ++- .../core/ingestion/workers/result-merge.ts | 5 +- gitnexus/src/core/run-analyze.ts | 8 ++ gitnexus/src/mcp/local/local-backend.ts | 36 +++++++ gitnexus/src/mcp/tools.ts | 10 +- gitnexus/src/storage/parse-cache.ts | 12 +-- gitnexus/src/storage/repo-meta.ts | 18 +++- gitnexus/src/types/pipeline.ts | 4 + .../convex-impact-epistemic-e2e.test.ts | 8 +- .../impact-epistemic-lower-bound.test.ts | 102 +++++++++++++++++- .../impact-scope-omission-persistence.test.ts | 75 +++++++++++++ .../impact-undecided-satisfaction.test.ts | 1 + .../skip-optional-pipeline.test.ts | 1 + .../test/unit/incremental-parse-cache.test.ts | 28 ++++- .../index-freshness-graph-collapse.test.ts | 50 ++++++++- gitnexus/test/unit/list-status-branch.test.ts | 2 + ...mpl-warm-cache-parsedfile-coverage.test.ts | 25 ++++- .../unit/preprocess-source-parity.test.ts | 15 +++ gitnexus/test/unit/repo-manager.test.ts | 24 +++++ gitnexus/test/unit/resources.test.ts | 1 + gitnexus/test/unit/result-merge.test.ts | 31 ++++++ .../test/unit/run-analyze-fts-repair.test.ts | 4 + .../unit/scope-extraction-failures.test.ts | 54 ++++++++++ .../scope-resolution-phase-failures.test.ts | 94 ++++++++++++++++ .../scope-resolution/run-progress.test.ts | 47 ++++++++ gitnexus/vitest.config.ts | 4 + 37 files changed, 803 insertions(+), 31 deletions(-) create mode 100644 gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts create mode 100644 gitnexus/test/integration/impact-scope-omission-persistence.test.ts create mode 100644 gitnexus/test/unit/scope-extraction-failures.test.ts create mode 100644 gitnexus/test/unit/scope-resolution-phase-failures.test.ts diff --git a/GUARDRAILS.md b/GUARDRAILS.md index e157ade1e..72e9c1e59 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -52,6 +52,12 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. - **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal. +### Scope extraction is incomplete + +- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["scope-extraction-failed"]` when files were omitted, or `incompleteReasons: ["scope-extraction-unverified"]` when the index predates the completeness receipt or its metadata is unreadable. `impact`/`context` reports the same uncertainty as `epistemic: "lower-bound"`; confirmed omissions set `causes.scopeExtractionFiles > 0`. +- **Do:** Re-run `npx gitnexus analyze` (`--force` for a full graph rebuild). If the reason persists, inspect the scope-extraction warnings and treat impact counts as floors until the affected source is supported or corrected. +- **Why:** Parsing continued, but scope captures for the reported file count could not be produced even after the main-thread fallback. Calls, inheritance, imports, or accesses originating there may therefore be absent from the graph. + ### Analyze reports INCOMPLETE with a collapsed graph write - **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["graph-write-collapsed"]`; the analyze summary printed `Repository indexed INCOMPLETELY` naming an expected and a persisted relationship count, and the CLI exited non-zero. diff --git a/RUNBOOK.md b/RUNBOOK.md index 0f5c8b7bb..d16ccd52d 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -46,6 +46,17 @@ npx gitnexus status npx gitnexus list ``` +**Scope extraction incomplete:** `npx gitnexus status` reports +`incompleteReasons: ["scope-extraction-failed"]` when one or more files still +lack scope captures after the worker and fallback passes. `impact` and `context` +then report a lower bound with `causes.scopeExtractionFiles` set to the affected +file count. Re-run `npx gitnexus analyze --force`; if the reason remains, inspect +the scope-extraction warnings for the unsupported or malformed source file. +Every pre-existing index remains unverified until it is analyzed once by a +version that writes the completeness receipt. An older index or unreadable completeness record reports +`incompleteReasons: ["scope-extraction-unverified"]`; re-analyze it before treating +empty impact results as exact. + --- ## Embeddings diff --git a/gitnexus/src/core/index-freshness.ts b/gitnexus/src/core/index-freshness.ts index ea577f834..52e62e8d8 100644 --- a/gitnexus/src/core/index-freshness.ts +++ b/gitnexus/src/core/index-freshness.ts @@ -1,11 +1,14 @@ import { checkpointKind } from './embedding-checkpoint.js'; import type { RepoMeta } from '../storage/repo-manager.js'; +import { scopeExtractionFailureTotal } from './ingestion/scope-resolution/scope-extraction-failures.js'; export const INDEX_INCOMPLETE_REASONS = [ 'incremental-in-progress', 'embedding-checkpoint-pending', 'embedding-count-unverified', 'graph-write-collapsed', + 'scope-extraction-unverified', + 'scope-extraction-failed', ] as const; export type IndexIncompleteReason = (typeof INDEX_INCOMPLETE_REASONS)[number]; @@ -130,7 +133,14 @@ export function detectGraphWriteCollapse( /** Stable machine-readable reasons an index cannot be certified complete. */ export function getIndexIncompleteReasons( meta: - | Pick + | Pick< + RepoMeta, + | 'incrementalInProgress' + | 'embeddingCheckpoint' + | 'graphWriteCollapsed' + | 'scopeExtractionFailures' + | 'scopeExtractionReceipt' + > | null | undefined, ): IndexIncompleteReason[] { @@ -142,6 +152,13 @@ export function getIndexIncompleteReasons( // answers from a graph missing most of its edges, which is indistinguishable // from a codebase that genuinely has no such relationships. if (meta?.graphWriteCollapsed) reasons.push('graph-write-collapsed'); + if (meta?.scopeExtractionReceipt !== 1) { + reasons.push('scope-extraction-unverified'); + } else { + const total = scopeExtractionFailureTotal(meta.scopeExtractionFailures); + if (total === undefined) reasons.push('scope-extraction-unverified'); + else if (total > 0) reasons.push('scope-extraction-failed'); + } if (meta?.embeddingCheckpoint) { // The three checkpoint kinds are not one operator-facing state. GUARDRAILS // and the runbook document `embedding-checkpoint-pending` as "N node(s) diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index c91df2953..e4df7dae5 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -56,6 +56,8 @@ export interface WorkerExtractedData { * finalize-orchestrator. */ parsedFiles: ParsedFile[]; + /** Scope-extraction omissions represented by this worker/cache result. */ + scopeExtractionFailures: string[]; } type ParsedGraphNode = ParseWorkerResult['nodes'][number]; @@ -126,6 +128,7 @@ export const mergeChunkResults = ( const allORMQueries: ExtractedORMQuery[] = []; const fileScopeBindingsByFile: FileScopeBindings[] = []; const allParsedFiles: ParsedFile[] = []; + const scopeExtractionFailures: string[] = []; for (const result of chunkResults) { // Worker jobs and input files are already merged in stable start-index/path @@ -178,6 +181,9 @@ export const mergeChunkResults = ( if (result.fileScopeBindings) for (const item of result.fileScopeBindings) fileScopeBindingsByFile.push(item); if (result.parsedFiles) for (const item of result.parsedFiles) allParsedFiles.push(item); + for (const filePath of result.scopeExtractionFailures ?? []) { + scopeExtractionFailures.push(filePath); + } } return { @@ -195,6 +201,7 @@ export const mergeChunkResults = ( springTypes: allSpringTypes, fileScopeBindings: fileScopeBindingsByFile, parsedFiles: allParsedFiles, + scopeExtractionFailures, }; }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 0f4ba1b2e..cd4b8edf4 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -482,6 +482,9 @@ export async function runChunkedParseAndResolve( * cache analyze run can skip the dominant `extractParsedFile` cost * (otherwise ~58s on a 1000-file repo). */ parsedFiles: import('gitnexus-shared').ParsedFile[]; + scopeExtractionFailures: string[]; + /** Files excluded because their non-standalone language parser was unavailable. */ + unavailableScopeLanguageFiles: number; }> { const model = createSemanticModel(); const symbolTable = model.symbols; @@ -514,6 +517,10 @@ export async function runChunkedParseAndResolve( ); } } + const unavailableScopeLanguageFiles = [...skippedByLang.values()].reduce( + (total, count) => total + count, + 0, + ); // Sort parseableScanned alphabetically for stable chunk membership // across runs (Finding 4). Without this, filesystem-scan order can @@ -743,6 +750,7 @@ export async function runChunkedParseAndResolve( // the second-half of the parse-cache speedup since scope-resolution's // re-parse otherwise dominates the warm-cache wall-clock time. const allParsedFiles: import('gitnexus-shared').ParsedFile[] = []; + const scopeExtractionFailures = new Set(); // Incremental parse cache (Option B): chunk-level content-addressed. // When the chunk's (filePath, content-hash) signature matches a prior @@ -844,6 +852,9 @@ export async function runChunkedParseAndResolve( chunkStartMs: number | null, ): Promise => { if (chunkWorkerData) { + for (const filePath of chunkWorkerData.scopeExtractionFailures) { + scopeExtractionFailures.add(filePath); + } if (chunkWorkerData.parsedFiles?.length) { if (parsedFileStorePath) { await persistParsedFileChunk( @@ -1616,5 +1627,7 @@ export async function runChunkedParseAndResolve( // cache: when the file's ParsedFile is here, scope-resolution skips its own // `extractParsedFile` call. parsedFiles: allParsedFiles, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), + unavailableScopeLanguageFiles, }; } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 2484baa01..38bd4601b 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -80,6 +80,10 @@ export interface ParseOutput { * costing ~58s on a 1000-file repo). */ readonly parsedFiles: readonly ParsedFile[]; + /** Files whose scope extraction failed while legacy parsing continued. */ + readonly scopeExtractionFailures: readonly string[]; + /** Files omitted because their non-standalone language parser was unavailable. */ + readonly unavailableScopeLanguageFiles: number; } export const parsePhase: PipelinePhase = { diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 34ecfc111..69858ff28 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -370,14 +370,16 @@ export const runPipelineFromRepo = async ( } // Extract final results for the PipelineResult contract - const { totalFiles, usedWorkerPool } = getPhaseOutput<{ + const { totalFiles, usedWorkerPool, unavailableScopeLanguageFiles } = getPhaseOutput<{ totalFiles: number; usedWorkerPool: boolean; + unavailableScopeLanguageFiles: number; }>(results, 'parse'); let communityResult: CommunitiesOutput['communityResult'] | undefined; let processResult: ProcessesOutput['processResult'] | undefined; const scopeResolutionOutput = getPhaseOutput(results, 'scopeResolution'); + const scopeExtractionFailures = scopeResolutionOutput.scopeExtractionFailures; const resolutionOutcomes = scopeResolutionOutput.resolutionOutcomes; const undecidedSatisfaction = scopeResolutionOutput.undecidedSatisfaction; // Streamed PDG-emit manifest (#2202): present only when streaming was on. @@ -424,6 +426,8 @@ export const runPipelineFromRepo = async ( resolutionOutcomes, undecidedSatisfaction, usedWorkerPool, + scopeExtractionFailures, + unavailableScopeLanguageFiles, pdgEmitManifest, propertyInference, }; diff --git a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts index b87fa3b19..17f639941 100644 --- a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts +++ b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts @@ -64,7 +64,17 @@ export function extractParsedFile( const message = `scope extraction failed for ${filePath}: ${ err instanceof Error ? err.message : String(err) }`; - if (onWarn !== undefined) onWarn(message); + if (onWarn !== undefined) { + try { + onWarn(message); + } catch (warnErr) { + logger.warn( + `scope extraction warning callback failed for ${filePath}: ${ + warnErr instanceof Error ? warnErr.message : String(warnErr) + }`, + ); + } + } logger.warn(message); return undefined; } diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index 0ca8b2bea..8218a9656 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -50,6 +50,7 @@ import { buildPropertyNameIndex } from '../passes/unique-name-properties.js'; import { PdgEmitSink, type PdgEmitManifest } from '../../../lbug/pdg-emit-sink.js'; import { resolveNativeSafeStorageDir } from '../../../lbug/lbug-config.js'; import type { ScopeResolver } from '../contract/scope-resolver.js'; +import { reconcileScopeExtractionFailures } from '../scope-extraction-failures.js'; import { logger } from '../../../logger.js'; export interface ScopeResolutionOutput { @@ -61,6 +62,8 @@ export interface ScopeResolutionOutput { readonly importsEmitted: number; /** Reference (CALLS / ACCESSES / INHERITS / USES) edges emitted. */ readonly referenceEdgesEmitted: number; + /** Files still missing scope captures after the main-thread fallback. */ + readonly scopeExtractionFailures: readonly string[]; /** Additive stream of resolver diagnostics; does not affect graph edges. */ readonly resolutionOutcomes: readonly ResolutionOutcome[]; /** @@ -125,6 +128,7 @@ const NOOP_OUTPUT: ScopeResolutionOutput = Object.freeze({ filesProcessed: 0, importsEmitted: 0, referenceEdgesEmitted: 0, + scopeExtractionFailures: [], resolutionOutcomes: [], // Deliberately absent, not `[]`: nothing ran, so nothing was decided either. perLanguage: new Map(), @@ -174,6 +178,7 @@ export const scopeResolutionPhase: PipelinePhase = { const { scannedFiles } = getPhaseOutput(deps, 'structure'); const parseOutput = getPhaseOutput(deps, 'parse'); const { model, parsedFiles: workerParsedFiles } = parseOutput; + const scopeExtractionFailures = new Set(parseOutput.scopeExtractionFailures); // SemanticModel populated during `parse`: scope-resolution consumes // TypeRegistry / MethodRegistry / SymbolTable lookups instead of // rebuilding parallel indexes. See ARCHITECTURE.md § "Semantic-model @@ -538,6 +543,14 @@ export const scopeResolutionPhase: PipelinePhase = { provider, ); + // Worker warnings are provisional: scope-resolution retries missing + // ParsedFiles on the main thread. Persist only final omissions. + reconcileScopeExtractionFailures( + scopeExtractionFailures, + files.map((file) => file.path), + stats.scopeExtractionFailedPaths, + ); + // Release file contents and pre-extracted entries after each language // to reduce memory pressure. For large codebases (16K+ PHP files), // holding all source code simultaneously with scope trees causes OOM. @@ -651,13 +664,20 @@ export const scopeResolutionPhase: PipelinePhase = { // Even when no language ran, surface a finalized manifest (its CSVs are on // disk) so loadGraphToLbug COPYs them rather than orphaning them — empty in // the no-files case, harmless. - if (!anyRan) return pdgEmitManifest ? { ...NOOP_OUTPUT, pdgEmitManifest } : NOOP_OUTPUT; + if (!anyRan) { + return { + ...NOOP_OUTPUT, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), + ...(pdgEmitManifest ? { pdgEmitManifest } : {}), + }; + } return { ran: true, filesProcessed: totalFiles, importsEmitted: totalImports, referenceEdgesEmitted: totalRefs, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), resolutionOutcomes, undecidedSatisfaction, perLanguage, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index fcc456d66..689780d40 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -464,6 +464,8 @@ interface RunScopeResolutionInput { interface RunScopeResolutionStats { readonly filesProcessed: number; readonly filesSkipped: number; + /** Files still missing a ParsedFile after the main-thread fallback. */ + readonly scopeExtractionFailedPaths: readonly string[]; readonly importsEmitted: number; readonly resolve: ResolveStats; readonly referenceEdgesEmitted: number; @@ -564,6 +566,7 @@ export function runScopeResolution( // ── Phase 1: extract each file → ParsedFile ──────────────────────────── const parsedFiles: ParsedFile[] = []; + const scopeExtractionFailedPaths: string[] = []; let filesSkipped = 0; const treeCache = input.treeCache; const preExtracted = input.preExtractedParsedFiles; @@ -587,15 +590,20 @@ export function runScopeResolution( } if (parsed === undefined) { const cachedTree = treeCache?.get(file.path); + let extractionWarned = false; parsed = extractParsedFile( provider.languageProvider, file.content, file.path, - onWarn, + (warning) => { + extractionWarned = true; + onWarn(warning); + }, cachedTree, ); if (parsed === undefined) { filesSkipped++; + if (extractionWarned) scopeExtractionFailedPaths.push(file.path); continue; } } @@ -643,6 +651,7 @@ export function runScopeResolution( return { filesProcessed: parsedFiles.length, filesSkipped, + scopeExtractionFailedPaths, importsEmitted: 0, resolve: { sitesProcessed: 0, referencesEmitted: 0, unresolved: 0 }, referenceEdgesEmitted: 0, @@ -680,6 +689,7 @@ export function runScopeResolution( return { filesProcessed: 0, filesSkipped, + scopeExtractionFailedPaths, importsEmitted: 0, resolve: { sitesProcessed: 0, referencesEmitted: 0, unresolved: 0 }, referenceEdgesEmitted: 0, @@ -1644,6 +1654,7 @@ export function runScopeResolution( return { filesProcessed: parsedFiles.length, filesSkipped, + scopeExtractionFailedPaths, importsEmitted, resolve: resolveStats, referenceEdgesEmitted: diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts b/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts new file mode 100644 index 000000000..ce3f6de57 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts @@ -0,0 +1,49 @@ +export interface ScopeExtractionFailureSummary { + /** Exact number of unique files whose scope extraction failed. */ + readonly total: number; + /** Deterministic sample of repo-relative paths for diagnostics. */ + readonly paths: readonly string[]; + /** True when `paths` is a capped sample rather than the full set. */ + readonly truncated?: boolean; +} + +export const SCOPE_EXTRACTION_FAILURE_PATH_LIMIT = 25; + +/** Read a persisted summary without trusting its runtime JSON shape. */ +export function scopeExtractionFailureTotal(summary: unknown): number | undefined { + if (summary === undefined) return 0; + if (typeof summary !== 'object' || summary === null) return undefined; + const total = (summary as { total?: unknown }).total; + if (total === 0) return 0; + return typeof total === 'number' && Number.isInteger(total) && total > 0 ? total : undefined; +} + +/** Replace provisional worker failures with the final fallback outcome. */ +export function reconcileScopeExtractionFailures( + failures: Set, + attemptedPaths: readonly string[], + failedPaths: readonly string[], +): void { + const stillFailed = new Set(failedPaths); + for (const filePath of attemptedPaths) { + if (stillFailed.has(filePath)) failures.add(filePath); + else failures.delete(filePath); + } +} + +export function summarizeScopeExtractionFailures( + paths: readonly string[] = [], + limit: number = SCOPE_EXTRACTION_FAILURE_PATH_LIMIT, +): ScopeExtractionFailureSummary | undefined { + const unique = [ + ...new Set(paths.filter((path): path is string => typeof path === 'string' && path.length > 0)), + ].sort(); + if (unique.length === 0) return undefined; + const boundedLimit = + Number.isInteger(limit) && limit >= 0 ? limit : SCOPE_EXTRACTION_FAILURE_PATH_LIMIT; + return { + total: unique.length, + paths: unique.slice(0, boundedLimit), + ...(unique.length > boundedLimit ? { truncated: true } : {}), + }; +} diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 946d060a6..29c94e8ee 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -499,6 +499,12 @@ export interface ParseWorkerResult { * finalize-orchestrator. */ parsedFiles: ParsedFile[]; + /** + * Repo-relative paths whose scope-capture/extraction step threw. Optional for + * parse-cache compatibility; unlike transient worker telemetry this must be + * replayed on a cache hit so the persisted index cannot claim completeness. + */ + scopeExtractionFailures?: string[]; skippedLanguages: Record; /** * Files whose parse output carried a value the structured-clone algorithm @@ -1587,14 +1593,19 @@ const processFileGroup = ( // see parsedfile-store.ts). parse-impl flushes `result.parsedFiles` to disk // per chunk and does NOT retain them in main-thread heap, so this no longer // costs ~1× the semantic model in RAM during parse. + let scopeExtractionFailed = false; const parsedFile = extractParsedFile( provider, parseContent, file.path, - reportWarning, + (message) => { + scopeExtractionFailed = true; + reportWarning(message); + }, tree, scopeSourceKind, ); + if (scopeExtractionFailed) (result.scopeExtractionFailures ??= []).push(file.path); if (parsedFile !== undefined) { // Capture-time side-channel (#1983): `extractParsedFile` just ran the // provider's `emitScopeCaptures`, which (for C++ ADL/namespace marks, diff --git a/gitnexus/src/core/ingestion/workers/result-merge.ts b/gitnexus/src/core/ingestion/workers/result-merge.ts index 014c9fb8a..fc6a24ac6 100644 --- a/gitnexus/src/core/ingestion/workers/result-merge.ts +++ b/gitnexus/src/core/ingestion/workers/result-merge.ts @@ -58,11 +58,14 @@ export const mergeResult = (target: ParseWorkerResult, src: ParseWorkerResult): appendAll(target.constructorBindings, src.constructorBindings); appendAll(target.fileScopeBindings, src.fileScopeBindings); appendAll(target.parsedFiles, src.parsedFiles); + if (src.scopeExtractionFailures && src.scopeExtractionFailures.length > 0) { + appendAll((target.scopeExtractionFailures ??= []), src.scopeExtractionFailures); + } for (const [lang, count] of Object.entries(src.skippedLanguages)) { target.skippedLanguages[lang] = (target.skippedLanguages[lang] || 0) + count; } if (src.skippedPaths && src.skippedPaths.length > 0) { - (target.skippedPaths ??= []).push(...src.skippedPaths); + appendAll((target.skippedPaths ??= []), src.skippedPaths); } target.fileCount += src.fileCount; }; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 3e8739e4c..f3889d444 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -22,6 +22,7 @@ import { summarizeUnresolvedReceivers, } from './ingestion/scope-resolution/unresolved-receivers.js'; import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/undecided-satisfaction.js'; +import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js'; import type { KnowledgeGraph } from './graph/types.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { @@ -3548,6 +3549,13 @@ async function runFullAnalysisInner( // Git-only: non-git repos never take the incremental path. schemaFingerprint: hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, unresolvedReceiverMembers: summarizeUnresolvedReceivers(resolutionOutcomes), + scopeExtractionFailures: summarizeScopeExtractionFailures( + pipelineResult.scopeExtractionFailures, + ), + // A receipt certifies that every scope-capable source file was inspected. + // Optional grammars may be unavailable by design; omitting the receipt in + // that case makes readers report an unverified lower bound. + scopeExtractionReceipt: pipelineResult.unavailableScopeLanguageFiles === 0 ? 1 : undefined, // Carried forward ONLY when this run could not measure — `saveMeta` writes // a fresh object, so omitting the key deletes a prior record and turns a // hedged answer back into a confident one. A run that DID measure always diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 1679a3b80..cf197c732 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -124,6 +124,7 @@ import { } from '../../core/ingestion/scope-resolution/unresolved-receivers.js'; import type { UnresolvedReceiverSummary } from '../../core/ingestion/scope-resolution/unresolved-receivers.js'; import type { UndecidedSatisfactionSummary } from '../../core/ingestion/scope-resolution/undecided-satisfaction.js'; +import { scopeExtractionFailureTotal } from '../../core/ingestion/scope-resolution/scope-extraction-failures.js'; import { lookupCount } from '../../core/ingestion/scope-resolution/summary-maps.js'; import { DEFERRED_IMPORT_REASON_SUFFIX, @@ -654,6 +655,8 @@ export interface CodebaseContext { * number of SENTENCES, which has no relation to how much is missing. */ export interface EpistemicCauses { + /** Files whose scope-extraction output is absent from this index. */ + readonly scopeExtractionFiles: number; /** * Call SITES dropped at index time because the receiver's type could not be * established. Unit: call sites, taken from the index's @@ -717,6 +720,7 @@ function epistemicFrom(dropped: { external: number; undecided: number; dispatch: number; + scopeExtraction: number; }): { epistemic: 'exact' | 'lower-bound'; boundaries?: string[]; @@ -730,6 +734,7 @@ function epistemicFrom(dropped: { ? { epistemic: 'exact', causes: { + scopeExtractionFiles: dropped.scopeExtraction, receiverTyping: 0, dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, @@ -745,6 +750,7 @@ function epistemicFrom(dropped: { // prose saying `2 call sites` — a consumer branching on the number // would read a different magnitude than the human reading the text. causes: { + scopeExtractionFiles: dropped.scopeExtraction, receiverTyping: dropped.sites, dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, @@ -753,6 +759,29 @@ function epistemicFrom(dropped: { }; } +function scopeExtractionBoundaries( + summary: unknown, + receipt: unknown, +): { notes: string[]; files: number } { + const unknown = { + notes: [ + 'Scope-extraction completeness was not recorded for this index, so actual impact may be higher.', + ], + files: 0, + }; + if (receipt !== 1) return unknown; + const total = scopeExtractionFailureTotal(summary); + if (total === undefined) return unknown; + if (total === 0) return { notes: [], files: 0 }; + return { + notes: [ + `Scope extraction failed for ${total} ${total === 1 ? 'file' : 'files'} while this index was built. ` + + `Scope-resolution edges from ${total === 1 ? 'that file are' : 'those files are'} absent, so actual impact may be higher.`, + ], + files: total, + }; +} + /** * Boundary notes for call sites the analyzer dropped because it could not type * their receiver, when the queried symbol's name is among them (#2744). @@ -6823,6 +6852,10 @@ export class LocalBackend { meta = undefined; } const receiverDrops = unresolvedReceiverBoundaries(meta?.unresolvedReceiverMembers, symName); + const scopeExtractionDrops = scopeExtractionBoundaries( + meta?.scopeExtractionFailures, + meta?.scopeExtractionReceipt, + ); // #2873 — satisfaction checks the analyzer never completed. Read on the // same footing as the receiver drops, and BEFORE the heritage probe for the // same reason: this cause leaves no edge for that probe to find, so a @@ -6860,6 +6893,7 @@ export class LocalBackend { ...receiverDrops, notes: [ ...receiverDrops.notes, + ...scopeExtractionDrops.notes, ...undecidedDrops.notes, ...(convexDispatch === undefined ? [] : [convexDispatch.boundary]), ], @@ -6868,6 +6902,7 @@ export class LocalBackend { // count of omitted symbols. Keep the magnitude at zero rather than // inventing one from the presence of a note. dispatch: 0, + scopeExtraction: scopeExtractionDrops.files, }; try { // Discover the interface / abstract supertypes on the target's boundary. @@ -6954,6 +6989,7 @@ export class LocalBackend { epistemic: 'lower-bound', boundaries: [...droppedBoundaries.notes, ...boundaries], causes: { + scopeExtractionFiles: droppedBoundaries.scopeExtraction, receiverTyping: droppedBoundaries.sites, dispatchBoundary: droppedBoundaries.dispatch + dispatchBoundarySymbols, externalBoundary: droppedBoundaries.external, diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index e7e453a1b..be793cbeb 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -287,13 +287,14 @@ NOTE: ACCESSES edges (field read/write tracking) are included in context results COMPLETENESS OF incoming: alongside symbol/incoming/outgoing the result carries the same epistemic envelope impact() returns: - epistemic: 'exact' | 'lower-bound' — 'lower-bound' means callers exist that this view provably does not list. - boundaries: string[] — one plain-language sentence per reason. Prose for humans; branch on causes instead. -- causes: { receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: + - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — RESOLVER GAP: the analyzer dropped that many call sites on this name because it could not type the receiver, so they are missing from incoming. Do not read an absent caller as proof none exists. - causes.externalBoundary (unit: call sites) > 0 — the calls left the indexed program (System.out.println, fetch(...)). NOT a defect: no in-graph node could have been reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary static analysis cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not decide whether a type satisfies an interface, so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Usually fixable by making the missing dependency available to analysis. -REQUIRES RE-INDEX: causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. +REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. GROUP MODE: set "repo" to "@" to run context in each member repo (aggregated list), or "@/" for one member. If you use "@" only, the member defaults to the lexicographically first key in group.yaml "repos". @@ -482,14 +483,15 @@ Output includes: - byDepth: affected symbols grouped by traversal depth (paginated by limit/offset; omitted when summaryOnly:true — use byDepthCounts for totals per depth, pagination object when truncated). Each item includes a processes:[{id,label,processType,step}] field listing the execution flows that symbol participates in. Empty when the symbol has no process membership. Can ALSO be empty when partial:true is set — either the process-aggregation pass hit its cap before detecting affected processes, or per-symbol enrichment was capped on a very large page. When partial:true, do NOT treat processes:[] as proof of no participation; cross-check the top-level affected_processes list. - epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the walk provably missed callers, so the count is a floor. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). - boundaries: string[] — one plain-language sentence per reason the count is short. Prose for humans; branch on causes instead. -- causes: { receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: + - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — the RESOLVER GAP signal: the analyzer dropped that many call sites because it could not establish the receiver's type (unresolved constructor, factory, chained expression). Those callers are absent from byDepth. Treat the result as incomplete: grep the symbol name before deleting or renaming. - causes.externalBoundary (unit: call sites) > 0 — those calls left the indexed program (System.out.println, fetch(...), os.environ.*). NOT a defect and NOT a reason the count is short: there is no in-graph node any edge could have reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary a static walk cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not DECIDE whether a type satisfies an interface (a type in a required signature named a package it could not resolve), so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Distinct from every cause above, which count decided facts that could not be attributed; this one counts questions never answered. It is the only cause that shortens a result WITHOUT leaving a trace in the graph, so an unhedged zero on a symbol reached only through such an interface would otherwise read as 'nobody calls this'. Usually fixable: it most often means a dependency is missing from the analyzed tree. -REQUIRES RE-INDEX: causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. +REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. Depth groups: - d=1: WILL BREAK (direct callers/importers) diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 8cc639900..636b73486 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -626,11 +626,11 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // the re-check line below is for, and why it says AT MERGE rather than // when you pick the number. // -// 77 is free at this merge: origin/main is 76, and the open PRs touching this -// constant are #2840 (a stale 71) and #1616 (a stale 2). Scan with the contents -// API at each PR head, not `gh pr diff` — that exits non-zero on an -// inaccessible fork and prints nothing, so a grep over its output skips the PR -// silently. #2840 was missed exactly that way this round. +// 78 was claimed concurrently by #3060 while this branch was in review. Both +// branches keep the same package version, so sharing 78 would replay +// incompatible worker output without a textual merge conflict. This branch +// therefore takes 79, the next free value above origin/main and every open PR +// found by the contents-API scan at their exact head SHAs. // // WHY THIS IS STILL A HAND-PICKED NUMBER, when `SCHEMA_FINGERPRINT` next door // is a derived sha256 that cannot collide. The derivation exists and already @@ -650,7 +650,7 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // `route-extractors/` and `workers/` module content — would close the missing- // bump axis without invalidating on unrelated churn, and is the real follow-up. // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING. -const SCHEMA_BUMP = 77; +const SCHEMA_BUMP = 79; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 16bd5ede9..3cda63545 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -21,7 +21,7 @@ * `isMissingFilesystemError`) so every existing import site keeps working * unchanged. * - * Imports `node:fs`/`node:path` and two type-only shapes. Keep it that way: a + * Imports `node:fs`/`node:path` and a few type-only summary shapes. Keep it that way: a * value import here would land in every consumer of `storage/`. */ @@ -29,6 +29,7 @@ import fs from 'fs/promises'; import path from 'path'; import type { UnresolvedReceiverSummary } from '../core/ingestion/scope-resolution/unresolved-receivers.js'; import type { UndecidedSatisfactionSummary } from '../core/ingestion/scope-resolution/undecided-satisfaction.js'; +import type { ScopeExtractionFailureSummary } from '../core/ingestion/scope-resolution/scope-extraction-failures.js'; /** The `.gitnexus` directory name, relative to a repo root. */ export const GITNEXUS_DIR = '.gitnexus'; @@ -245,6 +246,21 @@ export interface RepoMeta { * this adds no runtime dependency from storage/ on core/. */ unresolvedReceiverMembers?: UnresolvedReceiverSummary; + /** + * Files omitted from scope-resolution because their provider capture or + * extraction step threw. The rest of each file may still be present in the + * graph, so this is an index-completeness signal rather than a parse failure. + * Absent means the successful run recorded no such omission; older indexes + * also read as absent until re-analyzed. + */ + scopeExtractionFailures?: ScopeExtractionFailureSummary; + /** + * Completeness receipt for scope extraction in the successful run represented + * by this metadata. A missing or different value means completeness is + * unknown (legacy, malformed, or unreadable metadata), not that zero files + * were omitted. + */ + scopeExtractionReceipt?: 1; /** * Interfaces whose structural-satisfaction check this run could not COMPLETE * (#2873) — not interfaces found to have no implementors. diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index 015696ed6..950cd3f31 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -40,6 +40,10 @@ export interface PipelineResult { * affordance so regression suites can prove the pool engaged. */ usedWorkerPool: boolean; + /** Files omitted from scope-resolution while the rest of analysis continued. */ + scopeExtractionFailures: readonly string[]; + /** Files scope resolution could not inspect because their parser was unavailable. */ + unavailableScopeLanguageFiles: number; /** * Streamed PDG-emit COPY manifest (#2202). Present only when streaming/chunked * PDG emit was active (full rebuild + `--pdg` + enabled): the BasicBlock node diff --git a/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts b/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts index 74e1f1073..ae81e9570 100644 --- a/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts +++ b/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts @@ -14,7 +14,7 @@ import { pruneAndSaveDurableParsedFileStore, } from '../../src/storage/parsedfile-store.js'; import { LocalBackend } from '../../src/mcp/local/local-backend.js'; -import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { listRegisteredRepos, saveMeta, type RepoMeta } from '../../src/storage/repo-manager.js'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({ @@ -196,6 +196,12 @@ export const javascriptQuery = query({ handler: async () => null }); const adapter = await import('../../src/core/lbug/lbug-adapter.js'); await adapter.loadGraphToLbug(replay.graph, repoDir, storageDir); + await saveMeta(storageDir, { + repoPath: repoDir, + lastCommit: 'convex-e2e', + indexedAt: new Date(0).toISOString(), + scopeExtractionReceipt: 1, + } satisfies RepoMeta); }, poolAdapter: true, afterSetup: async (handle) => { diff --git a/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts b/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts index 6ca1d6dde..2ef3e6c78 100644 --- a/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts +++ b/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts @@ -16,15 +16,16 @@ * container, so impact("EmailLogger", upstream) finds no direct caller — but * must flag that the true blast radius is higher. */ -import { it, expect, beforeAll, vi } from 'vitest'; +import { it, expect, beforeAll, beforeEach, vi } from 'vitest'; import { LocalBackend } from '../../src/mcp/local/local-backend.js'; -import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { listRegisteredRepos, loadMeta } from '../../src/storage/repo-manager.js'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; vi.mock('../../src/storage/repo-manager.js', () => ({ listRegisteredRepos: vi.fn().mockResolvedValue([]), cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), findSiblingClones: vi.fn().mockResolvedValue([]), + loadMeta: vi.fn().mockResolvedValue({ scopeExtractionReceipt: 1 }), })); const SEED = [ @@ -66,6 +67,11 @@ withTestLbugDB( beforeAll(() => { backend = (handle as any)._backend; }); + beforeEach(() => { + vi.mocked(loadMeta).mockResolvedValue({ + scopeExtractionReceipt: 1, + } as Awaited>); + }); it('flags impact() on a concrete impl behind an interface as lower-bound', async () => { const result = await backend.callTool('impact', { @@ -116,6 +122,78 @@ withTestLbugDB( expect(result.impactedCount).toBeGreaterThanOrEqual(1); }); + it('marks impact as a lower bound when scope extraction omitted files', async () => { + vi.mocked(loadMeta).mockResolvedValueOnce({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 2, + paths: ['src/broken-a.ts', 'src/broken-b.ts'], + }, + } as Awaited>); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain('Scope extraction failed for 2 files'); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 2 }); + }); + + it('never renders repository-controlled failure paths in boundary prose', async () => { + vi.mocked(loadMeta).mockResolvedValueOnce({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 1, + paths: ['src/`break`\n\u001b[31m\u202e\u200binject.ts'], + }, + } as Awaited>); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + const prose = result.boundaries.join(' '); + + expect(prose).toContain('Scope extraction failed for 1 file'); + expect(prose).not.toMatch(/[\n\u001b\u202e\u200b`]/u); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 1 }); + }); + + it.each([ + ['missing receipt', {}], + ['missing metadata', null], + ['malformed summary', { scopeExtractionReceipt: 1, scopeExtractionFailures: 'invalid' }], + ])('treats %s as an unknown lower bound', async (_label, metadata) => { + vi.mocked(loadMeta).mockResolvedValueOnce(metadata as Awaited>); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain( + 'Scope-extraction completeness was not recorded', + ); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 0 }); + }); + + it('treats a metadata read failure as an unknown lower bound', async () => { + vi.mocked(loadMeta).mockRejectedValueOnce(new Error('metadata unavailable')); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain( + 'Scope-extraction completeness was not recorded', + ); + }); + it.each([ ['listOrders', 'query'], ['createOrder', 'mutation'], @@ -159,6 +237,26 @@ withTestLbugDB( expect(result.epistemic).toBe('lower-bound'); }); + it('context() reports persisted scope extraction omissions as a lower bound', async () => { + vi.mocked(loadMeta).mockResolvedValueOnce({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 2, + paths: ['src/broken-a.ts', 'src/broken-b.ts'], + }, + } as Awaited>); + + const result = await backend.callTool('context', { + name: 'formatDate', + file_path: 'src/util.ts', + }); + + expect(result.status).toBe('found'); + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain('Scope extraction failed for 2 files'); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 2 }); + }); + it('context() on a leaf interface itself is lower-bound (#1858 review F3)', async () => { // Logger is a leaf interface — it implements/extends nothing, so the only // boundary signal is computeEpistemicBoundary's symType==='Interface' diff --git a/gitnexus/test/integration/impact-scope-omission-persistence.test.ts b/gitnexus/test/integration/impact-scope-omission-persistence.test.ts new file mode 100644 index 000000000..83661475b --- /dev/null +++ b/gitnexus/test/integration/impact-scope-omission-persistence.test.ts @@ -0,0 +1,75 @@ +import { beforeEach, expect, it, vi } from 'vitest'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { listRegisteredRepos, saveMeta } from '../../src/storage/repo-manager.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; + +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, listRegisteredRepos: vi.fn() }; +}); + +const SEED = [ + `CREATE (:Function {id: 'Function:src/util.ts:formatDate', name: 'formatDate', filePath: 'src/util.ts', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, +]; + +withTestLbugDB( + 'impact-scope-omission-persistence', + (handle) => { + beforeEach(() => { + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date(0).toISOString(), + lastCommit: 'abc123', + stats: { files: 1, nodes: 1, communities: 0, processes: 0 }, + }, + ]); + }); + + it('persists omissions for reopened readers and clears them after a clean run', async () => { + const baseMeta = { + repoPath: '/test/repo', + lastCommit: 'abc123', + indexedAt: new Date(0).toISOString(), + scopeExtractionReceipt: 1 as const, + }; + await saveMeta(handle.tmpHandle.dbPath, { + ...baseMeta, + scopeExtractionFailures: { total: 1, paths: ['src/broken.ts'] }, + }); + + const incompleteBackend = new LocalBackend(); + await incompleteBackend.init(); + const impact = await incompleteBackend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + const context = await incompleteBackend.callTool('context', { + name: 'formatDate', + file_path: 'src/util.ts', + }); + expect(impact).toMatchObject({ + epistemic: 'lower-bound', + causes: { scopeExtractionFiles: 1 }, + }); + expect(context).toMatchObject({ + status: 'found', + epistemic: 'lower-bound', + causes: { scopeExtractionFiles: 1 }, + }); + + await saveMeta(handle.tmpHandle.dbPath, baseMeta); + const cleanBackend = new LocalBackend(); + await cleanBackend.init(); + const cleanImpact = await cleanBackend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + expect(cleanImpact).toMatchObject({ epistemic: 'exact' }); + expect(cleanImpact).not.toHaveProperty('boundaries'); + }); + }, + { seed: SEED, poolAdapter: true }, +); diff --git a/gitnexus/test/integration/impact-undecided-satisfaction.test.ts b/gitnexus/test/integration/impact-undecided-satisfaction.test.ts index ce79b4a01..f40a407c1 100644 --- a/gitnexus/test/integration/impact-undecided-satisfaction.test.ts +++ b/gitnexus/test/integration/impact-undecided-satisfaction.test.ts @@ -108,6 +108,7 @@ withTestLbugDB( // uses, and it is atomic and dual-writes the legacy mirror. Writing the // file directly would pin a shape no real analyze can produce. await saveMeta(path.dirname(h.dbPath), { + scopeExtractionReceipt: 1, undecidedInterfaceSatisfaction: { counts: { CtxStore: 2 }, totalInterfaces: 1, diff --git a/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts b/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts index d60fca427..26663fd8d 100644 --- a/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts +++ b/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts @@ -79,6 +79,7 @@ describe('optional-grammar pipeline exclusion (#2091/#2093)', () => { it('skips the Swift file at the parse phase (non-vacuity: Swift was present)', () => { expect(messages.some((m) => /Skipping 1 swift file\(s\)/.test(m))).toBe(true); + expect(result.unavailableScopeLanguageFiles).toBe(1); }); it('routes the opt-out message, not the missing-binding "npm rebuild" hint', () => { diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index d6500c702..b476f9743 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -240,14 +240,16 @@ describe('PARSE_CACHE_VERSION', () => { // collided, because each re-checked once and neither re-checked after the // other moved — which is why the rule is re-applied AT MERGE, not when the // number is picked. - it('pins SCHEMA_BUMP to 77 so concurrent bumps cannot silently collide (#2766)', () => { - expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(77); + it('pins SCHEMA_BUMP to 79 so concurrent bumps cannot silently collide (#2766, #3015)', () => { + expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(79); // The PREVIOUS version must fail the reuse gate, not merely differ from the // current one — a hardcoded number outside the conflict hunk rebases cleanly // while being wrong, which is exactly how the 37/38 exact clashes landed. // Every nearby historical or in-flight value is rejected, including 69, // which carried the route-table payload before this merge. - for (const taken of [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76]) { + for (const taken of [ + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + ]) { expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).not.toBe(taken); } }); @@ -634,12 +636,14 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { referenceSites: [], }, ], + scopeExtractionFailures: ['a.c'], }); const slim = slimParseWorkerResultsForCache([raw])[0]; expect(slim.calls).toEqual([]); expect(slim.assignments).toEqual([]); expect(slim.constructorBindings).toEqual([]); expect(slim.parsedFiles).toEqual([]); + expect(slim.scopeExtractionFailures).toEqual(['a.c']); expect(slim.fileCount).toBe(raw.fileCount); }); @@ -660,6 +664,24 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { expect(slim.nodes).toHaveLength(1); }); + it('round-trips scope extraction failures through a persisted cache shard', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const key = 'e'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[key, [minimalResult({ scopeExtractionFailures: ['src/broken.ts'] })]]]), + usedKeys: new Set([key]), + }); + + const loaded = await loadParseCache(dir); + const replayed = await loadParseCacheChunk(loaded, key); + expect(replayed?.[0]?.scopeExtractionFailures).toEqual(['src/broken.ts']); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('persistParseCacheChunk writes to disk without retaining in-memory entries', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { diff --git a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts index f92a8e375..d2c32a280 100644 --- a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts +++ b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts @@ -208,27 +208,69 @@ describe('graph-write-collapsed incomplete reason (B2)', () => { it('reports a collapsed write as incomplete rather than fresh', () => { expect( - getIndexIncompleteReasons({ graphWriteCollapsed: { expected: 23009, persisted: 2170 } }), + getIndexIncompleteReasons({ + graphWriteCollapsed: { expected: 23009, persisted: 2170 }, + scopeExtractionReceipt: 1, + }), ).toEqual(['graph-write-collapsed']); }); it('treats a missing relation table (zero persisted) the same way', () => { expect( - getIndexIncompleteReasons({ graphWriteCollapsed: { expected: 23009, persisted: 0 } }), + getIndexIncompleteReasons({ + graphWriteCollapsed: { expected: 23009, persisted: 0 }, + scopeExtractionReceipt: 1, + }), ).toEqual(['graph-write-collapsed']); }); it('says nothing on a healthy run', () => { - expect(getIndexIncompleteReasons({})).toEqual([]); - expect(getIndexIncompleteReasons(null)).toEqual([]); + expect(getIndexIncompleteReasons({ scopeExtractionReceipt: 1 })).toEqual([]); + }); + + it('marks missing metadata or receipt as scope-extraction-unverified', () => { + expect(INDEX_INCOMPLETE_REASONS).toContain('scope-extraction-unverified'); + expect(getIndexIncompleteReasons({})).toEqual(['scope-extraction-unverified']); + expect(getIndexIncompleteReasons(null)).toEqual(['scope-extraction-unverified']); }); it('reports alongside other reasons rather than masking them', () => { const reasons = getIndexIncompleteReasons({ incrementalInProgress: { startedAt: 1, toWriteCount: 0 }, graphWriteCollapsed: { expected: 500, persisted: 10 }, + scopeExtractionReceipt: 1, }); expect(reasons).toContain('incremental-in-progress'); expect(reasons).toContain('graph-write-collapsed'); }); }); + +describe('scope-extraction-failed incomplete reason (#3015)', () => { + it('is stable and reports a partial scope index as incomplete', () => { + expect(INDEX_INCOMPLETE_REASONS).toContain('scope-extraction-failed'); + expect( + getIndexIncompleteReasons({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { total: 2, paths: ['src/a.ts', 'src/b.ts'] }, + }), + ).toContain('scope-extraction-failed'); + }); + + it('does not report a malformed zero-count record as incomplete', () => { + expect( + getIndexIncompleteReasons({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { total: 0, paths: [] }, + }), + ).toEqual([]); + }); + + it('marks malformed summaries as unverified even when the receipt is present', () => { + expect( + getIndexIncompleteReasons({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { total: Number.NaN, paths: [] }, + }), + ).toEqual(['scope-extraction-unverified']); + }); +}); diff --git a/gitnexus/test/unit/list-status-branch.test.ts b/gitnexus/test/unit/list-status-branch.test.ts index 2e528cc8e..f472397f8 100644 --- a/gitnexus/test/unit/list-status-branch.test.ts +++ b/gitnexus/test/unit/list-status-branch.test.ts @@ -137,6 +137,7 @@ describe('status branch rendering (#2106)', () => { indexedAt: '2026-06-10T12:00:00.000Z', branch: 'main', runnerIdentity, + scopeExtractionReceipt: 1 as const, }, }; @@ -287,6 +288,7 @@ describe('status branch rendering (#2106)', () => { indexedAt: '2026-06-10T14:00:00.000Z', branch: 'feature/z', runnerIdentity, + scopeExtractionReceipt: 1, }); await statusCommand(); diff --git a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts index 36e9140ce..77ac0a48c 100644 --- a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts +++ b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts @@ -194,6 +194,7 @@ const reset = () => ({ routes: [], fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], routerIncludes: [], routerImports: [], toolDefs: [], ormQueries: [], constructorBindings: [], fileScopeBindings: [], parsedFiles: [], skippedLanguages: {}, fileCount: 0, + scopeExtractionFailures: [], }); let accumulated = reset(); parentPort.on('message', (msg) => { @@ -209,6 +210,7 @@ parentPort.on('message', (msg) => { accumulated.parsedFiles.push({ filePath, moduleScope: '', scopes: [], parsedImports: [], localDefs: [], referenceSites: [], }); + if (filePath.includes('broken')) accumulated.scopeExtractionFailures.push(filePath); accumulated.fileCount++; } parentPort.postMessage({ type: 'progress', filesProcessed: accumulated.fileCount }); @@ -293,9 +295,9 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { cache: ReturnType, files: { path: string; size: number }[], chunkByteBudget?: number, - ): Promise => { + ): Promise>> => { const rels = files.map((f) => f.path); - await runChunkedParseAndResolve( + return runChunkedParseAndResolve( createKnowledgeGraph(), files, rels, @@ -335,7 +337,7 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { const f = writeFile('src/degrade.ts', 'export function degrade() { return 1; }\n'); prepareOverride.impl = () => Promise.reject(new Error('EACCES: simulated cache failure')); try { - await expect(run(newCache(), [f])).resolves.toBeUndefined(); + await expect(run(newCache(), [f])).resolves.toBeDefined(); } finally { prepareOverride.impl = undefined; } @@ -379,6 +381,23 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { expect(fs.existsSync(markerPath)).toBe(false); // NO worker spawned on the warm hit }); + it('replays scope-extraction failures from a warm parse-cache hit', async () => { + const f = writeFile('src/broken.ts', 'export function broken() { return 1; }\n'); + const cache = newCache(); + + const cold = await run(cache, [f]); + expect(cold.scopeExtractionFailures).toEqual([f.path]); + await persistCaches(cache); + + const { loadParseCache } = await import('../../src/storage/parse-cache.js'); + const warm = await loadParseCache(storageDir); + fs.rmSync(markerPath, { force: true }); + + const replayed = await run(warm as ReturnType, [f]); + expect(fs.existsSync(markerPath)).toBe(false); + expect(replayed.scopeExtractionFailures).toEqual([f.path]); + }); + it('coherence gate: a parse-cache hit with NO durable shards re-dispatches the worker', async () => { const f = writeFile('src/cached.ts', 'export function cached() { return 1; }\n'); const cache = newCache(); diff --git a/gitnexus/test/unit/preprocess-source-parity.test.ts b/gitnexus/test/unit/preprocess-source-parity.test.ts index 6c3d04e6e..643d6cacd 100644 --- a/gitnexus/test/unit/preprocess-source-parity.test.ts +++ b/gitnexus/test/unit/preprocess-source-parity.test.ts @@ -4,6 +4,7 @@ import { providers, getProvider } from '../../src/core/ingestion/languages/index import { extractParsedFile } from '../../src/core/ingestion/scope-extractor-bridge.js'; import { isLanguageAvailable } from '../../src/core/tree-sitter/parser-loader.js'; import { ensureAndParse } from '../../src/core/embeddings/ast-utils.js'; +import type { LanguageProvider } from '../../src/core/ingestion/language-provider.js'; /** * Every provider that defines `preprocessSource` must produce the same @@ -55,6 +56,20 @@ const languagesWithHook = Object.entries(providers) .sort(); describe('LanguageProvider.preprocessSource parity', () => { + it('does not propagate an exception thrown by the warning callback', () => { + const provider = { + emitScopeCaptures: () => { + throw new Error('provider failed'); + }, + } as unknown as LanguageProvider; + + expect(() => + extractParsedFile(provider, 'const value = 1;', 'broken.ts', () => { + throw new Error('warning transport closed'); + }), + ).not.toThrow(); + }); + it('has a fixture for every provider defining the hook', () => { expect(Object.keys(FIXTURES).sort()).toEqual(languagesWithHook); }); diff --git a/gitnexus/test/unit/repo-manager.test.ts b/gitnexus/test/unit/repo-manager.test.ts index 1c6c20e5f..2080e2f45 100644 --- a/gitnexus/test/unit/repo-manager.test.ts +++ b/gitnexus/test/unit/repo-manager.test.ts @@ -230,6 +230,30 @@ describe('saveMeta dual-write', () => { expect(JSON.parse(legacy)).toEqual(meta); }); + it('round-trips scope extraction failure metadata through the production writer', async () => { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const withFailures: RepoMeta = { + ...meta, + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 3, + paths: ['src/a.ts', 'src/b.ts'], + truncated: true, + }, + }; + + await saveMeta(storagePath, withFailures); + + expect(await loadMeta(storagePath)).toMatchObject({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 3, + paths: ['src/a.ts', 'src/b.ts'], + truncated: true, + }, + }); + }); + it('leaves no stray tmp files behind after a successful write', async () => { const { storagePath } = getStoragePaths(tmpRepo.dbPath); await saveMeta(storagePath, meta); diff --git a/gitnexus/test/unit/resources.test.ts b/gitnexus/test/unit/resources.test.ts index aef0eac27..ff3894be1 100644 --- a/gitnexus/test/unit/resources.test.ts +++ b/gitnexus/test/unit/resources.test.ts @@ -539,6 +539,7 @@ describe('context resource freshness after out-of-process analyze (#2438)', () = lastCommit: 'current-head', indexedAt: '2026-07-18T12:00:00.000Z', incrementalInProgress: { startedAt: 1, toWriteCount: 2 }, + scopeExtractionReceipt: 1, embeddingCheckpoint: { at: '2026-07-18T12:00:00.000Z', nodesProcessed: 1, diff --git a/gitnexus/test/unit/result-merge.test.ts b/gitnexus/test/unit/result-merge.test.ts index dff56df3e..19d9b8b2e 100644 --- a/gitnexus/test/unit/result-merge.test.ts +++ b/gitnexus/test/unit/result-merge.test.ts @@ -58,6 +58,37 @@ describe('mergeResult', () => { expect(target.skippedPaths).toBeUndefined(); }); + it('unions scope-extraction failures across worker sub-batches', () => { + const target = emptyResult(); + mergeResult(target, { + ...emptyResult(), + scopeExtractionFailures: ['src/a.ts'], + }); + mergeResult(target, { + ...emptyResult(), + scopeExtractionFailures: ['src/b.ts'], + }); + expect(target.scopeExtractionFailures).toEqual(['src/a.ts', 'src/b.ts']); + }); + + it('leaves scope-extraction failures absent for backward-compatible results', () => { + const target = emptyResult(); + mergeResult(target, emptyResult()); + expect(target.scopeExtractionFailures).toBeUndefined(); + }); + + it('merges failure sets larger than the JavaScript argument limit', () => { + const target = emptyResult(); + const scopeExtractionFailures = Array.from( + { length: 70_000 }, + (_, index) => `src/failure-${index}.ts`, + ); + + expect(() => mergeResult(target, { ...emptyResult(), scopeExtractionFailures })).not.toThrow(); + expect(target.scopeExtractionFailures).toHaveLength(70_000); + expect(target.scopeExtractionFailures?.at(-1)).toBe('src/failure-69999.ts'); + }); + it('unions springTypes across sub-batch results, initializing the target when absent (#2288)', () => { const mkType = (name: string, filePath: string) => ({ filePath, diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index ea3914291..24d03844f 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -1516,6 +1516,8 @@ describe('runFullAnalysis Phase 5 embedding gate (#2790)', () => { runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, totalFileCount: 1, + scopeExtractionFailures: [], + unavailableScopeLanguageFiles: 0, graph: { forEachNode: (fn: (node: typeof stubNode) => void) => fn(stubNode), getNode: (id: string) => (id === GATE_NODE_ID ? stubNode : undefined), @@ -2122,6 +2124,8 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, totalFileCount: 1, + scopeExtractionFailures: [], + unavailableScopeLanguageFiles: 0, graph: { forEachNode: (fn: (node: typeof stubNode) => void) => fn(stubNode), getNode: (id: string) => (id === RESILIENCE_NODE_ID ? stubNode : undefined), diff --git a/gitnexus/test/unit/scope-extraction-failures.test.ts b/gitnexus/test/unit/scope-extraction-failures.test.ts new file mode 100644 index 000000000..54d589927 --- /dev/null +++ b/gitnexus/test/unit/scope-extraction-failures.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { + reconcileScopeExtractionFailures, + scopeExtractionFailureTotal, + summarizeScopeExtractionFailures, +} from '../../src/core/ingestion/scope-resolution/scope-extraction-failures.js'; + +describe('summarizeScopeExtractionFailures', () => { + it('deduplicates, sorts, and caps paths while retaining the exact total', () => { + expect(summarizeScopeExtractionFailures(['z.ts', 'a.ts', 'z.ts', 'b.ts'], 2)).toEqual({ + total: 3, + paths: ['a.ts', 'b.ts'], + truncated: true, + }); + }); + + it('returns undefined when no failure was recorded', () => { + expect(summarizeScopeExtractionFailures([])).toBeUndefined(); + expect(summarizeScopeExtractionFailures()).toBeUndefined(); + }); + + it('ignores malformed paths restored from a corrupt cache payload', () => { + expect( + summarizeScopeExtractionFailures(['valid.ts', null, undefined, 42] as unknown as string[]), + ).toEqual({ total: 1, paths: ['valid.ts'] }); + }); + + it('clears worker failures recovered by fallback and retains final omissions', () => { + const failures = new Set(['recovered.ts', 'still-broken.ts', 'untouched.ts']); + + reconcileScopeExtractionFailures( + failures, + ['recovered.ts', 'still-broken.ts', 'new-failure.ts'], + ['still-broken.ts', 'new-failure.ts'], + ); + + expect([...failures].sort()).toEqual(['new-failure.ts', 'still-broken.ts', 'untouched.ts']); + }); +}); + +describe('scopeExtractionFailureTotal', () => { + it.each([ + ['absent summary', undefined, 0], + ['clean summary', { total: 0, paths: [] }, 0], + ['failure summary', { total: 2, paths: ['a.ts', 'b.ts'] }, 2], + ['non-object', 'invalid', undefined], + ['null', null, undefined], + ['fractional count', { total: 1.5 }, undefined], + ['negative count', { total: -1 }, undefined], + ['missing count', {}, undefined], + ])('reads %s consistently', (_name, summary, expected) => { + expect(scopeExtractionFailureTotal(summary)).toBe(expected); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution-phase-failures.test.ts b/gitnexus/test/unit/scope-resolution-phase-failures.test.ts new file mode 100644 index 000000000..ca1816d58 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution-phase-failures.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const runScopeResolutionMock = vi.hoisted(() => vi.fn()); +vi.mock('../../src/core/ingestion/scope-resolution/pipeline/run.js', async (importOriginal) => { + const actual = + await importOriginal< + typeof import('../../src/core/ingestion/scope-resolution/pipeline/run.js') + >(); + return { ...actual, runScopeResolution: runScopeResolutionMock }; +}); + +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { createSemanticModel } from '../../src/core/ingestion/model/index.js'; +import { scopeResolutionPhase } from '../../src/core/ingestion/scope-resolution/pipeline/phase.js'; +import type { ParseOutput } from '../../src/core/ingestion/pipeline-phases/parse.js'; +import type { StructureOutput } from '../../src/core/ingestion/pipeline-phases/structure.js'; +import type { + PhaseResult, + PipelineContext, +} from '../../src/core/ingestion/pipeline-phases/types.js'; + +const phaseResult = (phaseName: string, output: T): PhaseResult => ({ + phaseName, + output, + durationMs: 0, +}); + +describe('scopeResolutionPhase failure reconciliation', () => { + let repoDir = ''; + + afterEach(() => { + runScopeResolutionMock.mockReset(); + if (repoDir) fs.rmSync(repoDir, { recursive: true, force: true }); + }); + + it('retains a parse failure when the main-thread provider fallback also fails', async () => { + repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scope-phase-failure-')); + fs.writeFileSync(path.join(repoDir, 'broken.py'), 'def broken(:\n'); + + runScopeResolutionMock.mockReturnValue({ + filesProcessed: 0, + filesSkipped: 1, + scopeExtractionFailedPaths: ['broken.py'], + importsEmitted: 0, + resolve: { unresolved: 0 }, + referenceEdgesEmitted: 0, + referenceSkipped: 0, + propertyDispatchSkippedKeys: 0, + importedValueRefEdges: 0, + uniqueNamePropertyEdges: 0, + uniqueNamePropertyAmbiguous: 0, + uniqueNamePropertyNarrowed: 0, + uniqueNamePropertyAmbiguousNames: [], + uniqueNamePropertyCrossLanguage: 0, + uniqueNamePropertyCrossLanguageNames: [], + resolutionOutcomes: [], + undecidedSatisfaction: [], + functionSummaries: [], + callSummaries: [], + }); + + const graph = createKnowledgeGraph(); + const ctx: PipelineContext = { + repoPath: repoDir, + graph, + onProgress: () => {}, + pipelineStart: Date.now(), + }; + const structure: StructureOutput = { + scannedFiles: [{ path: 'broken.py', size: 13 }], + allPaths: ['broken.py'], + allPathSet: new Set(['broken.py']), + totalFiles: 1, + }; + const parse = { + model: createSemanticModel(), + parsedFiles: [], + scopeExtractionFailures: ['broken.py'], + } as unknown as ParseOutput; + const deps = new Map>([ + ['structure', phaseResult('structure', structure)], + ['parse', phaseResult('parse', parse)], + ['crossFile', phaseResult('crossFile', {})], + ]); + + const output = await scopeResolutionPhase.execute(ctx, deps); + + expect(runScopeResolutionMock).toHaveBeenCalledOnce(); + expect(output.scopeExtractionFailures).toEqual(['broken.py']); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/run-progress.test.ts b/gitnexus/test/unit/scope-resolution/run-progress.test.ts index 45bbfe9d7..6894f96d8 100644 --- a/gitnexus/test/unit/scope-resolution/run-progress.test.ts +++ b/gitnexus/test/unit/scope-resolution/run-progress.test.ts @@ -7,6 +7,7 @@ import { import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { createSemanticModel } from '../../../src/core/ingestion/model/semantic-model.js'; import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; +import type { LanguageProvider } from '../../../src/core/ingestion/language-provider.js'; const mkScope = (id: ScopeId, filePath: string): Scope => ({ id, @@ -40,6 +41,12 @@ const stubProvider = { propagatesReturnTypesAcrossImports: false, } as unknown as ScopeResolver; +const providerWithEmitter = (emitScopeCaptures: LanguageProvider['emitScopeCaptures']) => + ({ + ...stubProvider, + languageProvider: { emitScopeCaptures } as LanguageProvider, + }) as ScopeResolver; + describe('runScopeResolution onProgress', () => { it('emits sub-phases in order for a 3-file input', () => { const files = [ @@ -104,4 +111,44 @@ describe('runScopeResolution onProgress', () => { expect(stats.filesProcessed).toBe(0); expect(calls).toEqual([{ subPhase: 'extracting', current: 0, total: 0 }]); }); + + it('counts empty migrated files as skipped without recording extraction failures', () => { + const stats = runScopeResolution( + { + graph: createKnowledgeGraph(), + model: createSemanticModel(), + files: [ + { path: 'empty.py', content: '' }, + { path: 'whitespace.py', content: ' \n\t' }, + ], + }, + providerWithEmitter(() => { + throw new Error('empty files must short-circuit before capture'); + }), + ); + + expect(stats.filesProcessed).toBe(0); + expect(stats.filesSkipped).toBe(2); + expect(stats.scopeExtractionFailedPaths).toEqual([]); + }); + + it('records a warned emitter failure as a final extraction omission', () => { + const warnings: string[] = []; + const stats = runScopeResolution( + { + graph: createKnowledgeGraph(), + model: createSemanticModel(), + files: [{ path: 'broken.py', content: 'value = 1' }], + onWarn: (warning) => warnings.push(warning), + }, + providerWithEmitter(() => { + throw new Error('capture failed'); + }), + ); + + expect(stats.filesProcessed).toBe(0); + expect(stats.filesSkipped).toBe(1); + expect(stats.scopeExtractionFailedPaths).toEqual(['broken.py']); + expect(warnings).toEqual([expect.stringContaining('capture failed')]); + }); }); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 390757b22..9d996279a 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -64,6 +64,8 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + 'test/integration/impact-epistemic-lower-bound.test.ts', + 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', @@ -136,6 +138,8 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + 'test/integration/impact-epistemic-lower-bound.test.ts', + 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', From f64cc8b7a86e48c1d1027d176677e88a510d0c22 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Sat, 29 Aug 2026 08:39:09 +0100 Subject: [PATCH 15/28] feat(group): add GraphQL cross-repo contracts (#3070) * feat(group): add GraphQL contract extraction * fix(group): tighten GraphQL contract guards * fix(group): complete GraphQL review hardening * fix(group): isolate bounded GraphQL reads * fix(group): harden GraphQL contract extraction --- gitnexus/README.md | 15 + gitnexus/package-lock.json | 10 + gitnexus/package.json | 1 + gitnexus/scripts/cross-platform-tests.ts | 2 + gitnexus/src/core/group/PIPELINE.md | 22 +- gitnexus/src/core/group/config-parser.ts | 28 +- .../src/core/group/extractors/fs-utils.ts | 91 +++ .../group/extractors/graphql-extractor.ts | 707 ++++++++++++++++++ .../group/extractors/manifest-extractor.ts | 11 +- gitnexus/src/core/group/matching.ts | 8 +- gitnexus/src/core/group/storage.ts | 1 + gitnexus/src/core/group/sync.ts | 13 + gitnexus/src/core/group/types.ts | 19 +- .../src/core/ingestion/utils/symbol-labels.ts | 4 +- .../group/graphql-resolve-symbol.test.ts | 145 ++++ .../test/unit/group/config-parser.test.ts | 45 ++ gitnexus/test/unit/group/fs-utils.test.ts | 89 +++ .../test/unit/group/graphql-extractor.test.ts | 502 +++++++++++++ .../unit/group/manifest-label-drift.test.ts | 18 + gitnexus/test/unit/group/matching.test.ts | 18 + gitnexus/test/unit/group/sync.test.ts | 28 + gitnexus/test/unit/group/types.test.ts | 4 +- gitnexus/vitest.config.ts | 2 + 23 files changed, 1764 insertions(+), 19 deletions(-) create mode 100644 gitnexus/src/core/group/extractors/graphql-extractor.ts create mode 100644 gitnexus/test/integration/group/graphql-resolve-symbol.test.ts create mode 100644 gitnexus/test/unit/group/fs-utils.test.ts create mode 100644 gitnexus/test/unit/group/graphql-extractor.test.ts diff --git a/gitnexus/README.md b/gitnexus/README.md index e9b16ac73..e524eaa19 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -281,6 +281,21 @@ gitnexus group status # Check staleness of repos in a group gitnexus group impact --target --repo # Cross-repo blast radius ``` +GraphQL contract matching is opt-in in the group's `group.yaml`: + +```yaml +detect: + graphql: true +``` + +The initial exact-only slice matches methods and properties on top-level NestJS `@Resolver` +classes using imported `@Query`, `@Mutation`, and `@Subscription` decorators. Named +`.graphql`/`.gql` operations are anchored by generated `Document` declarations; +object, static `gql` template, and `TypedDocumentString` initializers must prove the operation name +and root fields. Dynamic decorator names, anonymous operations, and ambiguous or missing graph +anchors are deliberately omitted. Add common infrastructure fields such as `/health` to +`matching.exclude_links_paths` to keep those GraphQL contracts visible without cross-linking them. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. ## Remote Embeddings diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 08aa3a863..7188c7e1c 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -23,6 +23,7 @@ "graphology": "^0.26.0", "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", + "graphql": "^16.14.2", "ignore": "^7.0.5", "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", @@ -3338,6 +3339,15 @@ "graphology-types": ">=0.23.0" } }, + "node_modules/graphql": { + "version": "16.14.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", + "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, "node_modules/guid-typescript": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 87d83a090..5fccc65d1 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -69,6 +69,7 @@ "graphology": "^0.26.0", "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", + "graphql": "^16.14.2", "ignore": "^7.0.5", "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index f5feeeed5..d38672724 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -90,6 +90,7 @@ const PLATFORM_LOGIC = [ 'test/unit/ignore-service.test.ts', 'test/unit/group/bridge-db.test.ts', 'test/unit/group/bridge-db-edge.test.ts', + 'test/unit/group/fs-utils.test.ts', 'test/unit/onnxruntime-node-resolver.test.ts', // Windows cmd.exe arg-quoting + compose-and-spawn for the npm install (#2372): // the quoting rules and win32 single-string spawn shape are OS-sensitive, so @@ -140,6 +141,7 @@ const LBUG_NATIVE = [ // opens them through the pool adapter (native addon + bridge file locking). // Windows is skipped in-file (describeReopen) due to the bridge reopen lock. 'test/integration/group/cross-trace-e2e.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/local-backend.test.ts', 'test/integration/local-backend-calltool.test.ts', 'test/integration/search-core.test.ts', diff --git a/gitnexus/src/core/group/PIPELINE.md b/gitnexus/src/core/group/PIPELINE.md index 9a8c4b972..72961cca3 100644 --- a/gitnexus/src/core/group/PIPELINE.md +++ b/gitnexus/src/core/group/PIPELINE.md @@ -16,10 +16,12 @@ flowchart TD D --> E1[TopicExtractor] D --> E2[HttpRouteExtractor] D --> E3[GrpcExtractor] + D --> E4[GraphqlExtractor] E1 --> F[ExtractedContract array
per repo] E2 --> F E3 --> F + E4 --> F B --> M[ManifestExtractor] M --> G[Manifest contracts
+ cross-links] @@ -59,14 +61,27 @@ flowchart TD **Strategy A** (graph-assisted) uses Cypher over edges already produced by the main ingestion pipeline: + - HTTP: `HANDLES_ROUTE` / `FETCHES` edges from `(File)-[]->(Route)` - topic: none (pipeline doesn't yet produce topic nodes — Strategy B only) - gRPC: none (Strategy B + proto map only) -**Strategy B** (source-scan) is 100% tree-sitter based after this PR. +**Strategy B** (source-scan) uses tree-sitter for language source and the +official GraphQL parser for `.graphql` / `.gql` operation documents. Each `*-patterns/.ts` plugin owns its grammar + S-expression queries; the top-level orchestrator imports neither. +GraphQL detection is opt-in with `detect.graphql: true`. The initial slice +recognizes imported NestJS `Query`, `Mutation`, and `Subscription` decorators on +top-level imported `Resolver` classes, plus named operation documents. Generated +object documents, static `gql` templates, and `TypedDocumentString` values are +verified against the operation and its resolved root fragments. Providers and +consumers must resolve to one exact, real graph symbol; anonymous operations, +dynamic decorator names, ambiguous generated symbols, malformed documents, +symlink escapes, and bounded-parser overflows are skipped rather than linked +approximately. `matching.exclude_links_paths` also suppresses configured GraphQL +root fields from exact cross-linking while retaining their registry entries. + ## Plugin architecture ```mermaid @@ -117,6 +132,7 @@ They use the `MATCH (n) WHERE labels(n) IN [...]` allowlist form, NOT the `MATCH (n:A|B)` disjunction — LadybugDB's parser rejects a disjunction that names a reserved keyword (e.g. `Macro`, `Union`), which is what broke the `custom` branch in #2325: + - `topic` → `labels(n) IN ['Function','Method','Class','Interface']` - `grpc`/`thrift` method → `labels(n) IN ['Function','Method']`, service → `labels(n) IN ['Class','Interface']` - `lib` → `labels(n) IN ['Module']` @@ -144,7 +160,7 @@ without coordinating through any shared state. ## Cross-repo trace (`cross-trace.ts`) A second consumer of the bridge. Where cross-impact fans a blast radius -*outward* from one symbol, cross-trace stitches a directed **path** between +_outward_ from one symbol, cross-trace stitches a directed **path** between two symbols that live in different repos: ```mermaid @@ -158,7 +174,7 @@ flowchart TD ``` It reuses the same `symbolUid` join as cross-impact, but issues its own -*pair* query (`listCrossingsBetween`) because a path needs BOTH endpoints of +_pair_ query (`listCrossingsBetween`) because a path needs BOTH endpoints of a crossing — the uid-filtered neighbor join (`resolveBridgeNeighbors`, shared with impact) returns only the far side. The crossing is clamped to one boundary (`MAX_SUPPORTED_CROSS_DEPTH`). With `pdg: true` the boundary-adjacent diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index 754f578f9..b831c6706 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -1,10 +1,15 @@ import { createRequire } from 'node:module'; -import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from './types.js'; +import type { + ContractRole, + GroupConfig, + GroupManifestLink, + ManifestContractType, +} from './types.js'; const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -const VALID_CONTRACT_TYPES: ContractType[] = [ +const VALID_CONTRACT_TYPES: ManifestContractType[] = [ 'http', 'grpc', 'thrift', @@ -26,6 +31,7 @@ const VALID_ROLES: ContractRole[] = ['provider', 'consumer']; // repos that need cross-repo header tracking. const DEFAULT_DETECT = { http: true, + graphql: false, grpc: true, thrift: true, topics: true, @@ -66,7 +72,7 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { if (!link.to || !repoPaths.has(link.to as string)) { throw new Error(`links[${i}].to "${link.to}" does not match any repo path in group`); } - if (!VALID_CONTRACT_TYPES.includes(link.type as ContractType)) { + if (!VALID_CONTRACT_TYPES.includes(link.type as ManifestContractType)) { throw new Error( `links[${i}].type "${link.type}" is invalid. Expected: ${VALID_CONTRACT_TYPES.join(', ')}`, ); @@ -84,13 +90,25 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { return { from: link.from as string, to: link.to as string, - type: link.type as ContractType, + type: link.type as ManifestContractType, contract: String(link.contract), role: link.role as ContractRole, }; }); - const detect = { ...DEFAULT_DETECT, ...((raw.detect as object) || {}) }; + const rawDetect = raw.detect; + if ( + rawDetect !== undefined && + (!rawDetect || typeof rawDetect !== 'object' || Array.isArray(rawDetect)) + ) { + throw new Error('detect must be a mapping of boolean flags'); + } + for (const [key, value] of Object.entries((rawDetect as Record) || {})) { + if (key in DEFAULT_DETECT && typeof value !== 'boolean') { + throw new Error(`detect.${key} must be true or false`); + } + } + const detect = { ...DEFAULT_DETECT, ...((rawDetect as object) || {}) }; const matching = { ...DEFAULT_MATCHING, ...((raw.matching as object) || {}) }; const packages = (raw.packages as Record>) || {}; diff --git a/gitnexus/src/core/group/extractors/fs-utils.ts b/gitnexus/src/core/group/extractors/fs-utils.ts index 384f63203..7f02bbd1d 100644 --- a/gitnexus/src/core/group/extractors/fs-utils.ts +++ b/gitnexus/src/core/group/extractors/fs-utils.ts @@ -21,3 +21,94 @@ export function readSafe(repoPath: string, rel: string): string | null { return null; } } + +/** Read a regular in-repo file without buffering more than `maxBytes`. */ +export async function readSafeBounded( + repoPath: string, + rel: string, + maxBytes: number, +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) return null; + const abs = path.resolve(repoPath, rel); + const base = path.resolve(repoPath); + const relToBase = path.relative(base, abs); + if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; + + try { + const canonicalBase = await fs.promises.realpath(base); + const canonicalFile = await fs.promises.realpath(abs); + const canonicalRelative = path.relative(canonicalBase, canonicalFile); + if (canonicalRelative.startsWith('..') || path.isAbsolute(canonicalRelative)) return null; + const beforeOpen = await fs.promises.lstat(canonicalFile); + if (!beforeOpen.isFile() || beforeOpen.size > maxBytes) return null; + if (maxBytes === 0) return beforeOpen.size === 0 ? '' : null; + + return await new Promise((resolve) => { + const stream = fs.createReadStream(canonicalFile, { + flags: 'r', + start: 0, + end: maxBytes, + autoClose: true, + }); + const chunks: Buffer[] = []; + let totalBytes = 0; + let validated = false; + let settled = false; + + const finish = (value: string | null): void => { + if (settled) return; + settled = true; + resolve(value); + }; + + stream.pause(); + stream.once('open', (fd) => { + try { + const opened = fs.fstatSync(fd); + if (!opened.isFile() || opened.size > maxBytes) { + finish(null); + stream.destroy(); + return; + } + + const currentCanonical = fs.realpathSync(canonicalFile); + const currentRelative = path.relative(canonicalBase, currentCanonical); + const current = fs.statSync(currentCanonical); + if ( + currentRelative.startsWith('..') || + path.isAbsolute(currentRelative) || + opened.dev !== current.dev || + opened.ino !== current.ino + ) { + finish(null); + stream.destroy(); + return; + } + + validated = true; + stream.resume(); + } catch { + finish(null); + stream.destroy(); + } + }); + stream.on('data', (chunk: Buffer | string) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += bytes.length; + if (totalBytes > maxBytes) { + finish(null); + stream.destroy(); + return; + } + chunks.push(bytes); + }); + stream.once('end', () => { + finish(validated ? Buffer.concat(chunks, totalBytes).toString('utf8') : null); + }); + stream.once('error', () => finish(null)); + stream.once('close', () => finish(null)); + }); + } catch { + return null; + } +} diff --git a/gitnexus/src/core/group/extractors/graphql-extractor.ts b/gitnexus/src/core/group/extractors/graphql-extractor.ts new file mode 100644 index 000000000..efafe4e2e --- /dev/null +++ b/gitnexus/src/core/group/extractors/graphql-extractor.ts @@ -0,0 +1,707 @@ +import { glob } from 'glob'; +import { + Kind, + parse, + type DocumentNode, + type FragmentDefinitionNode, + type OperationDefinitionNode, + type SelectionSetNode, +} from 'graphql'; +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import { createIgnoreFilter } from '../../../config/ignore-service.js'; +import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { logger } from '../../logger.js'; +import { ParseTimeoutError, parseSourceSafe } from '../../tree-sitter/safe-parse.js'; +import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; +import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafeBounded } from './fs-utils.js'; + +const PROVIDER_GLOB = '**/*.{ts,tsx,mts,cts}'; +const DOCUMENT_GLOB = '**/*.{graphql,gql}'; +const NEST_GRAPHQL_PACKAGE = '@nestjs/graphql'; +const MAX_GRAPHQL_TOKENS = 100_000; +const MAX_GRAPHQL_DEFINITIONS = 5_000; +const MAX_GRAPHQL_OPERATIONS = 500; +const MAX_GRAPHQL_SELECTIONS = 10_000; +const MAX_GRAPHQL_TRAVERSAL_DEPTH = 64; +const MAX_PROVIDER_AST_NODES = 100_000; +const MAX_PROVIDER_AST_DEPTH = 256; +const GRAPHQL_NAME = /^[_A-Za-z][_0-9A-Za-z]*$/; + +type GraphqlOperationKind = 'query' | 'mutation' | 'subscription'; + +interface ResolvedSymbol { + uid: string; + name: string; + filePath: string; +} + +interface DecoratorBindings { + operations: Map; + resolvers: Set; +} + +type DecoratorFieldName = + | { kind: 'absent' } + | { kind: 'literal'; value: string } + | { kind: 'dynamic' }; + +type GeneratedSymbolIndex = Map; +type GeneratedIndexCache = Map>; + +export const RESOLVE_METHOD_QUERY = ` +MATCH (n) +WHERE labels(n) IN ['Method','Function','Property','CodeElement'] + AND n.name = $name AND n.filePath = $filePath AND n.startLine = $startLine AND n.id <> '' +RETURN n.id AS uid, n.name AS name, n.filePath AS filePath +ORDER BY n.id ASC +LIMIT 2`; + +// LadybugDB returns labels(n) as a scalar string, not Neo4j's string array. +// The real-db integration test executes this exact query and guards that dialect contract. +export const RESOLVE_GENERATED_SYMBOL_QUERY = ` +MATCH (n) +WHERE labels(n) IN ['Const','Variable','Function','Method','CodeElement'] + AND n.name = $name AND n.filePath <> '' AND n.id <> '' +RETURN n.id AS uid, n.name AS name, n.filePath AS filePath +ORDER BY n.id ASC +LIMIT 2`; + +function rowValue(row: Record, key: string, position: number): string { + return String(row[key] ?? row[position] ?? ''); +} + +function uniqueRealSymbol(rows: Record[]): ResolvedSymbol | null { + if (rows.length !== 1) return null; + const row = rows[0]; + const symbol = { + uid: rowValue(row, 'uid', 0), + name: rowValue(row, 'name', 1), + filePath: rowValue(row, 'filePath', 2).replace(/\\/g, '/'), + }; + return symbol.uid && symbol.name && symbol.filePath ? symbol : null; +} + +function unquote(text: string): string | null { + const trimmed = text.trim(); + if (trimmed.length < 2) return null; + const quote = trimmed[0]; + if ((quote !== "'" && quote !== '"' && quote !== '`') || trimmed.at(-1) !== quote) return null; + const value = trimmed.slice(1, -1); + return value.includes('${') ? null : value; +} + +function unwrapExpression(node: Parser.SyntaxNode): Parser.SyntaxNode { + let current = node; + while ( + ['as_expression', 'satisfies_expression', 'parenthesized_expression'].includes(current.type) && + current.namedChildren[0] + ) { + current = current.namedChildren[0]; + } + return current; +} + +function objectPairValue(node: Parser.SyntaxNode, key: string): Parser.SyntaxNode | null { + const object = unwrapExpression(node); + if (object.type !== 'object') return null; + for (const pair of object.namedChildren) { + if (pair.type !== 'pair') continue; + const keyNode = pair.childForFieldName('key'); + const pairKey = keyNode ? (unquote(keyNode.text) ?? keyNode.text) : null; + if (pairKey === key) return pair.childForFieldName('value'); + } + return null; +} + +function literalValue(node: Parser.SyntaxNode | null): string | null { + return node ? unquote(unwrapExpression(node).text) : null; +} + +function graphqlNameValue(node: Parser.SyntaxNode | null): string | null { + return node ? literalValue(objectPairValue(node, 'value')) : null; +} + +function withinGeneratedAstBudget(root: Parser.SyntaxNode): boolean { + const pending: Array<{ node: Parser.SyntaxNode; depth: number }> = [{ node: root, depth: 0 }]; + let visited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + visited++; + if (visited > MAX_PROVIDER_AST_NODES || current.depth > MAX_PROVIDER_AST_DEPTH) return false; + for (let index = current.node.namedChildren.length - 1; index >= 0; index--) { + const child = current.node.namedChildren[index]; + if (child) pending.push({ node: child, depth: current.depth + 1 }); + } + } + return true; +} + +function generatedRootFields( + selectionSet: Parser.SyntaxNode | null, + fragments: ReadonlyMap, +): Set | null { + const fields = new Set(); + if (!selectionSet) return null; + const seenFragments = new Set(); + const pending: Array<{ selectionSet: Parser.SyntaxNode; depth: number }> = [ + { selectionSet, depth: 0 }, + ]; + let selectionsVisited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + const selections = objectPairValue(current.selectionSet, 'selections'); + const array = selections ? unwrapExpression(selections) : null; + if (!array || array.type !== 'array') return null; + for (const item of array.namedChildren) { + selectionsVisited++; + if (selectionsVisited > MAX_GRAPHQL_SELECTIONS) return null; + const selection = unwrapExpression(item); + const kind = literalValue(objectPairValue(selection, 'kind')); + if (kind === 'Field') { + const field = graphqlNameValue(objectPairValue(selection, 'name')); + if (field) fields.add(field); + continue; + } + if (kind === 'InlineFragment') { + const nested = objectPairValue(selection, 'selectionSet'); + if (nested) pending.push({ selectionSet: nested, depth: current.depth + 1 }); + continue; + } + if (kind !== 'FragmentSpread') continue; + const name = graphqlNameValue(objectPairValue(selection, 'name')); + if (!name || seenFragments.has(name)) continue; + const fragment = fragments.get(name); + if (!fragment) continue; + const nested = objectPairValue(fragment, 'selectionSet'); + if (!nested) continue; + seenFragments.add(name); + pending.push({ selectionSet: nested, depth: current.depth + 1 }); + } + } + return fields; +} + +function parsedDocumentProof( + source: string, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], +): boolean { + let document: DocumentNode; + try { + document = parse(source, { noLocation: true, maxTokens: MAX_GRAPHQL_TOKENS }); + } catch { + return false; + } + if (document.definitions.length > MAX_GRAPHQL_DEFINITIONS) return false; + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind === Kind.FRAGMENT_DEFINITION) + fragments.set(definition.name.value, definition); + } + for (const definition of document.definitions) { + if (definition.kind !== Kind.OPERATION_DEFINITION) continue; + if (definition.operation !== operationKind || definition.name?.value !== operationName) + continue; + const fields = rootFields(definition.selectionSet, fragments); + return fields !== null && requiredFields.every((field) => fields.includes(field)); + } + return false; +} + +function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { + const value = unwrapExpression(initializer); + if (value.type === 'string') { + if (value.text.startsWith('"')) { + try { + return JSON.parse(value.text) as string; + } catch { + return null; + } + } + return unquote(value.text); + } + if (value.type === 'template_string') return unquote(value.text); + + if (value.type === 'call_expression') { + const template = value.namedChildren.find((child) => child.type === 'template_string'); + return template ? unquote(template.text) : null; + } + + if (value.type !== 'new_expression') return null; + const constructor = value.childForFieldName('constructor') ?? value.namedChildren[0]; + if (!constructor || !constructor.text.endsWith('TypedDocumentString')) return null; + const args = value.childForFieldName('arguments'); + const first = args?.namedChildren[0]; + return first ? staticGraphqlSource(first) : null; +} + +export function hasGeneratedDocumentProof( + initializer: Parser.SyntaxNode, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], +): boolean { + if (!withinGeneratedAstBudget(initializer)) return false; + const staticSource = staticGraphqlSource(initializer); + if (staticSource !== null) { + return parsedDocumentProof(staticSource, operationKind, operationName, requiredFields); + } + const document = unwrapExpression(initializer); + if (literalValue(objectPairValue(document, 'kind')) !== 'Document') return false; + const definitions = objectPairValue(document, 'definitions'); + const array = definitions ? unwrapExpression(definitions) : null; + if (!array || array.type !== 'array') return false; + + const fragments = new Map(); + for (const item of array.namedChildren) { + const definition = unwrapExpression(item); + if (literalValue(objectPairValue(definition, 'kind')) !== 'FragmentDefinition') continue; + const name = graphqlNameValue(objectPairValue(definition, 'name')); + if (name) fragments.set(name, definition); + } + + for (const item of array.namedChildren) { + const definition = unwrapExpression(item); + if (literalValue(objectPairValue(definition, 'kind')) !== 'OperationDefinition') continue; + if (literalValue(objectPairValue(definition, 'operation')) !== operationKind) continue; + if (graphqlNameValue(objectPairValue(definition, 'name')) !== operationName) continue; + const fields = generatedRootFields(objectPairValue(definition, 'selectionSet'), fragments); + if (fields && requiredFields.every((field) => fields.has(field))) return true; + } + return false; +} + +function importedDecoratorBindings(root: Parser.SyntaxNode): DecoratorBindings { + const operations = new Map(); + const resolvers = new Set(); + for (const child of root.namedChildren) { + if (child.type !== 'import_statement') continue; + const source = child.childForFieldName('source'); + if (!source || unquote(source.text) !== NEST_GRAPHQL_PACKAGE) continue; + + const namedImports = child.namedChildren + .find((node) => node.type === 'import_clause') + ?.namedChildren.find((node) => node.type === 'named_imports'); + if (!namedImports) continue; + + for (const specifier of namedImports.namedChildren) { + if (specifier.type !== 'import_specifier') continue; + const imported = specifier.childForFieldName('name')?.text; + const local = specifier.childForFieldName('alias')?.text ?? imported; + if (!imported || !local) continue; + const kind = imported.toLowerCase(); + if (kind === 'query' || kind === 'mutation' || kind === 'subscription') { + operations.set(local, kind); + } else if (imported === 'Resolver') { + resolvers.add(local); + } + } + } + return { operations, resolvers }; +} + +function decoratorKind( + decorator: Parser.SyntaxNode, + bindings: Map, +): { kind: GraphqlOperationKind; argumentsNode?: Parser.SyntaxNode } | null { + const expression = decorator.namedChildren[0]; + if (!expression) return null; + if (expression.type === 'identifier') { + const kind = bindings.get(expression.text); + return kind ? { kind } : null; + } + if (expression.type !== 'call_expression') return null; + const callee = expression.childForFieldName('function'); + if (!callee || callee.type !== 'identifier') return null; + const kind = bindings.get(callee.text); + if (!kind) return null; + return { kind, argumentsNode: expression.childForFieldName('arguments') ?? undefined }; +} + +function decoratorFieldName(argumentsNode: Parser.SyntaxNode | undefined): DecoratorFieldName { + if (!argumentsNode || argumentsNode.namedChildren.length === 0) return { kind: 'absent' }; + const args = argumentsNode.namedChildren; + if (args[0] && ['string', 'template_string'].includes(args[0].type)) { + const direct = unquote(args[0].text); + return direct === null ? { kind: 'dynamic' } : { kind: 'literal', value: direct }; + } + + let sawOptions = false; + + for (const arg of args) { + if (arg.type !== 'object') continue; + sawOptions = true; + for (const pair of arg.namedChildren) { + if (pair.type === 'spread_element' || pair.type.startsWith('shorthand_property_identifier')) { + return { kind: 'dynamic' }; + } + if (pair.type !== 'pair') continue; + const key = pair.childForFieldName('key')?.text.replace(/^['"]|['"]$/g, ''); + if (key !== 'name') continue; + const value = pair.childForFieldName('value'); + if (!value || !['string', 'template_string'].includes(value.type)) { + return { kind: 'dynamic' }; + } + const literal = unquote(value.text); + return literal === null ? { kind: 'dynamic' } : { kind: 'literal', value: literal }; + } + } + if (sawOptions || args.length === 1) return { kind: 'absent' }; + return { kind: 'dynamic' }; +} + +function topLevelResolverClassBodies( + root: Parser.SyntaxNode, + resolverBindings: ReadonlySet, +): Parser.SyntaxNode[] | null { + if (!withinGeneratedAstBudget(root)) return null; + const bodies: Parser.SyntaxNode[] = []; + for (const statement of root.namedChildren) { + const classNode = + statement.type === 'class_declaration' + ? statement + : statement.type === 'export_statement' + ? statement.namedChildren.find((child) => child.type === 'class_declaration') + : undefined; + if (!classNode) continue; + const decorators = [ + ...new Set([ + ...statement.namedChildren.filter((child) => child.type === 'decorator'), + ...classNode.namedChildren.filter((child) => child.type === 'decorator'), + ]), + ]; + const isResolver = decorators.some((decorator) => { + const expression = decorator.namedChildren[0]; + if (!expression) return false; + const callee = + expression.type === 'call_expression' + ? expression.childForFieldName('function') + : expression; + return callee?.type === 'identifier' && resolverBindings.has(callee.text); + }); + if (!isResolver) continue; + const body = classNode.childForFieldName('body'); + if (body) bodies.push(body); + } + return bodies; +} + +function rootFields( + selectionSet: SelectionSetNode, + fragments: ReadonlyMap, +): string[] | null { + const fields: string[] = []; + const seenFragments = new Set(); + const pending: Array<{ selectionSet: SelectionSetNode; depth: number }> = [ + { selectionSet, depth: 0 }, + ]; + let selectionsVisited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + for (const selection of current.selectionSet.selections) { + selectionsVisited++; + if (selectionsVisited > MAX_GRAPHQL_SELECTIONS) return null; + if (selection.kind === Kind.FIELD) { + fields.push(selection.name.value); + continue; + } + if (selection.kind === Kind.INLINE_FRAGMENT) { + pending.push({ selectionSet: selection.selectionSet, depth: current.depth + 1 }); + continue; + } + const name = selection.name.value; + if (seenFragments.has(name)) continue; + const fragment = fragments.get(name); + if (!fragment) continue; + seenFragments.add(name); + pending.push({ selectionSet: fragment.selectionSet, depth: current.depth + 1 }); + } + } + return fields; +} + +function generatedCandidates(operation: OperationDefinitionNode): string[] { + const name = operation.name?.value; + return name ? [`${name}Document`] : []; +} + +async function generatedDocumentMatches( + repoPath: string, + symbol: ResolvedSymbol, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], + cache: GeneratedIndexCache, +): Promise { + const normalizedPath = symbol.filePath.replace(/\\/g, '/'); + let pendingIndex = cache.get(normalizedPath); + if (!pendingIndex) { + pendingIndex = buildGeneratedSymbolIndex(repoPath, normalizedPath); + cache.set(normalizedPath, pendingIndex); + } + const index = await pendingIndex; + const values = index?.get(symbol.name) ?? []; + return values.some((value) => + hasGeneratedDocumentProof(value, operationKind, operationName, requiredFields), + ); +} + +async function buildGeneratedSymbolIndex( + repoPath: string, + filePath: string, +): Promise { + const source = await readSafeBounded(repoPath, filePath, getMaxFileSizeBytes()); + if (source === null) return null; + const parser = new Parser(); + parser.setLanguage( + filePath.toLowerCase().endsWith('.tsx') ? TypeScript.tsx : TypeScript.typescript, + ); + let tree: Parser.Tree; + try { + tree = parseSourceSafe(parser, source, undefined, undefined, filePath); + } catch (error) { + if (error instanceof ParseTimeoutError) return null; + throw error; + } + + return indexGeneratedDeclarators(tree.rootNode); +} + +export function indexGeneratedDeclarators(root: Parser.SyntaxNode): GeneratedSymbolIndex { + const index: GeneratedSymbolIndex = new Map(); + const pending = [root]; + while (pending.length > 0) { + const node = pending.pop(); + if (!node) break; + if (node.type === 'variable_declarator') { + const name = node.childForFieldName('name')?.text; + const value = node.childForFieldName('value'); + if (name && value) { + const values = index.get(name) ?? []; + values.push(value); + index.set(name, values); + } + } + for (let child = node.namedChildren.length - 1; child >= 0; child--) { + pending.push(node.namedChildren[child]); + } + } + return index; +} + +function dedupe(contracts: ExtractedContract[]): ExtractedContract[] { + const seen = new Set(); + return contracts.filter((contract) => { + const key = `${contract.contractId}|${contract.role}|${contract.symbolUid}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +export class GraphqlExtractor implements ContractExtractor { + type = 'graphql' as const; + + async canExtract(_repo: RepoHandle): Promise { + return true; + } + + async extract( + dbExecutor: CypherExecutor | null, + repoPath: string, + _repo: RepoHandle, + ): Promise { + if (!dbExecutor) return []; + const ignore = await createIgnoreFilter(repoPath); + const [providerFiles, documentFiles] = await Promise.all([ + glob(PROVIDER_GLOB, { cwd: repoPath, ignore, nodir: true }), + glob(DOCUMENT_GLOB, { cwd: repoPath, ignore, nodir: true }), + ]); + const contracts = [ + ...(await this.extractProviders(dbExecutor, repoPath, providerFiles)), + ...(await this.extractConsumers(dbExecutor, repoPath, documentFiles)), + ]; + return dedupe(contracts); + } + + private async extractProviders( + dbExecutor: CypherExecutor, + repoPath: string, + files: string[], + ): Promise { + const parser = new Parser(); + const contracts: ExtractedContract[] = []; + const maxFileSizeBytes = getMaxFileSizeBytes(); + for (const rel of files) { + if (/\.(?:spec|test)\.[cm]?tsx?$/i.test(rel)) continue; + const source = await readSafeBounded(repoPath, rel, maxFileSizeBytes); + if (source === null || !source.includes(NEST_GRAPHQL_PACKAGE)) continue; + parser.setLanguage( + rel.toLowerCase().endsWith('.tsx') ? TypeScript.tsx : TypeScript.typescript, + ); + let tree: Parser.Tree; + try { + tree = parseSourceSafe(parser, source, undefined, undefined, rel); + } catch (error) { + if (error instanceof ParseTimeoutError) continue; + throw error; + } + const bindings = importedDecoratorBindings(tree.rootNode); + if (bindings.operations.size === 0 || bindings.resolvers.size === 0) continue; + const bodies = topLevelResolverClassBodies(tree.rootNode, bindings.resolvers); + if (bodies === null) continue; + for (const body of bodies) { + let decorators: Parser.SyntaxNode[] = []; + for (const member of body.namedChildren) { + if (member.type === 'comment') continue; + if (member.type === 'decorator') { + decorators.push(member); + continue; + } + if (member.type !== 'method_definition' && member.type !== 'public_field_definition') { + decorators = []; + continue; + } + const memberDecorators = [ + ...new Set([ + ...decorators, + ...member.namedChildren.filter((child) => child.type === 'decorator'), + ]), + ]; + const methodName = member.childForFieldName('name')?.text; + if (!methodName) { + decorators = []; + continue; + } + for (const decorator of memberDecorators) { + const operation = decoratorKind(decorator, bindings.operations); + if (!operation) continue; + const parsedField = decoratorFieldName(operation.argumentsNode); + if (parsedField.kind === 'dynamic') continue; + const field = parsedField.kind === 'literal' ? parsedField.value : methodName; + if (!GRAPHQL_NAME.test(field)) continue; + const filePath = rel.replace(/\\/g, '/'); + const symbol = uniqueRealSymbol( + await dbExecutor(RESOLVE_METHOD_QUERY, { + name: methodName, + filePath, + startLine: + member.type === 'public_field_definition' + ? (member.childForFieldName('value')?.startPosition.row ?? + member.startPosition.row) + 1 + : member.startPosition.row + 1, + }), + ); + if (!symbol) continue; + contracts.push({ + contractId: `graphql::${operation.kind}::${field}`, + type: 'graphql', + role: 'provider', + symbolUid: symbol.uid, + symbolRef: { filePath: symbol.filePath, name: symbol.name }, + symbolName: symbol.name, + confidence: 1, + meta: { + operationKind: operation.kind, + fieldName: field, + resolverPath: filePath, + extractionStrategy: 'nestjs_decorator', + }, + }); + } + decorators = []; + } + } + } + return contracts; + } + + private async extractConsumers( + dbExecutor: CypherExecutor, + repoPath: string, + files: string[], + ): Promise { + const contracts: ExtractedContract[] = []; + const generatedIndexCache: GeneratedIndexCache = new Map(); + const maxFileSizeBytes = getMaxFileSizeBytes(); + for (const rel of files) { + const source = await readSafeBounded(repoPath, rel, maxFileSizeBytes); + if (source === null) continue; + let document: DocumentNode; + try { + document = parse(source, { noLocation: true, maxTokens: MAX_GRAPHQL_TOKENS }); + } catch (error) { + logger.debug({ file: rel, error }, 'skipping invalid GraphQL document'); + continue; + } + if (document.definitions.length > MAX_GRAPHQL_DEFINITIONS) continue; + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind === Kind.FRAGMENT_DEFINITION) { + fragments.set(definition.name.value, definition); + } + } + const operations = document.definitions.filter( + (definition): definition is OperationDefinitionNode => + definition.kind === Kind.OPERATION_DEFINITION && definition.name !== undefined, + ); + if (operations.length > MAX_GRAPHQL_OPERATIONS) continue; + for (const definition of operations) { + const operationName = definition.name?.value; + if (!operationName) continue; + const documentPath = rel.replace(/\\/g, '/'); + const operationFields = rootFields(definition.selectionSet, fragments); + if (operationFields === null) continue; + const uniqueFields = [...new Set(operationFields)]; + let symbol: ResolvedSymbol | null = null; + for (const candidate of generatedCandidates(definition)) { + const resolved = uniqueRealSymbol( + await dbExecutor(RESOLVE_GENERATED_SYMBOL_QUERY, { name: candidate }), + ); + if ( + resolved && + (await generatedDocumentMatches( + repoPath, + resolved, + definition.operation, + operationName, + uniqueFields, + generatedIndexCache, + )) + ) { + symbol = resolved; + break; + } + } + if (!symbol) continue; + for (const field of uniqueFields) { + contracts.push({ + contractId: `graphql::${definition.operation}::${field}`, + type: 'graphql', + role: 'consumer', + symbolUid: symbol.uid, + symbolRef: { filePath: symbol.filePath, name: symbol.name }, + symbolName: symbol.name, + confidence: 1, + meta: { + operationKind: definition.operation, + operationName: definition.name.value, + fieldName: field, + documentPath, + extractionStrategy: 'graphql_ast', + }, + }); + } + } + } + return contracts; + } +} diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 272c27ce6..d4175442a 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -1,4 +1,9 @@ -import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js'; +import type { + CrossLink, + GroupManifestLink, + ManifestContractType, + StoredContract, +} from '../types.js'; import type { CypherExecutor } from '../contract-extractor.js'; import { logger } from '../../logger.js'; @@ -366,11 +371,11 @@ export class ManifestExtractor { * equality matching without requiring wildcard logic downstream. * * NOTE on exhaustiveness: the switch covers every current - * `ContractType` variant and falls through to a `never` assertion so + * manifest-declared contract type and falls through to a `never` assertion so * TypeScript fails the build if a new variant is added without a * corresponding case. */ - private buildContractId(type: ContractType, contract: string): string { + private buildContractId(type: ManifestContractType, contract: string): string { switch (type) { case 'http': { // Canonicalize method casing and path separators so logically diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index eea6fc102..2647cc009 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -37,7 +37,13 @@ function buildNoisyContractFilter( : new Set(); const excludeParamOnly = matchingConfig?.exclude_links_param_only_paths === true; - return function isNoisyHttpContract(contractId: string): boolean { + return function isNoisyContract(contractId: string): boolean { + if (contractId.startsWith('graphql::')) { + const parts = contractId.split('::'); + if (parts.length < 3) return false; + const field = parts.slice(2).join('::'); + return excludePaths.has(field) || excludePaths.has(`/${field}`); + } if (!contractId.startsWith('http::')) return false; const parts = contractId.split('::'); if (parts.length < 3) return false; diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index 6e82fbb0a..e196095ef 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -96,6 +96,7 @@ packages: {} detect: http: true + graphql: false grpc: true topics: true diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 7efd65ef7..d79cd4de5 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -22,6 +22,7 @@ import type { MatchType, } from './types.js'; import { HttpRouteExtractor } from './extractors/http-route-extractor.js'; +import { GraphqlExtractor } from './extractors/graphql-extractor.js'; import { GrpcExtractor } from './extractors/grpc-extractor.js'; import { ThriftExtractor } from './extractors/thrift-extractor.js'; import { TopicExtractor } from './extractors/topic-extractor.js'; @@ -295,6 +296,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis const entries = registryEntries; const resolve = opts?.resolveRepoHandle ?? defaultResolveHandle(entries); const httpEx = new HttpRouteExtractor(); + const graphqlEx = new GraphqlExtractor(); const grpcEx = new GrpcExtractor(); const thriftEx = new ThriftExtractor(); const topicEx = new TopicExtractor(); @@ -343,6 +345,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis } } + if (config.detect.graphql === true) { + const extracted = await graphqlEx.extract(executor, handle.repoPath, handle); + for (const c of extracted) { + repoContracts.push({ + ...c, + repo: groupPath, + service: assignService(c.symbolRef.filePath, boundaries), + }); + } + } + if (config.detect.grpc) { const extracted = await grpcEx.extract(executor, handle.repoPath, handle); for (const c of extracted) { diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index cf7191664..beeb6f053 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,4 +1,13 @@ -export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include'; +export type ContractType = + | 'http' + | 'graphql' + | 'grpc' + | 'thrift' + | 'topic' + | 'lib' + | 'custom' + | 'include'; +export type ManifestContractType = Exclude; export type MatchType = 'exact' | 'manifest' | 'wildcard'; export type ContractRole = 'provider' | 'consumer'; @@ -16,13 +25,14 @@ export interface GroupConfig { export interface GroupManifestLink { from: string; to: string; - type: ContractType; + type: ManifestContractType; contract: string; role: ContractRole; } export interface DetectConfig { http: boolean; + graphql?: boolean; grpc: boolean; thrift: boolean; topics: boolean; @@ -32,11 +42,12 @@ export interface DetectConfig { export interface MatchingConfig { /** - * HTTP paths to exclude from cross-link matching. Contracts at these paths + * HTTP paths or GraphQL root fields to exclude from cross-link matching. Contracts at these paths * are still extracted and visible in the registry, but they don't produce * cross-repo links. Useful for health-check endpoints (`/ping`, `/health`) * that every service exposes and would otherwise create N×M false links. - * Trailing slashes are normalized before comparison. + * Trailing slashes are normalized before comparison. GraphQL fields may be + * written as `health` or `/health`. * @default [] */ exclude_links_paths?: string[]; diff --git a/gitnexus/src/core/ingestion/utils/symbol-labels.ts b/gitnexus/src/core/ingestion/utils/symbol-labels.ts index a21df10b6..0751324dc 100644 --- a/gitnexus/src/core/ingestion/utils/symbol-labels.ts +++ b/gitnexus/src/core/ingestion/utils/symbol-labels.ts @@ -13,8 +13,8 @@ import type { NodeLabel } from 'gitnexus-shared'; * Single source of truth so the set can't silently drift the way the inline copy * did in #2379. * - * NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY` - * carries a near-identical hand-list that is intentionally a SUBSET — it excludes + * NOTE: group extractor queries in `manifest-extractor.ts` and `graphql-extractor.ts` + * carry near-identical hand-lists that are intentionally SUBSETS — they exclude * `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution * behavior check (would widen which nodes resolve as contract symbols), so it is * deliberately left separate for now. diff --git a/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts new file mode 100644 index 000000000..283f68842 --- /dev/null +++ b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts @@ -0,0 +1,145 @@ +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { afterAll, expect, it, vi } from 'vitest'; +import { GraphqlExtractor } from '../../../src/core/group/extractors/graphql-extractor.js'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import { closeLbug, executeParameterized } from '../../../src/core/lbug/pool-adapter.js'; +import type { GroupConfig, RepoHandle } from '../../../src/core/group/types.js'; +import { withTestLbugDB } from '../../helpers/test-indexed-db.js'; + +const SEED = [ + `CREATE (:Method {id:'method:health', name:'health', filePath:'src/health.resolver.ts', startLine:5, endLine:5, content:'', description:''})`, + `CREATE (:Const {id:'const:health-document', name:'HealthDocument', filePath:'src/generated.ts', startLine:1, endLine:1, content:'', description:''})`, +]; + +withTestLbugDB( + 'graphql-resolve-symbol', + (handle) => { + let providerRoot = ''; + let consumerRoot = ''; + + afterAll(async () => { + try { + await closeLbug(handle.repoId); + } catch { + /* best-effort */ + } + }); + + it('anchors provider and generated Document consumer through real LadybugDB queries', async () => { + providerRoot = path.join(handle.tmpHandle.dbPath, 'provider-repo'); + consumerRoot = path.join(handle.tmpHandle.dbPath, 'consumer-repo'); + await fs.mkdir(path.join(providerRoot, 'src'), { recursive: true }); + await fs.mkdir(path.join(consumerRoot, 'src'), { recursive: true }); + await fs.writeFile( + path.join(providerRoot, 'src/health.resolver.ts'), + `import { Query, Resolver } from '@nestjs/graphql';\n@Resolver()\nclass HealthResolver {\n @Query()\n health() { return 'ok'; }\n}`, + 'utf8', + ); + await fs.writeFile( + path.join(consumerRoot, 'src/health.graphql'), + 'query Health { health }', + 'utf8', + ); + await fs.writeFile( + path.join(consumerRoot, 'src/generated.ts'), + `export const HealthDocument = { + kind: 'Document', + definitions: [{ + kind: 'OperationDefinition', + operation: 'query', + name: { kind: 'Name', value: 'Health' }, + selectionSet: { + kind: 'SelectionSet', + selections: [{ kind: 'Field', name: { kind: 'Name', value: 'health' } }] + } + }] +};`, + 'utf8', + ); + const providerRepo: RepoHandle = { + id: handle.repoId, + path: 'api', + repoPath: providerRoot, + storagePath: handle.tmpHandle.dbPath, + }; + const consumerRepo: RepoHandle = { + id: handle.repoId, + path: 'web', + repoPath: consumerRoot, + storagePath: handle.tmpHandle.dbPath, + }; + + const execute = (query: string, params: Record = {}) => + executeParameterized(handle.repoId, query, params); + const contracts = [ + ...(await new GraphqlExtractor().extract(execute, providerRoot, providerRepo)), + ...(await new GraphqlExtractor().extract(execute, consumerRoot, consumerRepo)), + ]; + + expect(contracts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contractId: 'graphql::query::health', + role: 'provider', + symbolUid: 'method:health', + }), + expect.objectContaining({ + contractId: 'graphql::query::health', + role: 'consumer', + symbolUid: 'const:health-document', + }), + ]), + ); + + const repoManager = await import('../../../src/storage/repo-manager.js'); + const registrySpy = vi.spyOn(repoManager, 'readRegistryStrict').mockResolvedValue([]); + const config: GroupConfig = { + version: 1, + name: 'graphql-production-wiring', + description: '', + repos: { api: 'api', web: 'web' }, + links: [], + packages: {}, + detect: { + http: false, + graphql: true, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: false, + }, + matching: {}, + }; + try { + const synced = await syncGroup(config, { + resolveRepoHandle: async (_regName, groupPath) => + groupPath === 'api' ? providerRepo : consumerRepo, + skipWrite: true, + }); + expect( + synced.contracts.map((contract) => [ + contract.contractId, + contract.role, + contract.symbolUid, + ]), + ).toEqual([ + ['graphql::query::health', 'provider', 'method:health'], + ['graphql::query::health', 'consumer', 'const:health-document'], + ]); + expect(synced.crossLinks).toEqual([ + expect.objectContaining({ + contractId: 'graphql::query::health', + matchType: 'exact', + from: expect.objectContaining({ repo: 'web', symbolUid: 'const:health-document' }), + to: expect.objectContaining({ repo: 'api', symbolUid: 'method:health' }), + }), + ]); + } finally { + registrySpy.mockRestore(); + } + }); + }, + { seed: SEED, poolAdapter: true }, +); diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index 2ede066f4..9644b304b 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -45,6 +45,7 @@ describe('parseGroupConfig', () => { expect(config.packages['hr/common'].npm).toBe('@hr/common'); expect(config.detect.http).toBe(true); expect(config.detect.grpc).toBe(false); + expect(config.detect.graphql).toBe(false); }); it('applies defaults for missing optional fields', () => { @@ -175,6 +176,50 @@ detect: }); }); + describe('detect.graphql opt-in default', () => { + it('defaults GraphQL extraction to false', () => { + const config = parseGroupConfig(`version: 1\nname: test\nrepos: { app: my-app }\n`); + expect(config.detect.graphql).toBe(false); + }); + + it('honors explicit GraphQL extraction', () => { + const config = parseGroupConfig( + `version: 1\nname: test\nrepos: { app: my-app }\ndetect:\n graphql: true\n`, + ); + expect(config.detect.graphql).toBe(true); + }); + + it('rejects string-like detect booleans instead of silently changing behavior', () => { + expect(() => + parseGroupConfig( + `version: 1\nname: test\nrepos: { app: my-app }\ndetect:\n graphql: yes\n`, + ), + ).toThrow(/detect\.graphql must be true or false/i); + expect(() => + parseGroupConfig( + `version: 1\nname: test\nrepos: { app: my-app }\ndetect:\n http: "false"\n`, + ), + ).toThrow(/detect\.http must be true or false/i); + }); + + it('rejects GraphQL manifest links until they can resolve real endpoint symbols', () => { + const yaml = ` +version: 1 +name: test +repos: + web: web-repo + api: api-repo +links: + - from: web + to: api + type: graphql + contract: query::health + role: consumer +`; + expect(() => parseGroupConfig(yaml)).toThrow(/type "graphql" is invalid/i); + }); + }); + it('parses thrift manifest links', () => { const yaml = ` version: 1 diff --git a/gitnexus/test/unit/group/fs-utils.test.ts b/gitnexus/test/unit/group/fs-utils.test.ts new file mode 100644 index 000000000..7f2f45e88 --- /dev/null +++ b/gitnexus/test/unit/group/fs-utils.test.ts @@ -0,0 +1,89 @@ +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { readSafe, readSafeBounded } from '../../../src/core/group/extractors/fs-utils.js'; +import { cleanupTempDir } from '../../helpers/test-db.js'; + +const tempDirs: string[] = []; + +describe('group extractor readSafe', () => { + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => cleanupTempDir(dir))); + }); + + it('rejects a path whose canonical target escapes through a directory symlink', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-outside-')); + tempDirs.push(repo, outside); + await fs.writeFile(path.join(outside, 'secret.graphql'), 'query Secret { secret }', 'utf8'); + await fs.symlink( + outside, + path.join(repo, 'linked'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + + await expect(readSafeBounded(repo, 'linked/secret.graphql', 1024)).resolves.toBeNull(); + }); + + it('reads a regular file within the canonical repository root', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + await fs.writeFile(path.join(repo, 'schema.graphql'), 'query Health { health }', 'utf8'); + + expect(readSafe(repo, 'schema.graphql')).toBe('query Health { health }'); + await expect(readSafeBounded(repo, 'schema.graphql', 1024)).resolves.toBe( + 'query Health { health }', + ); + }); + + it('rejects an oversized sparse file before reading its contents', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + const file = await fs.open(path.join(repo, 'oversized.graphql'), 'w'); + try { + await file.truncate(16 * 1024 * 1024); + } finally { + await file.close(); + } + + await expect(readSafeBounded(repo, 'oversized.graphql', 1024)).resolves.toBeNull(); + }); + + it('accepts a regular file whose size is exactly maxBytes', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + await fs.writeFile(path.join(repo, 'exact.graphql'), '12345678', 'utf8'); + + await expect(readSafeBounded(repo, 'exact.graphql', 8)).resolves.toBe('12345678'); + await expect(readSafeBounded(repo, 'exact.graphql', 7)).resolves.toBeNull(); + }); + + it.skipIf(process.platform === 'win32')( + 'reads a final-file symlink whose canonical target stays inside the repository', + async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + await fs.writeFile(path.join(repo, 'schema.graphql'), 'query Health { health }', 'utf8'); + await fs.symlink('schema.graphql', path.join(repo, 'schema-link.graphql'), 'file'); + + await expect(readSafeBounded(repo, 'schema-link.graphql', 1024)).resolves.toBe( + 'query Health { health }', + ); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'rejects a final-file symlink whose canonical target escapes the repository', + async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-outside-')); + tempDirs.push(repo, outside); + const secret = path.join(outside, 'secret.graphql'); + await fs.writeFile(secret, 'query Secret { secret }', 'utf8'); + await fs.symlink(secret, path.join(repo, 'secret-link.graphql'), 'file'); + + await expect(readSafeBounded(repo, 'secret-link.graphql', 1024)).resolves.toBeNull(); + }, + ); +}); diff --git a/gitnexus/test/unit/group/graphql-extractor.test.ts b/gitnexus/test/unit/group/graphql-extractor.test.ts new file mode 100644 index 000000000..5e6d69505 --- /dev/null +++ b/gitnexus/test/unit/group/graphql-extractor.test.ts @@ -0,0 +1,502 @@ +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { parse } from 'graphql'; +import Parser from 'tree-sitter'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CypherExecutor } from '../../../src/core/group/contract-extractor.js'; +import { + GraphqlExtractor, + indexGeneratedDeclarators, +} from '../../../src/core/group/extractors/graphql-extractor.js'; +import type { RepoHandle } from '../../../src/core/group/types.js'; +import { cleanupTempDir } from '../../helpers/test-db.js'; + +const tempDirs: string[] = []; + +async function makeRepo( + files: Record, +): Promise<{ root: string; repo: RepoHandle }> { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-graphql-')); + tempDirs.push(root); + for (const [relative, content] of Object.entries(files)) { + const absolute = path.join(root, relative); + await fs.mkdir(path.dirname(absolute), { recursive: true }); + await fs.writeFile(absolute, content, 'utf8'); + } + return { + root, + repo: { id: 'test', path: 'app', repoPath: root, storagePath: path.join(root, '.gitnexus') }, + }; +} + +function executor(symbols: Record>>): CypherExecutor { + return async (_query, params = {}) => { + const name = String(params.name ?? ''); + const filePath = params.filePath ? `@${String(params.filePath)}` : ''; + return symbols[`${name}${filePath}`] ?? symbols[name] ?? []; + }; +} + +function generatedDocument( + operation: 'query' | 'mutation' | 'subscription', + name: string, + fields: string[], +): string { + const selections = fields + .map((field) => `{ kind: 'Field', name: { kind: 'Name', value: '${field}' } }`) + .join(', '); + return `{ kind: 'Document', definitions: [{ + kind: 'OperationDefinition', + operation: '${operation}', + name: { kind: 'Name', value: '${name}' }, + selectionSet: { kind: 'SelectionSet', selections: [${selections}] } + }] }`; +} + +describe('GraphqlExtractor', () => { + afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(tempDirs.splice(0).map((dir) => cleanupTempDir(dir))); + }); + + it('anchors NestJS providers and document consumers to exact real symbols', async () => { + const { root, repo } = await makeRepo({ + 'src/widget.resolver.ts': ` +import { Resolver, Query as GqlQuery, Mutation, Subscription } from '@nestjs/graphql'; +@Resolver() +class WidgetResolver { + @GqlQuery(() => Widget, { name: 'widget' }) + fetchWidget() { return null; } + @Mutation('saveWidget') + save() { return null; } + @Subscription() + widgetChanged() { return null; } +}`, + 'src/widget.graphql': ` +fragment WidgetRoot on Query { widget } +query GetWidget { alias: widget ...WidgetRoot } +mutation SaveWidget { saveWidget } +subscription WatchWidget { widgetChanged } +`, + 'src/generated.ts': ` +export const GetWidgetDocument = ${generatedDocument('query', 'GetWidget', ['widget'])}; +export const SaveWidgetDocument = ${generatedDocument('mutation', 'SaveWidget', ['saveWidget'])}; +export const WatchWidgetDocument = ${generatedDocument('subscription', 'WatchWidget', ['widgetChanged'])}; +`, + }); + const run = executor({ + 'fetchWidget@src/widget.resolver.ts': [ + { uid: 'method:fetch', name: 'fetchWidget', filePath: 'src/widget.resolver.ts' }, + ], + 'save@src/widget.resolver.ts': [ + { uid: 'method:save', name: 'save', filePath: 'src/widget.resolver.ts' }, + ], + 'widgetChanged@src/widget.resolver.ts': [ + { uid: 'method:watch', name: 'widgetChanged', filePath: 'src/widget.resolver.ts' }, + ], + GetWidgetDocument: [ + { uid: 'const:get', name: 'GetWidgetDocument', filePath: 'src/generated.ts' }, + ], + SaveWidgetDocument: [ + { uid: 'const:save', name: 'SaveWidgetDocument', filePath: 'src/generated.ts' }, + ], + WatchWidgetDocument: [ + { uid: 'const:watch', name: 'WatchWidgetDocument', filePath: 'src/generated.ts' }, + ], + }); + + const contracts = await new GraphqlExtractor().extract(run, root, repo); + + expect( + contracts.map((contract) => [contract.contractId, contract.role, contract.symbolUid]), + ).toEqual([ + ['graphql::query::widget', 'provider', 'method:fetch'], + ['graphql::mutation::saveWidget', 'provider', 'method:save'], + ['graphql::subscription::widgetChanged', 'provider', 'method:watch'], + ['graphql::query::widget', 'consumer', 'const:get'], + ['graphql::mutation::saveWidget', 'consumer', 'const:save'], + ['graphql::subscription::widgetChanged', 'consumer', 'const:watch'], + ]); + }); + + it('skips unproven decorators, ambiguous anchors, anonymous operations, and invalid documents', async () => { + const { root, repo } = await makeRepo({ + 'src/not-nest.ts': ` +function Query(): MethodDecorator { return () => undefined; } +class LocalResolver { @Query() localOnly() {} }`, + 'src/ambiguous.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +@Resolver() +class AmbiguousResolver { @Query() widget() {} }`, + 'src/anonymous.graphql': `query { widget }`, + 'src/invalid.graphql': `query Broken {`, + 'src/missing.graphql': `query MissingGenerated { widget }`, + }); + const ambiguous = [ + { uid: 'method:a', name: 'widget', filePath: 'src/ambiguous.resolver.ts' }, + { uid: 'method:b', name: 'widget', filePath: 'src/ambiguous.resolver.ts' }, + ]; + + const contracts = await new GraphqlExtractor().extract( + executor({ 'widget@src/ambiguous.resolver.ts': ambiguous }), + root, + repo, + ); + + expect(contracts).toEqual([]); + }); + + it('rejects provider fields that are not GraphQL Names', async () => { + const { root, repo } = await makeRepo({ + 'src/invalid.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +@Resolver() +class InvalidResolver { + @Query('bad::field') separator() {} + @Query('line\\nfeed') newline() {} + @Query('right\\u202etoLeft') bidi() {} + @Query('9startsWithDigit') digit() {} +}`, + }); + let lookups = 0; + const run: CypherExecutor = async () => { + lookups++; + return [{ uid: 'method:invalid', name: 'invalid', filePath: 'src/invalid.resolver.ts' }]; + }; + + expect(await new GraphqlExtractor().extract(run, root, repo)).toEqual([]); + expect(lookups).toBe(0); + }); + + it('skips a provider file whose AST exceeds the traversal depth cap', async () => { + const nested = `${'['.repeat(300)}0${']'.repeat(300)}`; + const { root, repo } = await makeRepo({ + 'src/deep.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +const nested = ${nested}; +@Resolver() +class DeepResolver { @Query() health() {} }`, + }); + let lookups = 0; + + expect( + await new GraphqlExtractor().extract( + async () => { + lookups++; + return [{ uid: 'method:health', name: 'health', filePath: 'src/deep.resolver.ts' }]; + }, + root, + repo, + ), + ).toEqual([]); + expect(lookups).toBe(0); + }); + + it('uses an exact unique Document symbol when no generated hook exists', async () => { + const { root, repo } = await makeRepo({ + 'src/health.graphql': `query Health { health }`, + 'src/generated/graphql.ts': `export const HealthDocument = ${generatedDocument('query', 'Health', ['health'])};`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + HealthDocument: [ + { uid: 'var:health', name: 'HealthDocument', filePath: 'src/generated/graphql.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts).toEqual([ + expect.objectContaining({ + contractId: 'graphql::query::health', + role: 'consumer', + symbolUid: 'var:health', + symbolRef: { filePath: 'src/generated/graphql.ts', name: 'HealthDocument' }, + }), + ]); + }); + + it('skips matching tokens that occur only in unrelated initializer metadata', async () => { + const { root, repo } = await makeRepo({ + 'src/health.graphql': `query Health { health }`, + 'src/generated/graphql.ts': `export const HealthDocument = { + metadata: { operation: 'Health', field: 'health' }, + kind: 'NotADocument' + };`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + HealthDocument: [ + { uid: 'var:health', name: 'HealthDocument', filePath: 'src/generated/graphql.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts).toEqual([]); + }); + + it('fails closed when a generated Document initializer exceeds the AST depth budget', async () => { + const { root, repo } = await makeRepo({ + 'src/deep.graphql': `query Deep { health }`, + 'src/generated.ts': `export const DeepDocument = ${'('.repeat(300)}${generatedDocument( + 'query', + 'Deep', + ['health'], + )}${')'.repeat(300)};`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + DeepDocument: [{ uid: 'const:deep', name: 'DeepDocument', filePath: 'src/generated.ts' }], + }), + root, + repo, + ); + + expect(contracts).toEqual([]); + }); + + it('fails closed for oversized, deeply nested, and excessive-operation documents', async () => { + const nested = `${'... on Query { '.repeat(65)}health${' }'.repeat(65)}`; + const manyOperations = Array.from( + { length: 501 }, + (_, index) => `query Op${index} { field${index} }`, + ).join('\n'); + const { root, repo } = await makeRepo({ + 'src/oversized.graphql': `${' '.repeat(1_000_001)}query Huge { huge }`, + 'src/deep.graphql': `query Deep { ${nested} }`, + 'src/many.graphql': manyOperations, + }); + let lookups = 0; + const run: CypherExecutor = async (_query, params = {}) => { + lookups++; + const name = String(params.name ?? ''); + return name.startsWith('useOp') + ? [{ uid: `fn:${name}`, name, filePath: 'src/generated.ts' }] + : name === 'useDeepQuery' + ? [{ uid: 'fn:deep', name, filePath: 'src/generated.ts' }] + : []; + }; + + const contracts = await new GraphqlExtractor().extract(run, root, repo); + + expect(contracts).toEqual([]); + expect(contracts).not.toContainEqual(expect.objectContaining({ symbolUid: 'fn:deep' })); + expect(lookups).toBe(0); + }); + + it('keeps decorators across comments and supports decorated resolver properties', async () => { + const { root, repo } = await makeRepo({ + 'src/commented.resolver.ts': ` +import { Query, Mutation, Resolver } from '@nestjs/graphql'; +@Resolver() +export class CommentedResolver { + @Query() + /** Public schema description. */ + health() { return true; } + + @Mutation() + // The comment is not an ownership boundary. + save = async () => true; +}`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + 'health@src/commented.resolver.ts': [ + { uid: 'method:health', name: 'health', filePath: 'src/commented.resolver.ts' }, + ], + 'save@src/commented.resolver.ts': [ + { uid: 'property:save', name: 'save', filePath: 'src/commented.resolver.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.contractId)).toEqual([ + 'graphql::query::health', + 'graphql::mutation::save', + ]); + }); + + it('requires a top-level imported Resolver and skips dynamic field names', async () => { + const { root, repo } = await makeRepo({ + 'src/scoped.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +const FIELD = 'viewer'; +const NAMES = { viewer: FIELD }; +const name = FIELD; +const opts = { name: FIELD }; +class Helper { @Query() helperOnly() {} } +function factory() { + @Resolver() + class NestedResolver { @Query() nestedOnly() {} } + return NestedResolver; +} +@Resolver() +class RealResolver { + @Query(() => String, { name: FIELD }) dynamicName() {} + @Query(() => String, { name: NAMES.viewer }) memberName() {} + @Query(() => String, { name }) shorthandName() {} + @Query(() => String, { ...opts }) spreadOptions() {} + @Query(() => String, { name: \`get\${FIELD}\` }) interpolatedName() {} + @Query(() => String, { name: 'get' + 'Widget' }) concatenatedName() {} + @Query(() => String) stableName() {} +}`, + }); + const lookedUp: string[] = []; + const run: CypherExecutor = async (_query, params = {}) => { + lookedUp.push(String(params.name)); + return [ + { + uid: `method:${String(params.name)}`, + name: String(params.name), + filePath: 'src/scoped.resolver.ts', + }, + ]; + }; + + const contracts = await new GraphqlExtractor().extract(run, root, repo); + + expect(lookedUp).toEqual(['stableName']); + expect(contracts).toEqual([ + expect.objectContaining({ contractId: 'graphql::query::stableName' }), + ]); + }); + + it('does not extract co-located spec resolvers', async () => { + const { root, repo } = await makeRepo({ + 'src/widget.resolver.spec.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +@Resolver() +class MockResolver { @Query() widget() {} }`, + }); + let lookups = 0; + + const contracts = await new GraphqlExtractor().extract( + async () => { + lookups++; + return [{ uid: 'method:mock', name: 'widget', filePath: 'src/widget.resolver.spec.ts' }]; + }, + root, + repo, + ); + + expect(contracts).toEqual([]); + expect(lookups).toBe(0); + }); + + it('indexes a generated declaration after more than 100000 earlier AST nodes', () => { + const filler = { type: 'identifier', namedChildren: [] } as unknown as Parser.SyntaxNode; + const name = { text: 'LateDocument' } as Parser.SyntaxNode; + const value = { type: 'object', namedChildren: [] } as unknown as Parser.SyntaxNode; + const declaration = { + type: 'variable_declarator', + namedChildren: [], + childForFieldName: (field: string) => + field === 'name' ? name : field === 'value' ? value : null, + } as unknown as Parser.SyntaxNode; + const root = { + type: 'program', + namedChildren: [...Array(100_001).fill(filler), declaration], + } as unknown as Parser.SyntaxNode; + + expect(indexGeneratedDeclarators(root).get('LateDocument')).toEqual([value]); + }); + + it('proves generated root fields through fragment spreads and inline fragments', async () => { + const { root, repo } = await makeRepo({ + 'src/widgets.graphql': ` +query GetWidgets { widget ...MoreRoots ... on Query { inlineRoot } } +fragment MoreRoots on Query { gadget } +`, + 'src/generated.ts': ` +export const GetWidgetsDocument = ${JSON.stringify( + parse(` +query GetWidgets { widget ...MoreRoots ... on Query { inlineRoot } } +fragment MoreRoots on Query { gadget } +`), + )};`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + GetWidgetsDocument: [ + { uid: 'const:widgets', name: 'GetWidgetsDocument', filePath: 'src/generated.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.contractId).sort()).toEqual([ + 'graphql::query::gadget', + 'graphql::query::inlineRoot', + 'graphql::query::widget', + ]); + }); + + it('accepts static gql tags and TypedDocumentString initializers', async () => { + const { root, repo } = await makeRepo({ + 'src/tagged.graphql': `query Tagged { tagged }`, + 'src/string.graphql': `query StringMode { stringMode }`, + 'src/generated.ts': ` +export const TaggedDocument = gql\`query Tagged { tagged }\`; +export const StringModeDocument = new TypedDocumentString("query StringMode {\\n stringMode\\n}"); +`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + TaggedDocument: [ + { uid: 'const:tagged', name: 'TaggedDocument', filePath: 'src/generated.ts' }, + ], + StringModeDocument: [ + { uid: 'const:string', name: 'StringModeDocument', filePath: 'src/generated.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.symbolUid).sort()).toEqual([ + 'const:string', + 'const:tagged', + ]); + }); + + it('parses one generated module once and continues past a shadowed candidate', async () => { + const { root, repo } = await makeRepo({ + 'src/one.graphql': `query One { one }`, + 'src/two.graphql': `query Two { two }`, + 'src/generated.ts': ` +function shadow() { const OneDocument = { kind: 'NotADocument' }; } +export const OneDocument = ${generatedDocument('query', 'One', ['one'])}; +export const TwoDocument = ${generatedDocument('query', 'Two', ['two'])}; +`, + }); + const parseSpy = vi.spyOn(Parser.prototype, 'parse'); + + const contracts = await new GraphqlExtractor().extract( + executor({ + OneDocument: [{ uid: 'const:one', name: 'OneDocument', filePath: 'src/generated.ts' }], + TwoDocument: [{ uid: 'const:two', name: 'TwoDocument', filePath: 'src/generated.ts' }], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.symbolUid).sort()).toEqual([ + 'const:one', + 'const:two', + ]); + expect(parseSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/test/unit/group/manifest-label-drift.test.ts b/gitnexus/test/unit/group/manifest-label-drift.test.ts index 18802e8c5..dea8553c6 100644 --- a/gitnexus/test/unit/group/manifest-label-drift.test.ts +++ b/gitnexus/test/unit/group/manifest-label-drift.test.ts @@ -9,6 +9,10 @@ */ import { describe, it, expect } from 'vitest'; import { CUSTOM_CONTRACT_RESOLVE_QUERY } from '../../../src/core/group/extractors/manifest-extractor.js'; +import { + RESOLVE_GENERATED_SYMBOL_QUERY, + RESOLVE_METHOD_QUERY, +} from '../../../src/core/group/extractors/graphql-extractor.js'; import { SYMBOL_NODE_LABELS } from '../../../src/core/ingestion/utils/symbol-labels.js'; describe('manifest contract-resolve label list vs SYMBOL_NODE_LABELS (#2380)', () => { @@ -32,3 +36,17 @@ describe('manifest contract-resolve label list vs SYMBOL_NODE_LABELS (#2380)', ( expect(diff).toEqual(['Module', 'Namespace', 'Variable']); }); }); + +describe.each([ + ['GraphQL provider', RESOLVE_METHOD_QUERY], + ['GraphQL generated symbol', RESOLVE_GENERATED_SYMBOL_QUERY], +])('%s query label list vs SYMBOL_NODE_LABELS', (_name, query) => { + const match = query.match(/labels\(n\) IN \[([^\]]+)\]/); + const queryLabels = (match?.[1]?.match(/'([^']+)'/g) ?? []).map((label) => label.slice(1, -1)); + const symbolLabels = new Set(SYMBOL_NODE_LABELS); + + it('keeps every hand-listed label in the shared symbol label set', () => { + expect(queryLabels.length).toBeGreaterThan(0); + for (const label of queryLabels) expect(symbolLabels.has(label)).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/group/matching.test.ts b/gitnexus/test/unit/group/matching.test.ts index a10f1d017..bc9937e85 100644 --- a/gitnexus/test/unit/group/matching.test.ts +++ b/gitnexus/test/unit/group/matching.test.ts @@ -183,6 +183,24 @@ describe('runExactMatch', () => { expect(matched).toHaveLength(0); }); + it('suppresses configured GraphQL root fields from exact matching', () => { + const provider = { + ...makeContract('graphql::query::health', 'provider', 'api'), + type: 'graphql' as const, + }; + const consumer = { + ...makeContract('graphql::query::health', 'consumer', 'web'), + type: 'graphql' as const, + }; + + const { matched, unmatched } = runExactMatch([provider, consumer], undefined, { + exclude_links_paths: ['/health'], + }); + + expect(matched).toEqual([]); + expect(unmatched).toEqual([]); + }); + it('does not match same-repo when only one has service', () => { const contracts: StoredContract[] = [ { diff --git a/gitnexus/test/unit/group/sync.test.ts b/gitnexus/test/unit/group/sync.test.ts index 68fe13c50..95bdd96d7 100644 --- a/gitnexus/test/unit/group/sync.test.ts +++ b/gitnexus/test/unit/group/sync.test.ts @@ -23,6 +23,7 @@ describe('syncGroup', () => { packages: {}, detect: { http: true, + graphql: false, grpc: false, thrift: false, topics: false, @@ -72,6 +73,33 @@ describe('syncGroup', () => { expect(result.unmatched).toHaveLength(0); }); + it('exact-matches GraphQL root fields across repositories', async () => { + const config = makeConfig({ api: 'api-repo', web: 'web-repo' }); + const contracts: StoredContract[] = [ + { + ...makeContract('graphql::query::widget', 'provider', 'api'), + type: 'graphql', + }, + { + ...makeContract('graphql::query::widget', 'consumer', 'web'), + type: 'graphql', + }, + ]; + + const result = await syncGroup(config, { + extractorOverride: async () => contracts, + skipWrite: true, + }); + + expect(result.crossLinks).toEqual([ + expect.objectContaining({ + type: 'graphql', + contractId: 'graphql::query::widget', + matchType: 'exact', + }), + ]); + }); + it('reports missing repos', async () => { const config = makeConfig({ 'app/backend': 'nonexistent-repo' }); diff --git a/gitnexus/test/unit/group/types.test.ts b/gitnexus/test/unit/group/types.test.ts index cfa9abba2..94b0b556c 100644 --- a/gitnexus/test/unit/group/types.test.ts +++ b/gitnexus/test/unit/group/types.test.ts @@ -20,6 +20,7 @@ describe('Group types', () => { packages: {}, detect: { http: true, + graphql: true, grpc: true, thrift: true, topics: true, @@ -48,7 +49,7 @@ describe('Group types', () => { }); it('ExtractedContract accepts all contract types', () => { - const types: ContractType[] = ['http', 'grpc', 'topic', 'lib', 'custom']; + const types: ContractType[] = ['http', 'graphql', 'grpc', 'topic', 'lib', 'custom']; types.forEach((t) => { const contract: ExtractedContract = { contractId: `${t}::test`, @@ -88,6 +89,7 @@ describe('Group types', () => { packages: {}, detect: { http: true, + graphql: true, grpc: true, thrift: true, topics: true, diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 9d996279a..4f9d73256 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -95,6 +95,7 @@ export default defineConfig({ 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/manifest-synthetic-impact-lbug.test.ts', 'test/integration/group/http-route-resolve-symbol.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/fts-stemmer-sweep.test.ts', 'test/integration/lbug-multiwriter-deadlock.test.ts', 'test/integration/extension-binary-real.test.ts', @@ -169,6 +170,7 @@ export default defineConfig({ 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/manifest-synthetic-impact-lbug.test.ts', 'test/integration/group/http-route-resolve-symbol.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/skills-e2e.test.ts', 'test/integration/fts-extension-e2e.test.ts', 'test/integration/fts-stemmer-sweep.test.ts', From 38a0837e4b3295eb3c916329a9ba572694cde259 Mon Sep 17 00:00:00 2001 From: "John R. Eakin" Date: Sat, 29 Aug 2026 02:46:30 -0500 Subject: [PATCH 16/28] feat(wiki): add grok local CLI provider (#3069) * feat(wiki): add grok local CLI provider Wiki generation can use `gitnexus wiki --provider grok` to spawn the authenticated Grok Build CLI (`grok --prompt-file`) instead of an HTTP API key. * style(wiki): prettier grok-client for CI format check CI quality/format failed on grok-client.ts. Auto-format matches repo prettier so the GitNexus /autofix comment is applied locally. * Update Grok CLI configuration to use empty allowlist and increase max tu * Replace Grok tool allowlist with explicit denylist and strict sandbox * Increase Grok max turns to 15 to accommodate prompt variance * chore(wiki): drop Unreleased CHANGELOG hunk and restore lockfile libc selectors Feature PRs do not own CHANGELOG.md. Restore the 16 libc platform selectors deleted from package-lock.json with no dependency change. * fix(wiki): resolve grok CLI through Windows cmd.exe shims Extract resolveWindowsCliCommand from the local CLI client and use it for grok detect/spawn so npm .cmd installs work without a shell. Keep detectGrokCLI() returning the display name for the wiki menu. * fix(wiki): wait for grok child close before timeout cleanup Do not reject the grok spawn promise on the timeout timer. Kill the child, escalate SIGKILL after 2s, and reject only on close (or a second 2s hard deadline) so callGrokLLM cannot rm the sandbox while the process is alive. * fix(wiki): reject incomplete grok stopReason and distinct parse errors Honor JSON stopReason (end_turn or omitted succeeds; anything else throws). Split empty-output / non-JSON / missing-text messages and include a truncated stdout excerpt. Drop unused GrokConfig.workingDirectory. * fix(wiki): keep grok temp dir on hung timeout and ignore stdin Hard-deadline reject no longer removes --cwd while the child may still be running. Spawn stdin is ignored so grok's unused pipe cannot EPIPE the wiki process. Co-Authored-By: Grok 4.6 * fix(wiki): require grok stopReason=end_turn for a finished page Live grok 1.0.5 with wiki spawn flags returns stopReason end_turn. Omitted, null, or empty stopReason is no longer treated as success, so generateLeafPage cannot write a page that never completed. Co-Authored-By: Grok 4.6 * style(wiki): deslop grok parse nesting and extra comments Flatten parseGrokOutput with early returns and drop narrative comments that restated the timeout/stdin/stopReason constraints. Behavior unchanged. Co-Authored-By: Grok 4.6 * test(wiki): make grok Windows spawn tests match real cmd.exe On Windows CI, detectGrokCLI also calls where.exe, ComSpec is an absolute cmd.exe path, and waitForSpawn must wait for real fs I/O. Co-Authored-By: Grok 4.6 * test(wiki): expect taskkill on Windows grok timeout, not child.kill killChildTree uses taskkill /T /F on win32 and only falls back to child.kill() if that fails. Co-Authored-By: Grok 4.6 * test(wiki): remove grok temp dir after hard-deadline leak assertion The hard-deadline test must keep the dir until close, then emit close so late cleanup runs and the temp directory is not left behind. Co-Authored-By: Grok 4.6 * test(wiki): wait for grok temp dir rm after late close Windows CI failed the hard-deadline test because 30 setImmediate ticks cannot observe fire-and-forget fs.rm. Poll with real timers after close. --------- Co-authored-by: Grok 4.6 --- .claude/skills/gitnexus-cli/SKILL.md | 8 +- README.md | 1 + .../skills/gitnexus-cli/SKILL.md | 40 +- gitnexus/README.md | 1 + gitnexus/skills/gitnexus-cli.md | 8 +- gitnexus/src/cli/i18n/en.ts | 2 +- gitnexus/src/cli/i18n/zh-CN.ts | 2 +- gitnexus/src/cli/index.ts | 2 +- gitnexus/src/cli/wiki.ts | 24 +- gitnexus/src/core/wiki/generator.ts | 8 + gitnexus/src/core/wiki/grok-client.ts | 359 ++++++++++ gitnexus/src/core/wiki/llm-client.ts | 8 +- gitnexus/src/core/wiki/local-cli-client.ts | 38 +- gitnexus/src/storage/repo-manager.ts | 2 + gitnexus/test/integration/cli-e2e.test.ts | 9 + gitnexus/test/unit/cli-index-help.test.ts | 1 + .../test/unit/local-cli-subprocess.test.ts | 622 +++++++++++++++++- gitnexus/test/unit/wiki-flags.test.ts | 118 ++++ 18 files changed, 1207 insertions(+), 46 deletions(-) create mode 100644 gitnexus/src/core/wiki/grok-client.ts diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index 853d44860..e993c38f8 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -55,15 +55,19 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. | Flag | Effect | | ------------------- | ----------------------------------------- | -| `--force` | Force full regeneration | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | | `--model ` | LLM model (default: MiniMax-M3) | | `--base-url ` | LLM API base URL | | `--api-key ` | LLM API key | | `--concurrency ` | Parallel LLM calls (default: 3) | +| `--timeout ` | LLM request timeout in seconds (default: disabled) | +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | ### list — Show all indexed repos diff --git a/README.md b/README.md index e26373aff..e89ddcdb3 100644 --- a/README.md +++ b/README.md @@ -767,6 +767,7 @@ gitnexus wiki # Use a custom model or provider (default model: minimax/minimax-m2.5) gitnexus wiki --model gpt-4o gitnexus wiki --base-url https://api.anthropic.com/v1 +gitnexus wiki --provider grok # local Grok Build CLI (uses `grok login`, no API key) # Force full regeneration gitnexus wiki --force diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index 9c7a1b599..e993c38f8 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -19,14 +19,14 @@ node .gitnexus/run.cjs analyze Run from the project root. This parses all source files, builds the knowledge graph, writes it to `.gitnexus/`, and generates CLAUDE.md / AGENTS.md context files. -| Flag | Effect | -|------|--------| -| `--force` | Force full re-index even if up to date | +| Flag | Effect | +| -------------- | ---------------------------------------------------------------- | +| `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | -**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. +**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. ### status — Check index freshness @@ -44,10 +44,10 @@ node .gitnexus/run.cjs clean Deletes the `.gitnexus/` directory and unregisters the repo from the global registry. Use before re-indexing if the index is corrupt or after removing GitNexus from a project. -| Flag | Effect | -|------|--------| -| `--force` | Skip confirmation prompt | -| `--all` | Clean all indexed repos, not just the current one | +| Flag | Effect | +| --------- | ------------------------------------------------- | +| `--force` | Skip confirmation prompt | +| `--all` | Clean all indexed repos, not just the current one | ### wiki — Generate documentation from the graph @@ -55,19 +55,21 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. -| Flag | Effect | -|------|--------| -| `--force` | Force full regeneration, also required to re-gerenate an existing wiki in a different language | -| `--model ` | LLM model (default: MiniMax-M3) | -| `--base-url ` | LLM API base URL | -| `--api-key ` | LLM API key | -| `--concurrency ` | Parallel LLM calls (default: 3) | -| `--gist` | Publish wiki as a public GitHub Gist | +| Flag | Effect | +| ------------------- | ----------------------------------------- | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | +| `--model ` | LLM model (default: MiniMax-M3) | +| `--base-url ` | LLM API base URL | +| `--api-key ` | LLM API key | +| `--concurrency ` | Parallel LLM calls (default: 3) | | `--timeout ` | LLM request timeout in seconds (default: disabled) | -| `--retries ` | Max LLM retry attempts per request (default: 3) | -| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese)| +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | +| `--gist` | Publish wiki as a public GitHub Gist | + ### list — Show all indexed repos ```bash diff --git a/gitnexus/README.md b/gitnexus/README.md index e524eaa19..8013026f2 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -256,6 +256,7 @@ gitnexus clean # Delete index for current repo gitnexus clean --all --force # Delete all indexes gitnexus wiki [path] # Generate LLM-powered docs from knowledge graph gitnexus wiki --model # Wiki with custom LLM model (default: minimax/minimax-m2.5) +gitnexus wiki --provider grok # Local Grok Build CLI (uses `grok login`, no API key) gitnexus wiki --base-url http://llama-box.local:8080/v1 --allow-insecure-connection llama-box.local # Allow an exact LAN/self-hosted HTTP LLM host; env: GITNEXUS_ALLOW_INSECURE_CONNECTION gitnexus doctor # Show runtime platform capabilities and embedding configuration diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index 853d44860..e993c38f8 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -55,15 +55,19 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. | Flag | Effect | | ------------------- | ----------------------------------------- | -| `--force` | Force full regeneration | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | | `--model ` | LLM model (default: MiniMax-M3) | | `--base-url ` | LLM API base URL | | `--api-key ` | LLM API key | | `--concurrency ` | Parallel LLM calls (default: 3) | +| `--timeout ` | LLM request timeout in seconds (default: disabled) | +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | ### list — Show all indexed repos diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index e893e61ac..c22da5211 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -234,7 +234,7 @@ export const en = { 'Clean parked LadybugDB recovery sidecars (missing-shadow WAL quarantines and dirty-recovery parks)', 'help.option.wiki.force': 'Force full regeneration even if up to date', 'help.option.wiki.provider': - 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: minimax)', + 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax)', 'help.option.wiki.model': 'LLM model or deployment name (default: MiniMax-M3)', 'help.option.wiki.baseUrl': 'LLM API base URL. Azure v1: https://{resource}.openai.azure.com/openai/v1', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 2c066f010..7ef2d244b 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -220,7 +220,7 @@ export const zhCN = { '清理已暂存的 LadybugDB 恢复 sidecar（missing-shadow WAL 隔离文件与 dirty-recovery 暂存文件）', 'help.option.wiki.force': '即使已是最新也强制完整重新生成', 'help.option.wiki.provider': - 'LLM 提供商：minimax、openai、openrouter、azure、custom、cursor、claude、codex 或 opencode（默认：minimax）', + 'LLM 提供商：minimax、openai、openrouter、azure、custom、cursor、claude、codex、opencode 或 grok（默认：minimax）', 'help.option.wiki.model': 'LLM 模型或 deployment 名称（默认：MiniMax-M3）', 'help.option.wiki.baseUrl': 'LLM API base URL。Azure v1：https://{resource}.openai.azure.com/openai/v1', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 1ccf75c2f..a1edf0b53 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -303,7 +303,7 @@ program .option('-f, --force', 'Force full regeneration even if up to date') .option( '--provider ', - 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: minimax)', + 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax)', ) .option('--model ', 'LLM model or deployment name (default: MiniMax-M3)') .option( diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index d65d130a7..007451ef5 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -25,6 +25,7 @@ import { type LLMProvider, } from '../core/wiki/llm-client.js'; import { detectCursorCLI } from '../core/wiki/cursor-client.js'; +import { detectGrokCLI } from '../core/wiki/grok-client.js'; import { detectLocalCLI } from '../core/wiki/local-cli-client.js'; import { logger } from '../core/logger.js'; @@ -65,20 +66,22 @@ function parsePositiveIntegerOption( function isLocalProvider( provider: LLMProvider | undefined, -): provider is 'cursor' | 'claude' | 'codex' | 'opencode' { +): provider is 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok' { return ( provider === 'cursor' || provider === 'claude' || provider === 'codex' || - provider === 'opencode' + provider === 'opencode' || + provider === 'grok' ); } -function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode') { +function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok') { if (provider === 'cursor') return 'cursorModel'; if (provider === 'claude') return 'claudeModel'; if (provider === 'codex') return 'codexModel'; if (provider === 'opencode') return 'opencodeModel'; + if (provider === 'grok') return 'grokModel'; throw new Error(`Unsupported local provider: ${provider satisfies never}`); } @@ -287,7 +290,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) if (!llmConfig.apiKey && !isLocalProvider(llmConfig.provider)) { console.log(' Error: No LLM API key found.'); console.log(' Set MINIMAX_API_KEY, GITNEXUS_API_KEY, or OPENAI_API_KEY,'); - console.log(' or pass --api-key , or use --provider cursor|claude|codex|opencode.\n'); + console.log( + ' or pass --api-key , or use --provider cursor|claude|codex|opencode|grok.\n', + ); process.exitCode = 1; return; } @@ -301,9 +306,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) const hasClaude = detectLocalCLI('claude'); const hasCodex = detectLocalCLI('codex'); const hasOpenCode = detectLocalCLI('opencode'); + const hasGrok = detectGrokCLI(); const localChoices: Array<{ choice: string; - provider: 'cursor' | 'claude' | 'codex' | 'opencode'; + provider: 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok'; }> = []; // Provider selection @@ -346,6 +352,14 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) }); console.log(` [${choice}] OpenCode CLI (local, uses your OpenCode login/config)`); } + if (hasGrok) { + const choice = String(nextChoice++); + localChoices.push({ + choice, + provider: 'grok', + }); + console.log(` [${choice}] Grok CLI (local, uses your Grok Build login)`); + } console.log(''); const maxChoice = String(nextChoice - 1); diff --git a/gitnexus/src/core/wiki/generator.ts b/gitnexus/src/core/wiki/generator.ts index 2e6180731..d1f3aaabc 100644 --- a/gitnexus/src/core/wiki/generator.ts +++ b/gitnexus/src/core/wiki/generator.ts @@ -40,6 +40,7 @@ import { } from './llm-client.js'; import { callCursorLLM, resolveCursorConfig } from './cursor-client.js'; +import { callGrokLLM, resolveGrokConfig } from './grok-client.js'; import { callClaudeLLM, callCodexLLM, @@ -225,6 +226,13 @@ export class WikiGenerator { }); return callCursorLLM(prompt, cursorConfig, systemPrompt, options); } + if (this.llmConfig.provider === 'grok') { + const grokConfig = resolveGrokConfig({ + model: this.llmConfig.model, + requestTimeoutMs: this.llmConfig.requestTimeoutMs, + }); + return callGrokLLM(prompt, grokConfig, systemPrompt, options); + } if ( this.llmConfig.provider === 'claude' || this.llmConfig.provider === 'codex' || diff --git a/gitnexus/src/core/wiki/grok-client.ts b/gitnexus/src/core/wiki/grok-client.ts new file mode 100644 index 000000000..43a2e0160 --- /dev/null +++ b/gitnexus/src/core/wiki/grok-client.ts @@ -0,0 +1,359 @@ +/** + * Grok Build CLI client for wiki generation. + * + * Uses headless `grok --prompt-file` so large wiki prompts are not placed on + * argv or stdin (Grok does not read stdin as the prompt). + */ + +import { spawn, execFileSync } from 'child_process'; +import { StringDecoder } from 'string_decoder'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import type { CallLLMOptions, LLMResponse } from './llm-client.js'; +import { resolveWindowsCliCommand, type ResolvedCliCommand } from './local-cli-client.js'; +import { logger } from '../logger.js'; + +export interface GrokConfig { + model?: string; + requestTimeoutMs?: number; +} + +// Verified live: an empty --tools allowlist does NOT disable the shell tool +// (`run_terminal_cmd`) — Grok still ran `find` across the user's entire home +// directory looking for project context, taking 10+ minutes per call and +// then hitting --max-turns anyway. A denylist naming the tool explicitly is +// what actually blocks it; internal tool IDs, not the CLI-facing names +// (shell is `run_terminal_cmd`, not `bash`). +const GROK_DISALLOWED_TOOLS = 'run_terminal_cmd,search_replace,web_search,web_fetch,spawn_subagent'; + +// Defense in depth beyond the tool denylist: a kernel-enforced (Landlock/ +// Seatbelt) sandbox so reads/writes stay confined to --cwd (our empty temp +// dir) + system paths even if a future tool slips past the denylist. +const GROK_SANDBOX_PROFILE = 'strict'; + +// Verified live with the denylist + sandbox above: turn counts vary run to +// run (observed 3-10 across identical prompts, including the large overview +// prompt that aggregates every module's summary). 15 gives real headroom +// over that variance without leaving a runaway session effectively uncapped. +const GROK_MAX_TURNS = '15'; + +let cachedGrokCommand: ResolvedCliCommand | null | undefined; + +function isVerbose(): boolean { + return process.env.GITNEXUS_VERBOSE === '1'; +} + +function verboseLog(...args: unknown[]): void { + if (isVerbose()) { + logger.info({ args }, '[grok-cli]'); + } +} + +function killChildTree(child: import('child_process').ChildProcess): void { + if (process.platform === 'win32' && child.pid !== undefined) { + try { + execFileSync('taskkill', ['/T', '/F', '/PID', String(child.pid)], { + stdio: 'ignore', + windowsHide: true, + }); + return; + } catch { + // Process may have already exited — fall through to child.kill() + } + } + child.kill(); +} + +/** Returns `'grok'` when the CLI is on PATH, else null. Cached. */ +export function detectGrokCLI(): string | null { + if (cachedGrokCommand !== undefined) return cachedGrokCommand?.displayName ?? null; + const resolved = resolveWindowsCliCommand('grok'); + try { + execFileSync(resolved.command, [...resolved.argsPrefix, '--version'], { + stdio: 'ignore', + windowsHide: true, + }); + cachedGrokCommand = resolved; + } catch (err: unknown) { + const isNotFound = + err instanceof Error && 'code' in err && (err as NodeJS.ErrnoException).code === 'ENOENT'; + if (!isNotFound && err instanceof Error) { + logger.warn( + `grok CLI found but --version failed (exit ${(err as { status?: number }).status ?? '?'}). ` + + 'Ensure it is authenticated: run `grok --version` manually.', + ); + } + cachedGrokCommand = null; + } + return cachedGrokCommand?.displayName ?? null; +} + +function getDetectedGrokCommand(): ResolvedCliCommand | null { + detectGrokCLI(); + return cachedGrokCommand ?? null; +} + +export function resolveGrokConfig(overrides?: Partial): GrokConfig { + return { + model: overrides?.model, + requestTimeoutMs: overrides?.requestTimeoutMs, + }; +} + +function excerpt(raw: string, max = 200): string { + const trimmed = raw.trim(); + if (trimmed.length <= max) return trimmed; + return `${trimmed.slice(0, max)}…`; +} + +function parseGrokOutput(stdout: string): string { + const trimmed = stdout.trim(); + if (!trimmed) { + throw new Error('grok CLI returned empty output'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + throw new Error(`grok CLI returned non-JSON output: ${excerpt(trimmed)}`); + } + + if (!parsed || typeof parsed !== 'object') { + throw new Error(`grok CLI JSON has no text field: ${excerpt(trimmed)}`); + } + + const record = parsed as { + type?: string; + message?: string; + text?: unknown; + stopReason?: unknown; + }; + if (record.type === 'error') { + throw new Error(record.message || 'grok CLI returned an error'); + } + if (typeof record.text !== 'string') { + throw new Error(`grok CLI JSON has no text field: ${excerpt(trimmed)}`); + } + + const content = record.text.trim(); + if (!content) { + throw new Error('grok CLI returned empty text'); + } + + const rawReason = + record.stopReason === undefined || record.stopReason === null + ? '' + : String(record.stopReason).trim(); + if (rawReason.toLowerCase() !== 'end_turn') { + throw new Error( + rawReason + ? `grok CLI stopped with stopReason=${rawReason}` + : 'grok CLI JSON is missing stopReason=end_turn', + ); + } + return content; +} + +/** + * Call Grok Build in headless mode and return the assistant text. + * + * `--cwd` is an empty temp directory (not the repo) so project AGENTS.md + * files are not injected into wiki generation. + */ +export async function callGrokLLM( + prompt: string, + config: GrokConfig, + systemPrompt?: string, + options?: CallLLMOptions, +): Promise { + const grokCmd = getDetectedGrokCommand(); + if (!grokCmd) { + throw new Error( + 'Grok CLI not found. Install Grok Build and ensure `grok` is on PATH. Run `grok login` if unauthenticated.', + ); + } + + const fullPrompt = systemPrompt ? `${systemPrompt}\n\n---\n\n${prompt}` : prompt; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-wiki-grok-')); + const promptPath = path.join(tempDir, 'prompt.txt'); + const childLifecycle = { spawned: false, closed: false }; + + try { + await fs.writeFile(promptPath, fullPrompt, 'utf-8'); + + const args = [ + '--prompt-file', + promptPath, + '--output-format', + 'json', + '--max-turns', + GROK_MAX_TURNS, + '--no-plan', + '--no-subagents', + '--disable-web-search', + '--disallowed-tools', + GROK_DISALLOWED_TOOLS, + '--sandbox', + GROK_SANDBOX_PROFILE, + '--cwd', + tempDir, + ]; + if (config.model) { + args.push('--model', config.model); + } + + verboseLog( + 'Spawning:', + grokCmd.command, + [...grokCmd.argsPrefix, ...args].join(' ').replace(promptPath, '[prompt-file]'), + ); + if (config.model) { + verboseLog('Model:', config.model); + } + + const content = await runGrok( + grokCmd.command, + [...grokCmd.argsPrefix, ...args], + tempDir, + config, + options, + childLifecycle, + ); + return { content }; + } finally { + // Skip rm while a live child may still be using cwd/sandbox/prompt-file. + if (!childLifecycle.spawned || childLifecycle.closed) { + await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined); + } + } +} + +function runGrok( + command: string, + args: string[], + cwd: string, + config: GrokConfig, + options: CallLLMOptions | undefined, + lifecycle: { spawned: boolean; closed: boolean }, +): Promise { + const startTime = Date.now(); + + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + // Prompt is --prompt-file; an unused stdin pipe can EPIPE the wiki process. + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + env: { + ...process.env, + CI: '1', + }, + }); + lifecycle.spawned = true; + + verboseLog('Process spawned with PID:', child.pid); + + let stdout = ''; + let stderr = ''; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + let settled = false; + let timedOut = false; + let killTimer: ReturnType | undefined; + let killEscalate: ReturnType | undefined; + let hardDeadline: ReturnType | undefined; + const timeoutMs = config.requestTimeoutMs; + const timeoutError = + timeoutMs !== undefined && timeoutMs > 0 + ? new Error( + `grok CLI timed out after ${ + timeoutMs >= 60_000 + ? `${Math.round(timeoutMs / 60_000)}m` + : `${Math.round(timeoutMs / 1_000)}s` + }. Increase --timeout or omit it to disable the request timeout.`, + ) + : undefined; + + const clearKillTimers = () => { + if (killTimer !== undefined) clearTimeout(killTimer); + if (killEscalate !== undefined) clearTimeout(killEscalate); + if (hardDeadline !== undefined) clearTimeout(hardDeadline); + }; + + const rejectOnce = (error: Error) => { + if (settled) return; + settled = true; + clearKillTimers(); + reject(error); + }; + + const resolveOnce = (value: string) => { + if (settled) return; + settled = true; + clearKillTimers(); + resolve(value); + }; + + const KILL_GRACE_MS = 2000; + if (timeoutMs !== undefined && timeoutMs > 0 && timeoutError) { + killTimer = setTimeout(() => { + timedOut = true; + killChildTree(child); + killEscalate = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch { + // Process may have already exited. + } + hardDeadline = setTimeout(() => { + rejectOnce(timeoutError); + }, KILL_GRACE_MS); + }, KILL_GRACE_MS); + }, timeoutMs); + } + + child.stdout.on('data', (chunk: Buffer) => { + const chunkStr = stdoutDecoder.write(chunk); + stdout += chunkStr; + options?.onChunk?.(stdout.length); + }); + + child.stderr.on('data', (chunk: Buffer) => { + stderr += stderrDecoder.write(chunk); + }); + + child.on('close', (code) => { + const alreadySettled = settled; + lifecycle.closed = true; + stdout += stdoutDecoder.end(); + stderr += stderrDecoder.end(); + verboseLog( + `Process exited with code ${code} after ${((Date.now() - startTime) / 1000).toFixed(1)}s`, + ); + + if (timedOut && timeoutError) { + rejectOnce(timeoutError); + } else if (code !== 0) { + const details = stderr.trim() || stdout.trim(); + rejectOnce(new Error(`grok CLI exited with code ${code}: ${details}`)); + } else { + try { + resolveOnce(parseGrokOutput(stdout)); + } catch (err) { + rejectOnce(err instanceof Error ? err : new Error(String(err))); + } + } + + if (alreadySettled) { + void fs.rm(cwd, { recursive: true, force: true }).catch(() => undefined); + } + }); + + child.on('error', (err) => { + lifecycle.closed = true; + rejectOnce(new Error(`Failed to spawn grok CLI: ${err.message}`)); + }); + }); +} diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts index 9e5988550..b8af62099 100644 --- a/gitnexus/src/core/wiki/llm-client.ts +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -18,6 +18,7 @@ export type LLMProvider = | 'claude' | 'codex' | 'opencode' + | 'grok' | 'minimax'; export const MINIMAX_OPENAI_BASE_URLS = { @@ -112,12 +113,15 @@ export async function resolveLLMConfig(overrides?: Partial): Promise< ? savedConfig.codexModel : savedProvider === 'opencode' ? savedConfig.opencodeModel - : undefined; + : savedProvider === 'grok' + ? savedConfig.grokModel + : undefined; const localProvider = savedProvider === 'cursor' || savedProvider === 'claude' || savedProvider === 'codex' || - savedProvider === 'opencode'; + savedProvider === 'opencode' || + savedProvider === 'grok'; const apiKey = overrides?.apiKey || diff --git a/gitnexus/src/core/wiki/local-cli-client.ts b/gitnexus/src/core/wiki/local-cli-client.ts index 4edfe57ec..30ffa86ac 100644 --- a/gitnexus/src/core/wiki/local-cli-client.ts +++ b/gitnexus/src/core/wiki/local-cli-client.ts @@ -29,12 +29,14 @@ const COMMANDS: Record = { opencode: 'opencode', }; -interface LocalCommand { +export interface ResolvedCliCommand { displayName: string; command: string; argsPrefix: string[]; } +type LocalCommand = ResolvedCliCommand; + function killChildTree(child: import('child_process').ChildProcess): void { if (process.platform === 'win32' && child.pid !== undefined) { try { @@ -387,10 +389,33 @@ function getDetectedCommand(provider: LocalAgentProvider): LocalCommand | null { return cachedCommands.get(provider) ?? null; } +/** + * Resolve a PATH command for spawn/execFile without a shell. + * On Windows, npm shims are `.cmd` files that Node cannot execFile/spawn + * directly — prefer a native `.exe` from `where.exe`, else `cmd.exe /d /s /c`. + */ +export function resolveWindowsCliCommand(displayName: string): ResolvedCliCommand { + if (process.platform !== 'win32') { + return { displayName, command: displayName, argsPrefix: [] }; + } + + const located = findWindowsCommand(`${displayName}.cmd`) || findWindowsCommand(displayName); + if (located && /\.exe$/i.test(located)) { + return { displayName, command: located, argsPrefix: [] }; + } + + // Last-resort fallback for installations that only expose a .cmd shim. + return { + displayName, + command: process.env.ComSpec || 'cmd.exe', + argsPrefix: ['/d', '/s', '/c', displayName], + }; +} + function resolveLocalCommand(provider: LocalAgentProvider): LocalCommand { const displayName = COMMANDS[provider]; if (process.platform !== 'win32') { - return { displayName, command: displayName, argsPrefix: [] }; + return resolveWindowsCliCommand(displayName); } const npmBin = findWindowsCommand(`${displayName}.cmd`) || findWindowsCommand(displayName); @@ -418,14 +443,7 @@ function resolveLocalCommand(provider: LocalAgentProvider): LocalCommand { } } - // Last-resort fallback for non-npm Windows installations that only expose a - // .cmd shim. Prompts are passed via stdin, so repo content is not placed on - // the command line. - return { - displayName, - command: process.env.ComSpec || 'cmd.exe', - argsPrefix: ['/d', '/s', '/c', displayName], - }; + return resolveWindowsCliCommand(displayName); } function findWindowsCommand(command: string): string | null { diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index ce2e594cf..b223d5f69 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -1582,11 +1582,13 @@ export interface CLIConfig { | 'claude' | 'codex' | 'opencode' + | 'grok' | 'minimax'; cursorModel?: string; claudeModel?: string; codexModel?: string; opencodeModel?: string; + grokModel?: string; /** Azure api-version query param (e.g. '2024-10-21'). Only used when provider is 'azure'. */ apiVersion?: string; /** Set true when the deployment is a reasoning model (o1, o3, o4-mini). Auto-detected for OpenAI; must be set for Azure deployments. */ diff --git a/gitnexus/test/integration/cli-e2e.test.ts b/gitnexus/test/integration/cli-e2e.test.ts index 957a3f4e7..d44b5c4a8 100644 --- a/gitnexus/test/integration/cli-e2e.test.ts +++ b/gitnexus/test/integration/cli-e2e.test.ts @@ -1150,6 +1150,7 @@ describe('CLI end-to-end', () => { expect(result.stdout).toContain('--provider '); expect(result.stdout).toContain('claude'); expect(result.stdout).toContain('codex'); + expect(result.stdout).toContain('grok'); expect(result.stdout).toContain('--review'); expect(result.stdout).toContain('-v, --verbose'); expect(result.stdout).toContain('--model '); @@ -1230,6 +1231,14 @@ describe('CLI end-to-end', () => { expect(combined).not.toMatch(/API key:/); }); + it('wiki --provider grok without API key does not prompt for key in non-TTY', () => { + const result = runCliRaw(['wiki', MINI_REPO, '--provider', 'grok'], repoRoot, 15000); + if (result.status === null) return; + + const combined = result.stdout + result.stderr; + expect(combined).not.toMatch(/API key:/); + }); + it('wiki --help includes --verbose flag description', () => { const result = runCliRaw(['wiki', '--help'], repoRoot); if (result.status === null) return; diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index b0c276bc2..a35541d18 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -249,6 +249,7 @@ describe('CLI help surface', () => { expect(result.stdout).toContain('--provider '); expect(result.stdout).toContain('claude'); expect(result.stdout).toContain('codex'); + expect(result.stdout).toContain('grok'); expect(result.stdout).toContain('--review'); expect(result.stdout).toContain('-v, --verbose'); expect(result.stdout).toContain('--model '); diff --git a/gitnexus/test/unit/local-cli-subprocess.test.ts b/gitnexus/test/unit/local-cli-subprocess.test.ts index ce0e0a2f8..9e46f75f4 100644 --- a/gitnexus/test/unit/local-cli-subprocess.test.ts +++ b/gitnexus/test/unit/local-cli-subprocess.test.ts @@ -14,6 +14,7 @@ function makeFakeChild(opts?: { stdout?: string; stderr?: string; stdinEndBehavior?: 'normal' | 'epipe'; + closeOnSpawn?: boolean; }) { const child = new EventEmitter() as any; child.stdout = new EventEmitter(); @@ -23,8 +24,7 @@ function makeFakeChild(opts?: { child.kill = vi.fn(); let stdinContent = ''; - child.stdin.end = vi.fn((data?: string) => { - if (data) stdinContent += data; + const finish = () => { queueMicrotask(() => { if (opts?.stdinEndBehavior === 'epipe') { child.stdin.emit('error', new Error('write EPIPE')); @@ -37,9 +37,14 @@ function makeFakeChild(opts?: { } child.emit('close', opts?.exitCode ?? 0); }); + }; + + child.stdin.end = vi.fn((data?: string) => { + if (data) stdinContent += data; + if (!opts?.closeOnSpawn) finish(); }); - return { child, getStdin: () => stdinContent }; + return { child, getStdin: () => stdinContent, complete: finish }; } // ─── Claude CLI argv contract ───────────────────────────────────────── @@ -623,3 +628,614 @@ describe('Codex CLI flag contract snapshot', () => { expect(args[args.length - 1]).toBe('-'); }); }); + +// ─── Grok CLI subprocess contract ───────────────────────────────────── + +describe('Grok CLI subprocess contract', () => { + let spawnSpy: ReturnType; + let fakeChild: ReturnType; + + function spawnGrokChild(onSpawn?: (...args: unknown[]) => void) { + spawnSpy = vi.fn((...args: unknown[]) => { + onSpawn?.(...args); + fakeChild.complete(); + return fakeChild.child; + }); + } + + function grokFake( + opts: Parameters[0] = { + stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }), + }, + onSpawn?: (...args: unknown[]) => void, + ) { + fakeChild = makeFakeChild({ ...opts, closeOnSpawn: true }); + spawnGrokChild(onSpawn); + } + + beforeEach(() => { + vi.resetModules(); + grokFake(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + async function loadGrokClient() { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockReturnValue('grok 1.0.5'), + execSync: vi.fn().mockReturnValue('grok 1.0.5'), + spawn: spawnSpy, + })); + return import('../../src/core/wiki/grok-client.js'); + } + + it('detectGrokCLI returns grok when grok --version succeeds and caches the result', async () => { + const execFileSync = vi.fn().mockReturnValue('grok 1.0.5'); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync, + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { detectGrokCLI } = await import('../../src/core/wiki/grok-client.js'); + + expect(detectGrokCLI()).toBe('grok'); + expect(detectGrokCLI()).toBe('grok'); + const versionCalls = execFileSync.mock.calls.filter( + (call: unknown[]) => Array.isArray(call[1]) && (call[1] as string[]).includes('--version'), + ); + expect(versionCalls).toHaveLength(1); + }); + + it('detectGrokCLI returns null on ENOENT', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockImplementation(() => { + const err = new Error('not found') as NodeJS.ErrnoException; + err.code = 'ENOENT'; + throw err; + }), + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { detectGrokCLI } = await import('../../src/core/wiki/grok-client.js'); + + expect(detectGrokCLI()).toBeNull(); + }); + + it('spawns grok with prompt-file, json output, max-turns 15, a tool denylist, and a strict sandbox', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('user prompt', {}); + + const args = spawnSpy.mock.calls[0][1] as string[]; + expect(args).toContain('--prompt-file'); + expect(args).toContain('--output-format'); + expect(args).toContain('json'); + const turnsIdx = args.indexOf('--max-turns'); + expect(turnsIdx).toBeGreaterThanOrEqual(0); + expect(args[turnsIdx + 1]).toBe('15'); + expect(args).toContain('--no-plan'); + expect(args).toContain('--no-subagents'); + expect(args).toContain('--disable-web-search'); + expect(args).toContain('--disallowed-tools'); + const denyIdx = args.indexOf('--disallowed-tools'); + expect(args[denyIdx + 1]).toBe( + 'run_terminal_cmd,search_replace,web_search,web_fetch,spawn_subagent', + ); + expect(args).toContain('--sandbox'); + const sandboxIdx = args.indexOf('--sandbox'); + expect(args[sandboxIdx + 1]).toBe('strict'); + expect(args).not.toContain('--tools'); + expect(args).not.toContain('--yolo'); + expect(args).not.toContain('--always-approve'); + expect(args.some((arg) => arg.includes('user prompt'))).toBe(false); + }); + + it('writes system + separator + user prompt to --prompt-file, not stdin', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('user prompt', {}, 'system prompt'); + + const args = spawnSpy.mock.calls[0][1] as string[]; + const fileIdx = args.indexOf('--prompt-file'); + const promptPath = args[fileIdx + 1]; + // File is removed after the call; capture contents via spawn-time read. + // The implementation writes before spawn, so the spy can read it if we hook spawn. + expect(fakeChild.getStdin()).toBe(''); + expect(promptPath).toContain('gitnexus-wiki-grok-'); + }); + + it('writes the concatenated prompt before spawn', async () => { + let promptContents = ''; + const fs = await import('fs'); + grokFake( + { stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }) }, + (..._spawnArgs: unknown[]) => { + const args = _spawnArgs[1] as string[]; + const fileIdx = args.indexOf('--prompt-file'); + promptContents = fs.readFileSync(args[fileIdx + 1], 'utf-8'); + }, + ); + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('user prompt', {}, 'system prompt'); + + expect(promptContents).toBe('system prompt\n\n---\n\nuser prompt'); + }); + + it('passes --cwd to an empty temp dir, not a repo path', async () => { + grokFake( + { stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }) }, + (_cmd, args, opts) => { + const argv = args as string[]; + const spawnOpts = opts as { cwd?: string }; + expect(argv).toContain('--cwd'); + const cwdIdx = argv.indexOf('--cwd'); + expect(argv[cwdIdx + 1]).toContain('gitnexus-wiki-grok-'); + expect(argv[cwdIdx + 1]).toBe(spawnOpts.cwd); + }, + ); + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + }); + + it('appends --model only when model is set', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', { model: 'grok-build' }); + + const args = spawnSpy.mock.calls[0][1] as string[]; + expect(args).toContain('--model'); + expect(args).toContain('grok-build'); + }); + + it('does not include --model when model is empty', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + + const args = spawnSpy.mock.calls[0][1] as string[]; + expect(args).not.toContain('--model'); + }); + + it('parses JSON text field as the LLM content', async () => { + const { callGrokLLM } = await loadGrokClient(); + + const result = await callGrokLLM('prompt', {}); + expect(result.content).toBe('wiki page'); + }); + + it('rejects with exit code and stderr on non-zero exit', async () => { + grokFake({ exitCode: 1, stderr: 'auth required' }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI exited with code 1: auth required', + ); + }); + + it('rejects when grok returns a JSON error object', async () => { + grokFake({ + stdout: JSON.stringify({ type: 'error', message: 'session failed' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow('session failed'); + }); + + it('rejects when JSON text is empty', async () => { + grokFake({ stdout: JSON.stringify({ text: ' ' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow('grok CLI returned empty text'); + }); + + it('rejects incomplete stopReason even when text is present', async () => { + grokFake({ + stdout: JSON.stringify({ text: 'cut off mid-sent', stopReason: 'max_tokens' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/stopReason=max_tokens/); + }); + + it('rejects PascalCase incomplete stopReason', async () => { + grokFake({ + stdout: JSON.stringify({ text: 'partial', stopReason: 'MaxTokens' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/stopReason=MaxTokens/); + }); + + it('accepts end_turn stopReason with text', async () => { + grokFake({ + stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).resolves.toEqual({ content: 'wiki page' }); + }); + + it('rejects omitted stopReason even when text is non-empty', async () => { + grokFake({ stdout: JSON.stringify({ text: 'wiki page' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI JSON is missing stopReason=end_turn', + ); + }); + + it('rejects null stopReason even when text is non-empty', async () => { + grokFake({ stdout: JSON.stringify({ text: 'wiki page', stopReason: null }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI JSON is missing stopReason=end_turn', + ); + }); + + it('rejects empty stopReason even when text is non-empty', async () => { + grokFake({ stdout: JSON.stringify({ text: 'wiki page', stopReason: '' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI JSON is missing stopReason=end_turn', + ); + }); + + it('rejects empty stdout with a dedicated message', async () => { + grokFake({ stdout: ' ' }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow('grok CLI returned empty output'); + }); + + it('rejects non-JSON stdout with an excerpt', async () => { + grokFake({ stdout: 'Update available\nnot-json' }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/non-JSON output:.*Update available/s); + }); + + it('rejects JSON without a text field with an excerpt', async () => { + grokFake({ stdout: JSON.stringify({ sessionId: 'abc' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/no text field:.*"sessionId"/s); + }); + + it('removes the temp prompt file and cwd after success', async () => { + const fs = await import('fs'); + let promptPath = ''; + let cwdPath = ''; + grokFake( + { stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }) }, + (_cmd, args) => { + const argv = args as string[]; + const fileIdx = argv.indexOf('--prompt-file'); + promptPath = argv[fileIdx + 1]; + cwdPath = argv[argv.indexOf('--cwd') + 1]; + expect(fs.existsSync(promptPath)).toBe(true); + }, + ); + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + + expect(fs.existsSync(promptPath)).toBe(false); + expect(fs.existsSync(cwdPath)).toBe(false); + }); + + it('removes the temp dir after failure', async () => { + const fs = await import('fs'); + let cwdPath = ''; + grokFake({ exitCode: 1, stderr: 'nope' }, (_cmd, args) => { + const argv = args as string[]; + cwdPath = argv[argv.indexOf('--cwd') + 1]; + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/exited with code 1/); + expect(fs.existsSync(cwdPath)).toBe(false); + }); + + it('throws when grok CLI is not on PATH', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockImplementation(() => { + const err = new Error('not found') as NodeJS.ErrnoException; + err.code = 'ENOENT'; + throw err; + }), + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { callGrokLLM } = await import('../../src/core/wiki/grok-client.js'); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/Grok CLI not found/); + }); + + it('sets CI=1 and windowsHide=true in spawn options', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + + const spawnOpts = spawnSpy.mock.calls[0][2]; + expect(spawnOpts.env.CI).toBe('1'); + expect(spawnOpts.windowsHide).toBe(true); + }); + + it('on Windows detects grok and spawns via cmd.exe /d /s /c, not a .cmd path', async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, 'platform', { value: 'win32' }); + + try { + const execFileSync = vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'where.exe') return 'C:\\Users\\me\\AppData\\Roaming\\npm\\grok.cmd\r\n'; + return 'grok 1.0.5'; + }); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync, + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { detectGrokCLI, callGrokLLM } = await import('../../src/core/wiki/grok-client.js'); + + expect(detectGrokCLI()).toBe('grok'); + await callGrokLLM('prompt', {}); + + const spawnCmd = spawnSpy.mock.calls[0][0] as string; + const spawnArgs = spawnSpy.mock.calls[0][1] as string[]; + expect(spawnCmd.toLowerCase()).not.toMatch(/\.cmd$/); + expect(spawnCmd).toBe(process.env.ComSpec || 'cmd.exe'); + expect(spawnArgs.slice(0, 4)).toEqual(['/d', '/s', '/c', 'grok']); + expect(spawnArgs).toContain('--prompt-file'); + expect(spawnArgs).toContain('--sandbox'); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + } + }); + + it('on Windows uses ComSpec when set instead of cmd.exe', async () => { + const originalPlatform = process.platform; + const originalComSpec = process.env.ComSpec; + Object.defineProperty(process, 'platform', { value: 'win32' }); + process.env.ComSpec = 'C:\\Windows\\System32\\cmd.exe'; + + try { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockReturnValue('grok 1.0.5'), + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { callGrokLLM } = await import('../../src/core/wiki/grok-client.js'); + + await callGrokLLM('prompt', {}); + + expect(spawnSpy.mock.calls[0][0]).toBe('C:\\Windows\\System32\\cmd.exe'); + expect((spawnSpy.mock.calls[0][1] as string[]).slice(0, 4)).toEqual([ + '/d', + '/s', + '/c', + 'grok', + ]); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + if (originalComSpec === undefined) delete process.env.ComSpec; + else process.env.ComSpec = originalComSpec; + } + }); +}); + +describe('Grok CLI timeout', () => { + beforeEach(() => { + vi.resetModules(); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + function hangingChild() { + const child = new EventEmitter() as any; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.stdin = new EventEmitter() as any; + child.pid = 99; + child.kill = vi.fn(); + child.stdin.end = vi.fn(); + return child; + } + + async function loadGrokWithChild(child: any, execFileSyncImpl?: (...args: any[]) => unknown) { + const spawnSpy = vi.fn(() => child); + const execFileSync = + execFileSyncImpl ?? + vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'taskkill') return ''; + return 'grok 1.0.5'; + }); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync, + execSync: vi.fn(), + spawn: spawnSpy, + })); + const mod = await import('../../src/core/wiki/grok-client.js'); + return { ...mod, spawnSpy, execFileSync }; + } + + async function waitForSpawn(spawnSpy: ReturnType) { + const deadline = Date.now() + 2000; + while (spawnSpy.mock.calls.length === 0 && Date.now() < deadline) { + await Promise.resolve(); + await new Promise((r) => setImmediate(r)); + } + expect(spawnSpy).toHaveBeenCalled(); + } + + it('kills child process after requestTimeoutMs and rejects with timeout error', async () => { + const child = hangingChild(); + const { callGrokLLM, spawnSpy, execFileSync } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 5000 }); + await waitForSpawn(spawnSpy); + vi.advanceTimersByTime(5000); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 5s'); + if (process.platform === 'win32') { + const taskkillCalls = execFileSync.mock.calls.filter((c: unknown[]) => c[0] === 'taskkill'); + expect(taskkillCalls.length).toBeGreaterThan(0); + } else { + expect(child.kill).toHaveBeenCalled(); + } + }); + + it('uses taskkill /T /F /PID on Windows for process-tree kill', async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, 'platform', { value: 'win32' }); + try { + const child = hangingChild(); + child.pid = 42; + const execFileSync = vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'taskkill') return ''; + if (cmd === 'where.exe') return 'C:\\npm\\grok.cmd\n'; + return 'grok 1.0.5'; + }); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child, execFileSync); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 3000 }); + await waitForSpawn(spawnSpy); + vi.advanceTimersByTime(3000); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 3s'); + const taskkillCalls = execFileSync.mock.calls.filter((c: unknown[]) => c[0] === 'taskkill'); + expect(taskkillCalls.length).toBe(1); + expect(taskkillCalls[0][1]).toEqual(['/T', '/F', '/PID', '42']); + expect(child.kill).not.toHaveBeenCalled(); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + } + }); + + it('falls back to child.kill() when taskkill fails on Windows', async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, 'platform', { value: 'win32' }); + try { + const child = hangingChild(); + child.pid = 42; + const execFileSync = vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'taskkill') throw new Error('taskkill: process not found'); + if (cmd === 'where.exe') return 'C:\\npm\\grok.cmd\n'; + return 'grok 1.0.5'; + }); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child, execFileSync); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 2000 }); + await waitForSpawn(spawnSpy); + vi.advanceTimersByTime(2000); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 2s'); + expect(child.kill).toHaveBeenCalled(); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + } + }); + + it('does not set a kill timer when requestTimeoutMs is undefined', async () => { + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', {}); + await waitForSpawn(spawnSpy); + child.stdout.emit('data', Buffer.from(JSON.stringify({ text: 'ok', stopReason: 'end_turn' }))); + child.emit('close', 0); + await expect(promise).resolves.toEqual({ content: 'ok' }); + expect(child.kill).not.toHaveBeenCalled(); + }); + + it('ignores grok stdin so an unused pipe cannot EPIPE', async () => { + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', {}); + await waitForSpawn(spawnSpy); + const spawnOpts = spawnSpy.mock.calls[0][2] as { stdio?: unknown }; + expect(spawnOpts.stdio).toEqual(['ignore', 'pipe', 'pipe']); + expect(child.stdin.end).not.toHaveBeenCalled(); + child.stdout.emit('data', Buffer.from(JSON.stringify({ text: 'ok', stopReason: 'end_turn' }))); + child.emit('close', 0); + await expect(promise).resolves.toEqual({ content: 'ok' }); + }); + + it('does not remove the temp dir until the child closes after timeout', async () => { + const fs = await import('fs'); + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 5000 }); + await waitForSpawn(spawnSpy); + const cwd = (spawnSpy.mock.calls[0][2] as { cwd: string }).cwd; + expect(fs.existsSync(cwd)).toBe(true); + let state: 'pending' | 'ok' | 'err' = 'pending'; + void promise.then( + () => { + state = 'ok'; + }, + () => { + state = 'err'; + }, + ); + vi.advanceTimersByTime(5000); + for (let i = 0; i < 30; i++) { + await new Promise((r) => setImmediate(r)); + } + expect(state).toBe('pending'); + expect(fs.existsSync(cwd)).toBe(true); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 5s'); + expect(fs.existsSync(cwd)).toBe(false); + }); + + it('does not remove the temp dir when the hard deadline rejects without a close event', async () => { + const fs = await import('fs'); + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 5000 }); + await waitForSpawn(spawnSpy); + const cwd = (spawnSpy.mock.calls[0][2] as { cwd: string }).cwd; + expect(fs.existsSync(cwd)).toBe(true); + + vi.advanceTimersByTime(5000 + 2000 + 2000); + await expect(promise).rejects.toThrow('grok CLI timed out after 5s'); + expect(fs.existsSync(cwd)).toBe(true); + + // Late close rms via fire-and-forget fs.rm. Fake setTimeout would starve + // that I/O; switch to real timers and poll until the dir is gone. + vi.useRealTimers(); + child.emit('close', null); + const goneDeadline = Date.now() + 2000; + while (fs.existsSync(cwd) && Date.now() < goneDeadline) { + await new Promise((r) => setTimeout(r, 10)); + } + expect(fs.existsSync(cwd)).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/wiki-flags.test.ts b/gitnexus/test/unit/wiki-flags.test.ts index 69ae6762a..49f4cc08e 100644 --- a/gitnexus/test/unit/wiki-flags.test.ts +++ b/gitnexus/test/unit/wiki-flags.test.ts @@ -223,6 +223,42 @@ describe('resolveLLMConfig', () => { expect(config.model).toBe(''); }); + it('uses grokModel when provider is grok', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('../../src/storage/repo-manager.js', () => ({ + loadCLIConfig: vi.fn().mockResolvedValue({ + provider: 'grok', + grokModel: 'grok-build', + }), + })); + + const { resolveLLMConfig } = await import('../../src/core/wiki/llm-client.js'); + const config = await resolveLLMConfig({ provider: 'grok' }); + + expect(config.provider).toBe('grok'); + expect(config.model).toBe('grok-build'); + }); + + it('does not inherit HTTP model defaults for grok local provider', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('../../src/storage/repo-manager.js', () => ({ + loadCLIConfig: vi.fn().mockResolvedValue({ + provider: 'openai', + model: 'legacy-http-model', + }), + })); + + const { resolveLLMConfig } = await import('../../src/core/wiki/llm-client.js'); + const config = await resolveLLMConfig({ provider: 'grok' }); + + expect(config.provider).toBe('grok'); + expect(config.model).toBe(''); + }); + it('does not inherit HTTP model defaults for local CLI providers', async () => { vi.doMock('../../src/storage/repo-manager.js', () => ({ loadCLIConfig: vi.fn().mockResolvedValue({ @@ -344,6 +380,9 @@ describe('wikiCommand provider switch persistence', () => { ) { const saveCLIConfig = vi.fn(); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); vi.doMock('../../src/storage/git.js', () => ({ getGitRoot: vi.fn(), isGitRepo: vi.fn().mockReturnValue(true), @@ -458,6 +497,26 @@ describe('wikiCommand provider switch persistence', () => { isReasoningModel: false, }); }); + + it('saves grok model to grokModel instead of the HTTP model field', async () => { + const saveCLIConfig = await saveProviderSwitch( + { + provider: 'minimax', + apiKey: 'old-minimax-key', + baseUrl: 'https://api.minimax.io/v1', + model: 'MiniMax-M3', + }, + { + provider: 'grok', + model: 'grok-build', + }, + ); + + const saved = saveCLIConfig.mock.calls[0][0] as Record; + expect(saved.provider).toBe('grok'); + expect(saved.grokModel).toBe('grok-build'); + expect(saved.model).toBeUndefined(); + }); }); // ─── --verbose flag ────────────────────────────────────────────────── @@ -1015,6 +1074,16 @@ describe('CLI config round-trip with cursor provider', () => { expect(loaded.apiKey).toBeUndefined(); }); + it('saves and loads grok provider config correctly', async () => { + const config = { provider: 'grok', grokModel: 'grok-build' }; + await fs.writeFile(configPath, JSON.stringify(config, null, 2)); + + const loaded = JSON.parse(await fs.readFile(configPath, 'utf-8')); + expect(loaded.provider).toBe('grok'); + expect(loaded.grokModel).toBe('grok-build'); + expect(loaded.apiKey).toBeUndefined(); + }); + it('saves openai provider config with model and apiKey', async () => { const config = { provider: 'openai', @@ -1245,6 +1314,55 @@ describe('WikiGenerator invokeLLM routing', () => { expect(result.content).toBe('opencode response'); }); + it('routes to callGrokLLM when provider is grok', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + const cursorClient = await import('../../src/core/wiki/cursor-client.js'); + const localClient = await import('../../src/core/wiki/local-cli-client.js'); + const grokClient = await import('../../src/core/wiki/grok-client.js'); + const llmClient = await import('../../src/core/wiki/llm-client.js'); + + const cursorSpy = vi + .spyOn(cursorClient, 'callCursorLLM') + .mockResolvedValue({ content: 'cursor response' }); + const claudeSpy = vi + .spyOn(localClient, 'callClaudeLLM') + .mockResolvedValue({ content: 'claude response' }); + const grokSpy = vi + .spyOn(grokClient, 'callGrokLLM') + .mockResolvedValue({ content: 'grok response' }); + const openaiSpy = vi + .spyOn(llmClient, 'callLLM') + .mockResolvedValue({ content: 'openai response' }); + + const { WikiGenerator } = await import('../../src/core/wiki/generator.js'); + + const storagePath = path.join(tmpDir, 'storage'); + const wikiDir = path.join(storagePath, 'wiki'); + await fs.mkdir(wikiDir, { recursive: true }); + + const repoPath = path.join(tmpDir, 'repo'); + await fs.mkdir(repoPath, { recursive: true }); + + const generator = new WikiGenerator(repoPath, storagePath, path.join(storagePath, 'lbug'), { + apiKey: '', + baseUrl: '', + model: 'grok-build', + maxTokens: 1000, + temperature: 0, + provider: 'grok', + }); + + const result = await (generator as any).invokeLLM('test prompt', 'system prompt'); + + expect(grokSpy).toHaveBeenCalledTimes(1); + expect(claudeSpy).not.toHaveBeenCalled(); + expect(cursorSpy).not.toHaveBeenCalled(); + expect(openaiSpy).not.toHaveBeenCalled(); + expect(result.content).toBe('grok response'); + }); + it('routes to callLLM when provider is openai', async () => { const cursorClient = await import('../../src/core/wiki/cursor-client.js'); const localClient = await import('../../src/core/wiki/local-cli-client.js'); From 7c723ce7940aa59009498b2cd3876207ea505e11 Mon Sep 17 00:00:00 2001 From: Chareonwit Kunna <21arenabreakout12@gmail.com> Date: Sat, 29 Aug 2026 16:57:59 +0700 Subject: [PATCH 17/28] fix(impact): resolve repo-relative file paths via filePath (fixes #3074) (#3084) * fix(impact): resolve repo-relative file paths via filePath (fixes #3074) - resolve repo-relative paths like supabase/functions/_shared/crypto.ts via n.filePath exact + anchored ENDS WITH suffix, not just n.id/n.name - return impactedCount:null on not_found so miss cannot be read as 0/UNKNOWN safe - relax parenthesised OR-clause test to allow extra filePath terms * fix(impact): scope filePath match to File nodes (review #3084 P1) * fix(impact): make file path resolution parseable and safe * docs(pdg): align result contract fixtures with v3 * test(impact): add exact path precedence and not_found contract assertions --- gitnexus/src/mcp/local/local-backend.ts | 40 ++++++++++-- gitnexus/src/mcp/local/pdg-impact.ts | 17 ++++-- gitnexus/src/mcp/tools.ts | 2 +- ...impact-pdg-callsummary-degradation.test.ts | 6 +- gitnexus/test/unit/calltool-dispatch.test.ts | 61 ++++++++++++++++++- .../test/unit/cli-impact-pdg-format.test.ts | 4 +- .../unit/impact-pdg-compose-dedup.test.ts | 2 +- 7 files changed, 113 insertions(+), 19 deletions(-) diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index cf197c732..6451b0d9b 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -3818,7 +3818,22 @@ export class LocalBackend { } else if (isQualified) { // Parenthesised because the kind filter below is appended with AND, which // binds tighter than OR. - whereClause = `WHERE (n.id = $symName OR n.name = $symName)`; + // #3074: a repo-relative file path (e.g. "supabase/functions/_shared/crypto.ts") + // is the most natural way to name a File and is exactly what `target.filePath` + // reports, but the old clause only matched `n.id` (= "File:") or basename + // `n.name`, so the same path the graph stores never resolved. Also match the + // repo-relative `n.filePath` exactly and via an anchored suffix (segment-boundary + // "ENDS WITH $suffix" where suffix is "/"+path) so "a.ts" does not spuriously + // match "mylib/a.ts" — same anchoring used in detect_changes (#2915). + const suffix = pathSuffixOf(name); + // File-path terms must be scoped to File nodes — n.filePath is shared by + // every symbol in the file, so an unlabeled predicate would turn + // "src/actions.ts" into every symbol in that file (bot review #3084 P1). + // LadybugDB does not allow label tests in WHERE (n:File), so scope via + // id prefix — File nodes are `File:`. + whereClause = `WHERE (n.id = $symName OR n.name = $symName OR (n.id STARTS WITH $filePrefix AND (n.filePath = $symName OR n.filePath ENDS WITH $suffix)))`; + queryParams.suffix = suffix; + queryParams.filePrefix = 'File:'; } else { whereClause = `WHERE n.name = $symName`; } @@ -3909,7 +3924,7 @@ export class LocalBackend { if (rows.length === 0) return { kind: 'not_found' }; // Normalise row shape across object / tuple returns from LadybugDB. - const normalized = rows.map((r: any) => ({ + let normalized = rows.map((r: any) => ({ id: (r.id ?? r[0]) as string, name: (r.name ?? r[1]) as string, type: (r.type ?? r[2] ?? '') as string, @@ -3919,6 +3934,16 @@ export class LocalBackend { ...(include_content ? { content: (r.content ?? r[6]) as string | undefined } : {}), })); + // An exact File path wins over anchored suffix candidates. Without this, + // `lib/a.ts` and `src/lib/a.ts` both score as File candidates and turn an + // otherwise unambiguous exact target into `ambiguous` (#3084 review P2). + if (isQualified) { + const exactFiles = normalized.filter( + (candidate) => candidate.id.startsWith('File:') && candidate.filePath === name, + ); + if (exactFiles.length > 0) normalized = exactFiles; + } + // The COUNT can never legitimately be below the page it accompanies, so a // value under `normalized.length` means the count leg failed or returned an // unreadable shape. Keep the window size as the floor — reporting zero would @@ -6127,6 +6152,7 @@ export class LocalBackend { direction: params.direction, suggestion, recoverySuggestion, + undetermined: true, }); return pdgErr; } @@ -6134,7 +6160,7 @@ export class LocalBackend { error: message, target: { name: params.target }, direction: params.direction, - impactedCount: 0, + impactedCount: null, risk: 'UNKNOWN', suggestion, ...(recoverySuggestion ? { recoverySuggestion } : {}), @@ -6229,6 +6255,7 @@ export class LocalBackend { `(single-repo PDG impact). Remove them or use mode:'callgraph' for cross-repo fan-out.`, target: crossDepthTarget, direction, + undetermined: true, }); return pdgErr; } @@ -6295,12 +6322,17 @@ export class LocalBackend { error: `Target '${missing}' not found`, target: notFoundTarget, direction, + undetermined: true, }) : { error: `Target '${missing}' not found`, target: { name: target }, direction, - impactedCount: 0, + // #3074 follow-up: do not ship a normal-shaped 0/UNKNOWN blast radius + // alongside the error — it reads as a real "nothing depends on this" + // answer. Null marks UNDETERMINED (same as the ambiguous path) so a + // consumer testing `impactedCount === 0` cannot misread a miss as safe. + impactedCount: null, risk: 'UNKNOWN', }; } diff --git a/gitnexus/src/mcp/local/pdg-impact.ts b/gitnexus/src/mcp/local/pdg-impact.ts index d2de59629..4cd311be1 100644 --- a/gitnexus/src/mcp/local/pdg-impact.ts +++ b/gitnexus/src/mcp/local/pdg-impact.ts @@ -118,8 +118,10 @@ export function splitCalleeIds(raw: unknown): string[] { * Bump on any breaking change to the PDG result fields. * v2: `startLine` in the result is now 1-based display (#2380), matching the * context/query/impact tools (was 0-based). + * v3: error envelopes use `impactedCount: null` when a target cannot be resolved + * (#3074), so consumers cannot read a miss as a measured zero. */ -export const PDG_RESULT_VERSION = 2 as const; +export const PDG_RESULT_VERSION = 3 as const; /** A reachable dependence block resolved to its source statement. */ export interface PdgStatement { @@ -742,7 +744,7 @@ export interface PdgInterproceduralImpact { export interface PdgImpactBaseResult extends PdgImpactParityFields { mode: 'pdg'; /** Contract version of the mode:'pdg' impact result shape; bump on any breaking change to the PDG result fields. */ - pdgResultVersion: 2; + pdgResultVersion: 3; target: PdgImpactTarget; direction: 'upstream' | 'downstream'; impactedCount: number; @@ -815,11 +817,11 @@ export interface PdgImpactDegradedResult extends PdgImpactBaseResult { export interface PdgImpactErrorResult { mode?: 'pdg'; /** Contract version of the mode:'pdg' impact result shape; bump on any breaking change to the PDG result fields. */ - pdgResultVersion: 2; + pdgResultVersion: 3; error: string; target: PdgImpactTarget; direction: 'upstream' | 'downstream'; - impactedCount: 0; + impactedCount: number | null; risk: 'UNKNOWN'; suggestion?: string; recoverySuggestion?: string; @@ -838,6 +840,8 @@ export function makePdgImpactErrorResult(input: { mode?: 'pdg'; suggestion?: string; recoverySuggestion?: string; + /** True when analysis did not obtain a measured impact count. */ + undetermined?: boolean; }): PdgImpactErrorResult { return { ...(input.mode ? { mode: input.mode } : {}), @@ -845,7 +849,10 @@ export function makePdgImpactErrorResult(input: { error: input.error, target: input.target, direction: input.direction, - impactedCount: 0, + // #3074 follow-up + P2 review: an unmeasured PDG result must not ship a + // confident-looking 0 blast radius. null marks UNDETERMINED, so a consumer + // testing === 0 cannot misread a miss or failed query as safe. + impactedCount: input.undetermined ? null : 0, risk: 'UNKNOWN', ...(input.suggestion ? { suggestion: input.suggestion } : {}), ...(input.recoverySuggestion ? { recoverySuggestion: input.recoverySuggestion } : {}), diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index be793cbeb..53700a7a2 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -469,7 +469,7 @@ MODE (opt-in): "callgraph" (default) walks symbol→symbol edges (CALLS/IMPORTS/ STATEMENT-ANCHORED PDG SLICE: with mode:'pdg', pass "line" (1-based source line within the target symbol) to seed the dependence slice on the statement at that line and return what depends on it in affectedStatements (line + text). Inter-procedural symbols are still reported through interproceduralByDepth/pdgInterprocedural and the compatibility byDepth bucket. Without "line", pdg returns whole-symbol inter-procedural reach plus local whole-symbol PDG diagnostics. -PDG OUTPUT CONTRACT: every mode:'pdg' result (success, empty, degraded, or error) carries pdgResultVersion:2 — a stable discriminator for external consumers that bumps on any breaking change to the PDG result shape (distinct from the DB schema version). Successful PDG results include mode:'pdg', a full target envelope (id/name/type/filePath), affectedStatements, affectedStatementCount, interproceduralByDepth/pdgInterprocedural for cross-function reach, compatibility byDepth/byDepthCounts, risk:'UNKNOWN', and a note describing the unified contract. Degraded PDG results (no-layer, sub-layer-missing, unknown) keep mode:'pdg', pdgResultVersion:2, target metadata when the target resolves, risk:'UNKNOWN', note/remediation, and empty byDepth parity fields — never a false-safe zero. If depth and limit both bound the slice, truncatedByReasons reports both causes while truncatedBy remains scalar. Return-value-ascent coverage is published structurally at pdgEvidence.ascent — present iff the inter-procedural descent ran, including on an empty slice — with referencesScanned (DISTINCT callees scanned for a CALL_SUMMARY: a distinct-id tally, not a call-site count — two call sites to the same callee count once), returnFlowFound (whether the ascent fired anywhere in the slice), undecodableSummaryCount, examinedComplete (whether that scan covered every callee the index recorded a resolved id for on the visited blocks), incompleteReasons ('traversal-truncated' | 'callee-list-capped' | 'callee-ids-unrecorded'), and callSummaryLayerPresent. Read callSummaryLayerPresent FIRST: false ⇒ a pre-CALL_SUMMARY index, so {referencesScanned:N>0, returnFlowFound:false} is self-consistent and says nothing about the callees — the scan ran, but no layer existed in which a return-flow could be recorded (remedy: re-run gitnexus analyze --pdg). Branch on those fields; the note narrates the same facts in prose for humans and is not a stable contract. +PDG OUTPUT CONTRACT: every mode:'pdg' result (success, empty, degraded, or error) carries pdgResultVersion:3 — a stable discriminator for external consumers that bumps on any breaking change to the PDG result shape (distinct from the DB schema version). Successful PDG results include mode:'pdg', a full target envelope (id/name/type/filePath), affectedStatements, affectedStatementCount, interproceduralByDepth/pdgInterprocedural for cross-function reach, compatibility byDepth/byDepthCounts, risk:'UNKNOWN', and a note describing the unified contract. Degraded PDG results (no-layer, sub-layer-missing, unknown) keep mode:'pdg', pdgResultVersion:3, target metadata when the target resolves, risk:'UNKNOWN', note/remediation, and empty byDepth parity fields — never a false-safe zero. If depth and limit both bound the slice, truncatedByReasons reports both causes while truncatedBy remains scalar. Return-value-ascent coverage is published structurally at pdgEvidence.ascent — present iff the inter-procedural descent ran, including on an empty slice — with referencesScanned (DISTINCT callees scanned for a CALL_SUMMARY: a distinct-id tally, not a call-site count — two call sites to the same callee count once), returnFlowFound (whether the ascent fired anywhere in the slice), undecodableSummaryCount, examinedComplete (whether that scan covered every callee the index recorded a resolved id for on the visited blocks), incompleteReasons ('traversal-truncated' | 'callee-list-capped' | 'callee-ids-unrecorded'), and callSummaryLayerPresent. Read callSummaryLayerPresent FIRST: false ⇒ a pre-CALL_SUMMARY index, so {referencesScanned:N>0, returnFlowFound:false} is self-consistent and says nothing about the callees — the scan ran, but no layer existed in which a return-flow could be recorded (remedy: re-run gitnexus analyze --pdg). Branch on those fields; the note narrates the same facts in prose for humans and is not a stable contract. WHEN TO USE: Before making code changes — especially refactoring, renaming, or modifying shared code. Shows what would break. AFTER THIS: Review d=1 items (WILL BREAK). Use context() on high-risk symbols. diff --git a/gitnexus/test/integration/impact-pdg-callsummary-degradation.test.ts b/gitnexus/test/integration/impact-pdg-callsummary-degradation.test.ts index ee9b31c92..eb43fc926 100644 --- a/gitnexus/test/integration/impact-pdg-callsummary-degradation.test.ts +++ b/gitnexus/test/integration/impact-pdg-callsummary-degradation.test.ts @@ -17,7 +17,7 @@ * "complete" result. * * This golden asserts the EXACT degraded envelope (not just non-crash): - * - the result is still mode:'pdg' with pdgResultVersion:2 (the contract + * - the result is still mode:'pdg' with pdgResultVersion:3 (the contract * discriminator); * - the intra slice is PRESENT (CALL_SUMMARY is NOT a required sub-layer — the * index is `ready`, pdgLayer is undefined, risk is UNKNOWN, epistemic is the @@ -75,14 +75,14 @@ withTestLbugDB( }); describe('CALL_SUMMARY-absent (v3 / pre-FU-C index): the ascent is silent but the user is TOLD', () => { - it('returns the EXACT degraded envelope — mode:pdg, pdgResultVersion:2, intra slice present, risk UNKNOWN', async () => { + it('returns the EXACT degraded envelope — mode:pdg, pdgResultVersion:3, intra slice present, risk UNKNOWN', async () => { const result = await slice(); // Golden envelope: the index is `ready` (CALL_SUMMARY is NOT a required // sub-layer), so this is a real traversal result — NOT a pdgLayer // degradation early-return. The intra slice ran and risk stays UNKNOWN. expect(result).toMatchObject({ mode: 'pdg', - pdgResultVersion: 2, + pdgResultVersion: 3, risk: 'UNKNOWN', epistemic: 'pdg-intra-procedural', target: { id: 'func:fnA', name: 'fnA' }, diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index e3f1892ce..f5693f090 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -1344,12 +1344,67 @@ describe('LocalBackend.callTool', () => { await backend.callTool('context', { name: 'src/a.ts:collide', kind: 'Function' }); const parenthesised = - /WHERE \(n\.id = \$symName OR n\.name = \$symName\) AND n\.id STARTS WITH \$kindPrefix/; + /WHERE \(n\.id = \$symName OR n\.name = \$symName OR \(n\.id STARTS WITH \$filePrefix AND \(n\.filePath = \$symName OR n\.filePath ENDS WITH \$suffix\)\)\) AND n\.id STARTS WITH \$kindPrefix/; const calls = resolverCalls(); expect(calls).toHaveLength(2); expect(calls.filter((c) => parenthesised.test(c.query))).toHaveLength(2); }); + it('exact File path wins over suffixed matches during qualified resolution (#3084 review P2)', async () => { + (executeParameterized as any).mockImplementation(async (_repo: string, query: string) => { + if (query.startsWith('MATCH (n)')) { + return [ + { + id: 'File:src/lib/a.ts', + name: 'a.ts', + filePath: 'src/lib/a.ts', + kind: 'File', + total_hits: 1, + }, + { + id: 'File:lib/a.ts', + name: 'a.ts', + filePath: 'lib/a.ts', + kind: 'File', + total_hits: 1, + }, + ]; + } + return [{ total: 2 }]; + }); + + const result = await backend.callTool('context', { name: 'lib/a.ts' }); + expect(result).toMatchObject({ + status: 'found', + symbol: { + filePath: 'lib/a.ts', + uid: 'File:lib/a.ts', + }, + }); + }); + + it('not_found impact queries return impactedCount null and risk UNKNOWN across modes (#3074 / #3084 review)', async () => { + (executeParameterized as any).mockImplementation(async () => []); + + const cgResult = await backend.callTool('impact', { target: 'nonexistent_target_xyz' }); + expect(cgResult).toMatchObject({ + error: "Target 'nonexistent_target_xyz' not found", + impactedCount: null, + risk: 'UNKNOWN', + }); + + const pdgResult = await backend.callTool('impact', { + target: 'nonexistent_target_xyz', + mode: 'pdg', + }); + expect(pdgResult).toMatchObject({ + error: "Target 'nonexistent_target_xyz' not found", + impactedCount: null, + risk: 'UNKNOWN', + pdgResultVersion: 3, + }); + }); + it('retries UNFILTERED when the kind hint matches no label prefix (#2787 review F5)', async () => { // `kind` is a free-form string on the tool schema. A miscased or // repo-absent kind must not turn a real name into `not_found` — the @@ -3241,7 +3296,7 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { expect(result.mode).toBe('pdg'); expect(result.target).toEqual({ name: 'missingSymbol' }); expect(result.direction).toBe('upstream'); - expect(result.impactedCount).toBe(0); + expect(result.impactedCount).toBeNull(); expect(result.risk).toBe('UNKNOWN'); }); @@ -3257,7 +3312,7 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { expect(result.mode).toBe('pdg'); expect(result.target).toEqual({ name: 'main' }); expect(result.direction).toBe('downstream'); - expect(result.impactedCount).toBe(0); + expect(result.impactedCount).toBeNull(); expect(result.risk).toBe('UNKNOWN'); expect(result.suggestion).toMatch(/context/); implSpy.mockRestore(); diff --git a/gitnexus/test/unit/cli-impact-pdg-format.test.ts b/gitnexus/test/unit/cli-impact-pdg-format.test.ts index cb17487a5..9e9f4870f 100644 --- a/gitnexus/test/unit/cli-impact-pdg-format.test.ts +++ b/gitnexus/test/unit/cli-impact-pdg-format.test.ts @@ -38,7 +38,7 @@ function pdgFindings(overrides: Record = {}): Record { // The PDG result family advertises a contract version (FIX #2) so external // MCP/agent consumers can version against future shape evolution. It is a // mode:'pdg'-only field — never on the default callgraph result. - expect(pdgFindings()).toMatchObject({ mode: 'pdg', pdgResultVersion: 2 }); + expect(pdgFindings()).toMatchObject({ mode: 'pdg', pdgResultVersion: 3 }); }); it('surfaces ambiguous-projection and unresolved block counts honestly', () => { diff --git a/gitnexus/test/unit/impact-pdg-compose-dedup.test.ts b/gitnexus/test/unit/impact-pdg-compose-dedup.test.ts index ecf1ce9e5..a3c92f4e7 100644 --- a/gitnexus/test/unit/impact-pdg-compose-dedup.test.ts +++ b/gitnexus/test/unit/impact-pdg-compose-dedup.test.ts @@ -36,7 +36,7 @@ const local = ( impactedCount: number, ): PdgImpactSuccessResult => ({ mode: 'pdg', - pdgResultVersion: 2, + pdgResultVersion: 3, target: { id: 'T', name: 'criterion', type: 'Function', filePath: 'src/a.ts' }, direction: 'downstream', risk: 'UNKNOWN', From bf7dcf98ca6a41a4fe21f85fead193020bbf3790 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Sat, 29 Aug 2026 11:40:56 +0100 Subject: [PATCH 18/28] feat(analyze): add incremental watch mode (#3072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(analyze): add incremental watch mode * fix(watch): harden control file reads * fix(watch): contain refresh errors and bound reads * fix(watch): stream strict control file reads * fix(watch): harden refresh recovery and lifecycle * fix(watch): report ignored repository defaults * fix(analyze): preserve signal exit semantics * style(analyze): format signal exit helper * test(config): exercise descriptor growth guard * test(watch): await source event before rename * fix(watch): keep live-index retries honest and ignore analyzer writes Hold retry backoff when events merge, stop only after a live-index mutation, skip .gitnexus self-writes, and reject the remaining one-shot watch flags. Export impact-risk scoring from gitnexus-shared so consumers can share the same scale. Co-authored-by: Cursor * fix(watch): contain queue edge cases after review Preserve overflow-only refreshes, contain synchronous refresh failures, and mark successful atomic publication before later operations can fail. Co-authored-by: Cursor --------- Co-authored-by: Gergo Magyar Co-authored-by: Cursor Co-authored-by: Gergő Magyar --- .claude/skills/gitnexus-cli/SKILL.md | 6 +- README.md | 23 + .../skills/gitnexus-cli/SKILL.md | 6 +- gitnexus-shared/src/impact-risk.ts | 129 +++++ gitnexus-shared/src/index.ts | 11 + gitnexus/README.md | 26 + gitnexus/package-lock.json | 77 +-- gitnexus/package.json | 1 + gitnexus/scripts/cross-platform-shard.ts | 6 +- gitnexus/scripts/cross-platform-tests.ts | 3 +- gitnexus/skills/gitnexus-cli.md | 6 +- gitnexus/src/cli/analyze-config.ts | 21 +- gitnexus/src/cli/analyze-options.ts | 4 + gitnexus/src/cli/analyze.ts | 55 +- gitnexus/src/cli/help-i18n.ts | 2 + gitnexus/src/cli/i18n/en.ts | 2 + gitnexus/src/cli/i18n/zh-CN.ts | 2 + gitnexus/src/cli/index.ts | 11 +- gitnexus/src/cli/watch-queue.ts | 184 +++++++ gitnexus/src/cli/watch.ts | 501 ++++++++++++++++++ gitnexus/src/config/ignore-service.ts | 47 +- gitnexus/src/config/repo-control-file.ts | 117 ++++ .../ingestion/pipeline-phases/parse-impl.ts | 9 + .../core/ingestion/pipeline-phases/parse.ts | 2 + gitnexus/src/core/ingestion/pipeline.ts | 13 +- gitnexus/src/core/run-analyze.ts | 97 +++- gitnexus/src/storage/parse-cache.ts | 12 +- gitnexus/src/types/pipeline.ts | 2 + .../integration/analyze-atomic-swap.test.ts | 112 +++- gitnexus/test/integration/cli-e2e.test.ts | 192 +++++++ .../test/integration/watch-filesystem.test.ts | 230 ++++++++ gitnexus/test/unit/analyze-config.test.ts | 80 ++- .../test/unit/analyze-heap-respawn.test.ts | 9 + .../test/unit/cross-platform-shard.test.ts | 4 +- .../unit/incremental-orchestration.test.ts | 7 + .../test/unit/incremental-parse-cache.test.ts | 27 + .../test/unit/watch-failure-policy.test.ts | 29 + gitnexus/test/unit/watch-paths.test.ts | 180 +++++++ gitnexus/test/unit/watch-queue.test.ts | 348 ++++++++++++ 39 files changed, 2504 insertions(+), 89 deletions(-) create mode 100644 gitnexus-shared/src/impact-risk.ts create mode 100644 gitnexus/src/cli/watch-queue.ts create mode 100644 gitnexus/src/cli/watch.ts create mode 100644 gitnexus/src/config/repo-control-file.ts create mode 100644 gitnexus/test/integration/watch-filesystem.test.ts create mode 100644 gitnexus/test/unit/watch-failure-policy.test.ts create mode 100644 gitnexus/test/unit/watch-paths.test.ts create mode 100644 gitnexus/test/unit/watch-queue.test.ts diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index e993c38f8..1b1f733b6 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -21,6 +21,8 @@ Run from the project root. This parses all source files, builds the knowledge gr | Flag | Effect | | -------------- | ---------------------------------------------------------------- | +| `--watch` | Keep a Git repository index current with serialized refreshes | +| `--debounce ` | Watch quiet period before refresh (default: 300 ms) | | `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | @@ -28,6 +30,8 @@ Run from the project root. This parses all source files, builds the knowledge gr **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. + ### status — Check index freshness ```bash @@ -86,5 +90,5 @@ Lists all repositories registered in `~/.gitnexus/registry.json`. The MCP `list_ ## Troubleshooting - **"Not inside a git repository"**: Run from a directory inside a git repo -- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server +- **Index is stale after re-analyzing**: Wait for the next MCP tool call to reopen the published index; this normally takes no more than five seconds - **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding diff --git a/README.md b/README.md index e89ddcdb3..5e0536a48 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,7 @@ Everyday commands: ```bash gitnexus setup # Configure MCP for detected editors (one-time; -c to select) gitnexus analyze [path] # Index a repository (or update a stale index) +gitnexus analyze [path] --watch # Watch local files and serialize incremental refreshes gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus eval-server # Start lightweight evaluation HTTP tools (loopback by default) @@ -396,6 +397,28 @@ gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks You can also query the graph directly from the terminal — `gitnexus query`, `context`, `impact`, `trace`, `cypher`, `detect-changes`, and `check` mirror the MCP tools of the same names, and `gitnexus doctor` prints runtime platform capabilities. +`gitnexus analyze --watch` requires a Git repository. It runs one initial +analysis, then debounces scanner-admitted working-tree changes for 300 ms by +default and applies serialized incremental refreshes. Events arriving during a +refresh remain queued, and retryable failures retain the same batch with bounded +backoff. Invalid `.gitnexusrc` or ignore-file reloads pause ordinary refreshes +until the control file is fixed. Stop the watcher with Ctrl+C. + +Watch mode accepts `--debounce`, `--workers`, `--worker-timeout`, +`--max-file-size`, `--branch`, `--pdg`, `--name`, `--allow-duplicate-name`, and +`--verbose`. Explicit one-shot options such as `--force`, `--repair-fts`, +embedding flags, `--skills`, `--self-commit`, `--index-only`, and `--skip-git` +are rejected. Unsupported defaults from `.gitnexusrc` are ignored with a +warning rather than making an otherwise valid repository unwatchable. + +POSIX requests clone-first copy-and-swap publication when the live index has no +orphan sidecars. Windows and sidecar fallback runs update in place: failures +known to occur before writes are retried, while a failure that may have mutated +the live index stops the watcher. Watch mode does not pull remotes. Running MCP +and `serve` processes reopen a newly published index automatically; MCP observes +the replacement on its next tool call, typically within five seconds, so no +restart is required. +
Authenticated eval-server binding diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index e993c38f8..1b1f733b6 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -21,6 +21,8 @@ Run from the project root. This parses all source files, builds the knowledge gr | Flag | Effect | | -------------- | ---------------------------------------------------------------- | +| `--watch` | Keep a Git repository index current with serialized refreshes | +| `--debounce ` | Watch quiet period before refresh (default: 300 ms) | | `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | @@ -28,6 +30,8 @@ Run from the project root. This parses all source files, builds the knowledge gr **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. + ### status — Check index freshness ```bash @@ -86,5 +90,5 @@ Lists all repositories registered in `~/.gitnexus/registry.json`. The MCP `list_ ## Troubleshooting - **"Not inside a git repository"**: Run from a directory inside a git repo -- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server +- **Index is stale after re-analyzing**: Wait for the next MCP tool call to reopen the published index; this normally takes no more than five seconds - **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding diff --git a/gitnexus-shared/src/impact-risk.ts b/gitnexus-shared/src/impact-risk.ts new file mode 100644 index 000000000..6c18614f6 --- /dev/null +++ b/gitnexus-shared/src/impact-risk.ts @@ -0,0 +1,129 @@ +export type ImpactRisk = 'LOW' | 'MEDIUM' | 'HIGH' | 'CRITICAL' | 'UNKNOWN'; + +export type ImpactRiskAxis = 'processes' | 'modules'; + +export type UnusedImpactRiskReason = + | 'file-nodes-have-no-process-or-community-membership' + | 'enrichment-skipped' + | 'enrichment-budget-exhausted' + | 'enrichment-query-failed'; + +export interface UnusedImpactRiskAxis { + axis: ImpactRiskAxis; + reason: UnusedImpactRiskReason; +} + +export interface ImpactRiskInput { + direction: 'upstream' | 'downstream'; + directCount: number; + processCount: number; + moduleCount: number; + impactedCount: number; + unusedAxes?: readonly UnusedImpactRiskAxis[]; +} + +export interface ImpactRiskResult { + risk: ImpactRisk; + riskSharedAxes: ImpactRisk; + riskScale: { + comparableAcrossKinds: boolean; + unusedAxes: readonly UnusedImpactRiskAxis[]; + }; +} + +function score( + input: Pick< + ImpactRiskInput, + 'direction' | 'directCount' | 'processCount' | 'moduleCount' | 'impactedCount' + >, +): ImpactRisk { + const { direction, directCount, processCount, moduleCount, impactedCount } = input; + + if (direction === 'upstream' && impactedCount === 0) return 'UNKNOWN'; + if (directCount >= 30 || processCount >= 5 || moduleCount >= 5 || impactedCount >= 200) { + return 'CRITICAL'; + } + if (directCount >= 15 || processCount >= 3 || moduleCount >= 3 || impactedCount >= 100) { + return 'HIGH'; + } + if (directCount >= 5 || impactedCount >= 30) return 'MEDIUM'; + return 'LOW'; +} + +function countsWithUnusedAxesZeroed( + input: ImpactRiskInput, +): Pick< + ImpactRiskInput, + 'direction' | 'directCount' | 'processCount' | 'moduleCount' | 'impactedCount' +> { + let processCount = input.processCount; + let moduleCount = input.moduleCount; + for (const unused of input.unusedAxes ?? []) { + if (unused.axis === 'processes') processCount = 0; + if (unused.axis === 'modules') moduleCount = 0; + } + return { + direction: input.direction, + directCount: input.directCount, + processCount, + moduleCount, + impactedCount: input.impactedCount, + }; +} + +/** Map walk outcomes to unused process/module axes so comparability matches what was sampled. */ +export function unusedAxesForImpactWalk(input: { + isFileTarget: boolean; + skipEnrichment: boolean; + maxChunks: number; + processQueryFailed: boolean; + moduleQueryFailed: boolean; + /** When 0, a zero chunk budget is not an unused-axis event — there was nothing to enrich. */ + impactedCount?: number; +}): UnusedImpactRiskAxis[] { + if (input.isFileTarget) { + return [ + { + axis: 'processes', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + { + axis: 'modules', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + ]; + } + if (input.skipEnrichment) { + return [ + { axis: 'processes', reason: 'enrichment-skipped' }, + { axis: 'modules', reason: 'enrichment-skipped' }, + ]; + } + if (input.maxChunks === 0 && (input.impactedCount ?? 1) > 0) { + return [ + { axis: 'processes', reason: 'enrichment-budget-exhausted' }, + { axis: 'modules', reason: 'enrichment-budget-exhausted' }, + ]; + } + const unused: UnusedImpactRiskAxis[] = []; + if (input.processQueryFailed) { + unused.push({ axis: 'processes', reason: 'enrichment-query-failed' }); + } + if (input.moduleQueryFailed) { + unused.push({ axis: 'modules', reason: 'enrichment-query-failed' }); + } + return unused; +} + +export function scoreImpactRisk(input: ImpactRiskInput): ImpactRiskResult { + const unusedAxes = input.unusedAxes ?? []; + + return { + risk: score(countsWithUnusedAxesZeroed(input)), + riskSharedAxes: score({ ...input, processCount: 0, moduleCount: 0 }), + riskScale: { + comparableAcrossKinds: unusedAxes.length === 0, + unusedAxes, + }, + }; +} diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index 13c2eac5a..9857a60cc 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -25,6 +25,17 @@ export { } from './language-detection.js'; export type { MroStrategy } from './mro-strategy.js'; +// Impact risk scoring +export { scoreImpactRisk, unusedAxesForImpactWalk } from './impact-risk.js'; +export type { + ImpactRisk, + ImpactRiskAxis, + ImpactRiskInput, + ImpactRiskResult, + UnusedImpactRiskAxis, + UnusedImpactRiskReason, +} from './impact-risk.js'; + // Pipeline progress export type { PipelinePhase, PipelineProgress } from './pipeline.js'; diff --git a/gitnexus/README.md b/gitnexus/README.md index 8013026f2..93b2dd354 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -234,6 +234,7 @@ Your AI agent gets **17 tools** (15 per-repo + 2 group) automatically: gitnexus setup # Configure MCP for detected editors (one-time; use -c to select) gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks (add --force to apply) gitnexus analyze [path] # Index a repository (or update stale index) +gitnexus analyze [path] --watch # Watch local files and serialize incremental refreshes gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild gitnexus analyze --embeddings # Enable embedding generation (slower, better search) @@ -282,6 +283,31 @@ gitnexus group status # Check staleness of repos in a group gitnexus group impact --target --repo # Cross-repo blast radius ``` +`gitnexus analyze --watch` requires a Git repository. It performs an initial +analysis and then debounces scanner-admitted working-tree changes for 300 ms by +default into serialized incremental refreshes. Events arriving during a run +remain queued, and retryable failures retain the same batch with bounded +backoff. Invalid `.gitnexusrc` or ignore-file reloads pause ordinary refreshes +until the control file is fixed. Watch refreshes update only the graph: they +intentionally skip AGENTS.md / CLAUDE.md injection and standard skill +installation. Run a one-shot `gitnexus analyze` when those generated files need +updating. Stop watch mode with Ctrl+C. + +Watch mode accepts `--debounce`, `--workers`, `--worker-timeout`, +`--max-file-size`, `--branch`, `--pdg`, `--name`, `--allow-duplicate-name`, and +`--verbose`. Explicit one-shot options such as `--force`, `--repair-fts`, +embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, +`--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git` +are rejected. Unsupported defaults from `.gitnexusrc` are ignored with a warning. + +POSIX requests clone-first copy-and-swap publication when the live index has no +orphan sidecars. Windows and sidecar fallback runs update in place: failures +known to occur before writes are retried, while a failure that may have mutated +the live index stops the watcher. Watch mode does not pull remotes. Running MCP +and `serve` processes periodically check for a newly published index and reopen +it without a restart. MCP checks are throttled to once every five seconds, so a +tool call before the next check can briefly use the previous index. + GraphQL contract matching is opt-in in the group's `group.yaml`: ```yaml diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 7188c7e1c..b8b4c8b32 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -14,6 +14,7 @@ "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", "busboy": "^1.6.0", + "chokidar": "^4.0.3", "cli-progress": "^3.12.0", "commander": "^15.0.0", "cors": "^2.8.5", @@ -826,9 +827,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -845,9 +843,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -864,9 +859,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -883,9 +875,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -902,9 +891,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -921,9 +907,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -940,9 +923,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -959,9 +939,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -978,9 +955,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1003,9 +977,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1028,9 +999,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1053,9 +1021,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1078,9 +1043,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1103,9 +1065,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1128,9 +1087,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1153,9 +1109,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2429,6 +2382,21 @@ "url": "https://github.com/chalk/chalk-template?sponsor=1" } }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -4659,6 +4627,19 @@ "rc": "cli.js" } }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 5fccc65d1..cf511b17b 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -60,6 +60,7 @@ "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", "busboy": "^1.6.0", + "chokidar": "^4.0.3", "cli-progress": "^3.12.0", "commander": "^15.0.0", "cors": "^2.8.5", diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts index 8728d8e31..2c52dd86b 100644 --- a/gitnexus/scripts/cross-platform-shard.ts +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -3,7 +3,7 @@ * * WHY THIS EXISTS. `run-cross-platform.ts` used to hand vitest the whole file * list plus `--shard=i/n`, and vitest partitions by file COUNT. Runtime on this - * suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 361 s + * suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 621 s * and `worker-pool` 221 s, while most files are under a second — so a * count-split routinely put several of the heaviest suites on one shard. That * is #2449, and this file's sibling header has documented the symptom ("the @@ -44,7 +44,9 @@ * partition depend on the very machine load it is trying to protect against. */ export const WINDOWS_WEIGHTS_SEC: Readonly> = { - 'test/integration/cli-e2e.test.ts': 361, + // Re-measured after the analyze --watch e2e landed in #3072. The previous + // 361 s entry undercharged this suite and left shard 1 close to the watchdog. + 'test/integration/cli-e2e.test.ts': 621, 'test/integration/worker-pool.test.ts': 222, 'test/unit/incremental-vector-extension-ordering.test.ts': 87, // ESTIMATE, not a measurement (#2841): this suite drives more full diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index d38672724..3f96c5c40 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -234,7 +234,7 @@ const SPAWN_CLI = [ // Cheap: measured on the Windows runner at 448 ms, 53 ms and sub-second. An // earlier attempt to register them still turned the matrix red — not from // their own cost, but because vitest sharded by file COUNT, so inserting any - // file re-partitioned the list and happened to cluster `cli-e2e` (361 s) with + // file re-partitioned the list and happened to cluster `cli-e2e` (621 s) with // `cli-limit-e2e` (75 s) on one shard. The split is weight-aware now // (`scripts/cross-platform-shard.ts`), so a cheap file can no longer move a // heavy one. @@ -273,6 +273,7 @@ const NATIVE_ADDON_SMOKE = [ // platforms (CRLF, symlinks, permissions, temp dirs) const FILESYSTEM = [ 'test/integration/filesystem-walker.test.ts', + 'test/integration/watch-filesystem.test.ts', 'test/integration/markdown-processor-crlf.test.ts', 'test/integration/ignore-and-skip-e2e.test.ts', // Pins that the bridge pairing verdict is measured before the database is diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index e993c38f8..1b1f733b6 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -21,6 +21,8 @@ Run from the project root. This parses all source files, builds the knowledge gr | Flag | Effect | | -------------- | ---------------------------------------------------------------- | +| `--watch` | Keep a Git repository index current with serialized refreshes | +| `--debounce ` | Watch quiet period before refresh (default: 300 ms) | | `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | @@ -28,6 +30,8 @@ Run from the project root. This parses all source files, builds the knowledge gr **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. + ### status — Check index freshness ```bash @@ -86,5 +90,5 @@ Lists all repositories registered in `~/.gitnexus/registry.json`. The MCP `list_ ## Troubleshooting - **"Not inside a git repository"**: Run from a directory inside a git repo -- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server +- **Index is stale after re-analyzing**: Wait for the next MCP tool call to reopen the published index; this normally takes no more than five seconds - **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding diff --git a/gitnexus/src/cli/analyze-config.ts b/gitnexus/src/cli/analyze-config.ts index 6e1afc7bb..64b7c1573 100644 --- a/gitnexus/src/cli/analyze-config.ts +++ b/gitnexus/src/cli/analyze-config.ts @@ -30,6 +30,7 @@ import fs from 'node:fs'; import path from 'node:path'; +import { readRepoControlFile } from '../config/repo-control-file.js'; import type { AnalyzeOptions } from './analyze-options.js'; export const GITNEXUS_RC_FILENAME = '.gitnexusrc'; @@ -370,7 +371,6 @@ const normalizeLevel = ( */ export function loadAnalyzeConfig(repoRoot: string): Partial | undefined { const filePath = path.join(repoRoot, GITNEXUS_RC_FILENAME); - let raw: string; try { raw = fs.readFileSync(filePath, 'utf-8'); @@ -379,6 +379,25 @@ export function loadAnalyzeConfig(repoRoot: string): Partial | u throw new GitNexusRcError(`Could not read ${GITNEXUS_RC_FILENAME}: ${(err as Error).message}`); } + return parseAnalyzeConfig(raw); +} + +/** Load `.gitnexusrc` through the strict bounded reader used by watch mode. */ +export async function loadAnalyzeConfigStrict( + repoRoot: string, +): Promise | undefined> { + let raw: string | null; + try { + raw = await readRepoControlFile(repoRoot, GITNEXUS_RC_FILENAME); + } catch (err) { + throw new GitNexusRcError(`Could not read ${GITNEXUS_RC_FILENAME}: ${(err as Error).message}`); + } + return raw === null ? undefined : parseAnalyzeConfig(raw); +} + +function parseAnalyzeConfig(rawInput: string): Partial { + let raw = rawInput; + // Strip a leading UTF-8 BOM: Node's 'utf-8' decode keeps it, and JSON.parse // then fails with a confusing "Unexpected token" on an otherwise-valid file // (#1996 tri-review). Only one leading BOM is stripped; in-string control diff --git a/gitnexus/src/cli/analyze-options.ts b/gitnexus/src/cli/analyze-options.ts index c3d1b3e8f..460218954 100644 --- a/gitnexus/src/cli/analyze-options.ts +++ b/gitnexus/src/cli/analyze-options.ts @@ -15,6 +15,10 @@ * import cycle. `analyze.ts` re-exports the type for existing importers. */ export interface AnalyzeOptions { + /** Keep this repository current with serialized incremental refreshes. */ + watch?: boolean; + /** Watch quiet period in milliseconds. */ + debounce?: string; force?: boolean; repairFts?: boolean; /** diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index e2208a3c8..54bf12d11 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -363,6 +363,7 @@ interface RespawnExit { stdout?: string; stderr?: string; message?: string; + forwardedSignal?: NodeJS.Signals; } const appendOutputTail = (tail: string, chunk: unknown): string => { @@ -395,17 +396,28 @@ const runRespawnedAnalyze = ( let stdout = ''; let stderr = ''; let settled = false; - const finish = (exit: RespawnExit): void => { - if (settled) return; - settled = true; - resolve(exit); - }; - + let forwardedSignal: NodeJS.Signals | undefined; const child = spawn(process.execPath, [...args], { stdio: ['inherit', 'pipe', 'pipe'], windowsHide: true, env, }); + const forwardSignal = (signal: NodeJS.Signals): void => { + forwardedSignal ??= signal; + if (child.exitCode === null && child.signalCode === null) child.kill(signal); + }; + const forwardSigint = () => forwardSignal('SIGINT'); + const forwardSigterm = () => forwardSignal('SIGTERM'); + const finish = (exit: RespawnExit): void => { + if (settled) return; + settled = true; + process.removeListener('SIGINT', forwardSigint); + process.removeListener('SIGTERM', forwardSigterm); + resolve({ ...exit, forwardedSignal }); + }; + + process.once('SIGINT', forwardSigint); + process.once('SIGTERM', forwardSigterm); child.stdout?.on('data', (chunk) => { stdout = appendOutputTail(stdout, chunk); @@ -548,7 +560,16 @@ export function parseMaxOldSpaceMb(nodeOptions: string): number | null { * tooling), not a deliberate per-run choice: warn and respawn with the * auto cap. Pre-#2649 this returned early and large repos then OOM'd on * whatever heap the environment happened to specify. */ -async function ensureHeap(): Promise { +export function forwardedSignalExitCode(signal: NodeJS.Signals, cleanTermination: boolean): number { + if (cleanTermination) return 0; + if (signal === 'SIGINT') return 130; + if (signal === 'SIGTERM') return 143; + return 1; +} + +export async function ensureHeap( + options: { cleanForwardedTermination?: boolean } = {}, +): Promise { // Explicit opt-out disables auto-sizing ENTIRELY — both the ambient-pin // override and the default v8-limit respawn — and is honored SILENTLY: // the operator already made the call, and stderr-sensitive consumers @@ -590,6 +611,13 @@ async function ensureHeap(): Promise { }; if (shouldBridgeRespawnProgressTty()) childEnv[RESPAWN_PROGRESS_ENV] = '1'; const childExit = await runRespawnedAnalyze(childArgs, childEnv); + if (childExit.forwardedSignal !== undefined) { + process.exitCode = forwardedSignalExitCode( + childExit.forwardedSignal, + options.cleanForwardedTermination === true, + ); + return true; + } if (childExit.status !== 0 || childExit.signal) { if (childProcessLikelyOom(childExit)) { cliError( @@ -740,6 +768,19 @@ export const analyzeCommandWithRunnerIdentity = async ( options?: AnalyzeOptions, ): Promise => analyzeCommand(inputPath, options, runnerIdentityAtBootstrap); +export async function analyzeOrWatchCommandWithRunnerIdentity( + runnerIdentityAtBootstrap: AnalyzerRunnerIdentity, + inputPath?: string, + options: AnalyzeOptions = {}, +): Promise { + if (options.watch) { + const { watchCommandWithRunnerIdentity } = await import('./watch.js'); + await watchCommandWithRunnerIdentity(runnerIdentityAtBootstrap, inputPath, options); + return; + } + await analyzeCommandWithRunnerIdentity(runnerIdentityAtBootstrap, inputPath, options); +} + const analyzeCommandImpl = async ( inputPath?: string, cliOptions?: AnalyzeOptions, diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index eeea3eeae..cd4724d70 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -72,6 +72,8 @@ const OPTION_DESCRIPTION_KEYS = { 'analyze|--embedding-batch-size ': 'help.option.analyze.embeddingBatchSize', 'analyze|--embedding-sub-batch-size ': 'help.option.analyze.embeddingSubBatchSize', 'analyze|--embedding-device ': 'help.option.analyze.embeddingDevice', + 'analyze|--watch': 'help.option.analyze.watch', + 'analyze|--debounce ': 'help.option.analyze.debounce', 'index|-f, --force': 'help.option.index.force', 'index|--allow-non-git': 'help.option.index.allowNonGit', 'mcp|--http': 'help.option.mcp.http', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index c22da5211..15e1644e7 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -217,6 +217,8 @@ export const en = { 'help.option.analyze.embeddingBatchSize': 'Number of nodes per embedding batch', 'help.option.analyze.embeddingSubBatchSize': 'Number of chunks per embedding model call', 'help.option.analyze.embeddingDevice': 'Embedding device: auto, cpu, dml, cuda, or wasm', + 'help.option.analyze.watch': 'Keep the index current with serialized incremental refreshes', + 'help.option.analyze.debounce': 'Watch quiet period before refreshing (milliseconds)', 'help.option.index.force': 'Register even if index metadata is missing (stats will be empty)', 'help.option.index.allowNonGit': 'Allow registering folders that are not Git repositories', 'help.option.port': 'Port number', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 7ef2d244b..63e290e54 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -203,6 +203,8 @@ export const zhCN = { 'help.option.analyze.embeddingBatchSize': '每个嵌入批次的节点数', 'help.option.analyze.embeddingSubBatchSize': '每次嵌入模型调用的分块数', 'help.option.analyze.embeddingDevice': '嵌入设备：auto、cpu、dml、cuda 或 wasm', + 'help.option.analyze.watch': '监视本地源文件变更并串行执行增量刷新', + 'help.option.analyze.debounce': '刷新前的静默等待时间（毫秒）', 'help.option.index.force': '即使缺少索引元数据也注册（统计为空）', 'help.option.index.allowNonGit': '允许注册非 Git 仓库文件夹', 'help.option.port': '端口号', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index a1edf0b53..8296787af 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -57,6 +57,8 @@ let dimsEnvCaptured = false; program .command('analyze [path]') .description('Index a repository (full analysis)') + .option('--watch', 'Keep the index current with serialized incremental refreshes') + .option('--debounce ', 'Watch quiet period before refreshing (default: 300 milliseconds)') .option('-f, --force', 'Force full re-index even if up to date') .option('--repair-fts', 'Repair/rebuild search FTS indexes without full re-analysis') .option( @@ -162,6 +164,11 @@ program ) .addHelpText('after', () => t('help.analyze.environment')) .hook('preAction', (thisCommand: Command) => { + const analyzeOpts = thisCommand.opts(); + if (analyzeOpts['debounce'] !== undefined && analyzeOpts['watch'] !== true) { + process.stderr.write('\n --debounce requires --watch\n\n'); + process.exit(1); + } // ONLY GITNEXUS_EMBEDDING_DIMS must be set here: schema.ts reads it at // module-load time during the lazy import('./analyze.js') below (via the // static chain analyze.ts → run-analyze.ts → schema.ts), so deferring to @@ -169,7 +176,7 @@ program // lazily at runtime (readConfig), so analyzeCommandImpl is their sole // setter — keeping them out of this hook means they fall under the impl's // env snapshot/restore and don't leak across in-process invocations. - const dimsOpt = thisCommand.opts()['embeddingDims']; + const dimsOpt = analyzeOpts['embeddingDims']; if (dimsOpt !== undefined) { // Validate + normalize BEFORE writing the env var: schema.ts throws on a // bad value at module-load, which — on the synchronous program.parse() @@ -202,7 +209,7 @@ program createAnalyzerLbugLazyAction( () => import('../core/analyzer-identity.js'), () => import('./analyze.js'), - 'analyzeCommandWithRunnerIdentity', + 'analyzeOrWatchCommandWithRunnerIdentity', import.meta.url, ), ); diff --git a/gitnexus/src/cli/watch-queue.ts b/gitnexus/src/cli/watch-queue.ts new file mode 100644 index 000000000..3f701ef50 --- /dev/null +++ b/gitnexus/src/cli/watch-queue.ts @@ -0,0 +1,184 @@ +export type WatchRefresh = (paths: readonly string[]) => Promise; +export type WatchRefreshError = (error: unknown, paths: readonly string[]) => void; + +export const WATCH_FULL_REFRESH_PATH = '*'; + +export interface WatchRefreshQueueOptions { + readonly maxWaitMs?: number; + readonly maxPendingPaths?: number; + readonly retryBaseDelayMs?: number; + readonly retryMaxDelayMs?: number; + readonly holdEventsUntilInitialRefresh?: boolean; + readonly isPriorityPath?: (filePath: string) => boolean; +} + +/** Debounces filesystem events and guarantees that refreshes never overlap. */ +export class WatchRefreshQueue { + private readonly pending = new Set(); + private readonly idleWaiters = new Set<() => void>(); + private timer: ReturnType | undefined; + private active: Promise | undefined; + private closed = false; + private initialPending = false; + private firstPendingAt: number | undefined; + private overflowed = false; + private consecutiveFailures = 0; + private retryNotBefore: number | undefined; + + constructor( + private readonly refresh: WatchRefresh, + private readonly onError: WatchRefreshError, + private readonly debounceMs: number, + private readonly options: WatchRefreshQueueOptions = {}, + ) { + this.initialPending = options.holdEventsUntilInitialRefresh === true; + } + + enqueue(filePath: string): void { + if (this.closed) return; + this.addPendingPath(filePath); + this.firstPendingAt ??= Date.now(); + if (!this.initialPending && this.active === undefined) this.schedule(); + } + + private addPendingPath(filePath: string): void { + const maxPendingPaths = this.options.maxPendingPaths ?? 1_000; + const priority = this.options.isPriorityPath?.(filePath) === true; + if (this.pending.has(filePath)) { + // A duplicate does not increase memory use or imply that paths were dropped. + } else if (this.pending.size < maxPendingPaths) { + this.pending.add(filePath); + } else { + this.overflowed = true; + if (priority) { + const evictable = [...this.pending].find( + (pendingPath) => this.options.isPriorityPath?.(pendingPath) !== true, + ); + if (evictable !== undefined) { + this.pending.delete(evictable); + this.pending.add(filePath); + } + } + } + } + + /** Run the initial refresh while still queueing events that arrive during it. */ + async runInitial(): Promise { + if (this.closed) return; + if (this.active !== undefined) throw new Error('Watch refresh is already running'); + try { + await this.runBatch([], true); + } finally { + this.initialPending = false; + if (!this.closed && this.hasPendingWork()) this.schedule(); + else this.resolveIdleWaiters(); + } + } + + async waitForIdle(): Promise { + if (this.isIdle()) return; + await new Promise((resolve) => this.idleWaiters.add(resolve)); + } + + async close(): Promise { + this.closed = true; + if (this.timer !== undefined) clearTimeout(this.timer); + this.timer = undefined; + this.pending.clear(); + this.firstPendingAt = undefined; + this.overflowed = false; + this.consecutiveFailures = 0; + this.retryNotBefore = undefined; + // A refresh rejection is already surfaced through `onError` (or through + // runInitial). Closing from that handler can race the runBatch `finally`, + // so consume the same rejection here instead of reporting it twice. + await this.active?.catch(() => {}); + this.resolveIdleWaiters(); + } + + private schedule(retryDelayMs?: number): void { + if (this.timer !== undefined) clearTimeout(this.timer); + const maxWaitMs = this.options.maxWaitMs ?? Math.max(this.debounceMs, 2_000); + const now = Date.now(); + if (retryDelayMs !== undefined) this.retryNotBefore = now + retryDelayMs; + const elapsed = this.firstPendingAt === undefined ? 0 : now - this.firstPendingAt; + const debounced = Math.max(0, Math.min(this.debounceMs, maxWaitMs - elapsed)); + // An event arriving mid-backoff merges into the pending batch but must not + // pull the retry earlier than the deadline the backoff already committed to. + const delay = + retryDelayMs ?? + (this.retryNotBefore === undefined + ? debounced + : Math.max(debounced, this.retryNotBefore - now)); + this.timer = setTimeout(() => { + this.timer = undefined; + void this.drain(); + }, delay); + } + + private async drain(): Promise { + if (this.closed || this.active !== undefined || !this.hasPendingWork()) return; + const paths = [ + ...(this.overflowed ? [WATCH_FULL_REFRESH_PATH] : []), + ...[...this.pending].sort(), + ]; + this.pending.clear(); + this.firstPendingAt = undefined; + this.overflowed = false; + this.retryNotBefore = undefined; + await this.runBatch(paths, false); + } + + private async runBatch(paths: readonly string[], propagateError: boolean): Promise { + let work: Promise; + try { + work = this.refresh(paths); + } catch (error) { + work = Promise.reject(error); + } + this.active = work; + let retryDelayMs: number | undefined; + try { + await work; + this.consecutiveFailures = 0; + } catch (error) { + if (propagateError) throw error; + try { + await this.onError(error, paths); + } catch { + // Refresh failures are already handled here; a reporter must not + // reject the detached drain promise and become an unhandled rejection. + } + if (!this.closed) { + if (paths.includes(WATCH_FULL_REFRESH_PATH)) this.overflowed = true; + for (const filePath of paths) { + if (filePath !== WATCH_FULL_REFRESH_PATH) this.addPendingPath(filePath); + } + this.firstPendingAt = Date.now(); + this.consecutiveFailures++; + const base = this.options.retryBaseDelayMs ?? Math.max(250, this.debounceMs); + const maximum = this.options.retryMaxDelayMs ?? 30_000; + retryDelayMs = Math.min(maximum, base * 2 ** (this.consecutiveFailures - 1)); + } + } finally { + if (this.active === work) this.active = undefined; + if (!this.closed && !this.initialPending && this.hasPendingWork()) + this.schedule(retryDelayMs); + else this.resolveIdleWaiters(); + } + } + + private hasPendingWork(): boolean { + return this.overflowed || this.pending.size > 0; + } + + private isIdle(): boolean { + return this.active === undefined && this.timer === undefined && !this.hasPendingWork(); + } + + private resolveIdleWaiters(): void { + if (!this.isIdle() && !this.closed) return; + for (const resolve of this.idleWaiters) resolve(); + this.idleWaiters.clear(); + } +} diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts new file mode 100644 index 000000000..3cbec636f --- /dev/null +++ b/gitnexus/src/cli/watch.ts @@ -0,0 +1,501 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { watch, type FSWatcher } from 'chokidar'; +import { createWatchIgnorePredicate } from '../config/ignore-service.js'; +import { + analyzeFailureMayHaveMutatedLiveIndex, + runFullAnalysis, + type AnalyzeOptions as CoreAnalyzeOptions, + type AnalyzeResult, +} from '../core/run-analyze.js'; +import { getGitRoot, hasGitDir } from '../storage/git.js'; +import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; +import { GITNEXUS_DIR } from '../storage/repo-meta.js'; +import { + loadAnalyzeConfigStrict, + mergeAnalyzeOptions, + validateBranchName, +} from './analyze-config.js'; +import type { AnalyzeOptions } from './analyze-options.js'; +import { ensureHeap } from './analyze.js'; +import { cliError, cliInfo, cliWarn } from './cli-message.js'; +import { + WATCH_FULL_REFRESH_PATH, + WatchRefreshQueue, + type WatchRefreshError, +} from './watch-queue.js'; + +const DEFAULT_DEBOUNCE_MS = 300; +const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_FILE_SIZE_KB = 32 * 1024; +const TRANSIENT_WATCH_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'ENOTDIR', 'EPERM']); + +export type WatchCliOptions = AnalyzeOptions; + +function posixWatchPath(filePath: string): string { + return filePath.replace(/\\/g, '/').replace(/^\.\/+/, ''); +} + +export function isRelevantWatchPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath); + return ( + normalized.length > 0 && + normalized !== '.' && + !normalized.startsWith('../') && + !path.posix.isAbsolute(normalized) && + !path.win32.isAbsolute(filePath) + ); +} + +function isIgnoreControlPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath); + return normalized === '.gitignore' || normalized === '.gitnexusignore'; +} + +function isConfigControlPath(filePath: string): boolean { + return posixWatchPath(filePath) === '.gitnexusrc'; +} + +function isAnalyzerOwnedWatchPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath).replace(/\/+$/, ''); + return normalized === GITNEXUS_DIR || normalized.startsWith(`${GITNEXUS_DIR}/`); +} + +function repoRelativeWatchPath(repoPath: string, candidate: string): string | null { + const relative = path.relative(repoPath, candidate).replace(/\\/g, '/'); + if (!relative || relative.startsWith('../') || path.isAbsolute(relative)) return null; + return relative; +} + +export interface WatchEnvironmentBaseline { + readonly maxFileSize: string | undefined; + readonly workerTimeout: string | undefined; + readonly verbose: string | undefined; +} + +function setEnvironment(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +function positiveInteger( + value: string | undefined, + flag: string, + maximum?: number, +): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) + throw new Error(`${flag} must be a positive integer`); + if (maximum !== undefined && parsed > maximum) { + throw new Error(`${flag} must not exceed ${maximum}`); + } + return parsed; +} + +export async function resolveWatchOptions( + repoPath: string, + cli: WatchCliOptions, + baseline: WatchEnvironmentBaseline, + reportIgnoredConfig: (names: readonly string[]) => void = () => {}, +): Promise { + const config = (await loadAnalyzeConfigStrict(repoPath)) ?? {}; + const merged = mergeAnalyzeOptions(cli, config); + const unsupported = [ + ['--force', cli.force], + ['--repair-fts', cli.repairFts], + ['--embeddings', cli.embeddings], + ['--drop-embeddings', cli.dropEmbeddings], + ['--skills', cli.skills], + ['--default-branch', cli.defaultBranch], + ['--skip-agents-md', cli.skipAgentsMd], + ['--skip-skills', cli.skipSkills], + ['--no-stats', cli.stats === false], + ['--self-commit', cli.selfCommit], + ['--index-only', cli.indexOnly], + ['--skip-git', cli.skipGit], + ['walCheckpointThreshold', cli.walCheckpointThreshold], + ['embeddingThreads', cli.embeddingThreads], + ['embeddingBatchSize', cli.embeddingBatchSize], + ['embeddingSubBatchSize', cli.embeddingSubBatchSize], + ['embeddingDevice', cli.embeddingDevice], + ['embeddingBaseUrl', cli.embeddingBaseUrl], + ['embeddingModel', cli.embeddingModel], + ['--embedding-auth-token', cli.embeddingAuthToken], + ['--embedding-dims', cli.embeddingDims], + ].filter(([, value]) => value !== undefined && value !== false); + if (unsupported.length > 0) { + throw new Error( + `analyze --watch does not support ${unsupported.map(([name]) => name).join(', ')}`, + ); + } + reportIgnoredConfig( + [ + ['embeddings', config.embeddings], + ['dropEmbeddings', config.dropEmbeddings], + ['defaultBranch', config.defaultBranch], + ['skipAgentsMd', config.skipAgentsMd !== undefined], + ['skipSkills', config.skipSkills !== undefined], + ['stats', config.stats !== undefined], + ['walCheckpointThreshold', config.walCheckpointThreshold], + ['embeddingThreads', config.embeddingThreads], + ['embeddingBatchSize', config.embeddingBatchSize], + ['embeddingSubBatchSize', config.embeddingSubBatchSize], + ['embeddingDevice', config.embeddingDevice], + ['embeddingBaseUrl', config.embeddingBaseUrl], + ['embeddingModel', config.embeddingModel], + ] + .filter(([, value]) => value !== undefined && value !== false) + .map(([name]) => String(name)), + ); + const branch = + merged.branch === undefined ? undefined : validateBranchName(merged.branch, '--branch'); + const workerPoolSize = positiveInteger(merged.workers, '--workers'); + const workerTimeoutSeconds = positiveInteger(merged.workerTimeout, 'workerTimeout'); + const maxFileSize = positiveInteger(merged.maxFileSize, 'maxFileSize', MAX_FILE_SIZE_KB); + + setEnvironment( + 'GITNEXUS_MAX_FILE_SIZE', + maxFileSize === undefined ? baseline.maxFileSize : String(maxFileSize), + ); + if (workerTimeoutSeconds !== undefined) { + process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS = String(workerTimeoutSeconds * 1000); + } else { + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baseline.workerTimeout); + } + setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose); + + return { + pdg: merged.pdg, + branch, + registryName: merged.name, + allowDuplicateName: merged.allowDuplicateName, + workerPoolSize, + fetchWrappers: merged.fetchWrappers, + skipAgentsMd: true, + skipSkills: true, + noStats: true, + atomicIncremental: process.platform !== 'win32', + }; +} + +function refreshSummary( + result: AnalyzeResult, + observedPaths: readonly string[], + durationMs: number, + lastSuccessfulRefreshAt: string, +): string { + const measured = result.incrementalStats; + const changed = measured?.changedFiles ?? (result.alreadyUpToDate ? 0 : observedPaths.length); + const reparsed = + measured?.reparsedFiles ?? + (typeof result.pipelineResult?.reparsedFileCount === 'number' + ? result.pipelineResult.reparsedFileCount + : 0); + const dependents = measured?.affectedDependents ?? 0; + const mode = measured?.writeMode ?? (result.alreadyUpToDate ? 'no-op' : 'full'); + return ( + `Refresh complete: ${changed} changed, ${reparsed} re-parsed, ` + + `${dependents} affected dependent(s), ${durationMs}ms, ${mode}; ` + + `last success ${lastSuccessfulRefreshAt}` + ); +} + +async function waitUntilReady(watcher: FSWatcher): Promise { + await new Promise((resolve, reject) => { + const ready = () => { + watcher.off('error', failed); + resolve(); + }; + const failed = (error: unknown) => { + watcher.off('ready', ready); + reject(error); + }; + watcher.once('ready', ready); + watcher.once('error', failed); + }); +} + +export interface WatchFileLoop { + readonly waitForIdle: () => Promise; + readonly close: () => Promise; +} + +class WatchControlReloadError extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = 'WatchControlReloadError'; + } +} + +export function shouldStopAfterWatchRefreshFailure( + error: unknown, + paths: readonly string[], +): boolean { + return ( + paths.length > 0 && + !(error instanceof WatchControlReloadError) && + analyzeFailureMayHaveMutatedLiveIndex(error) + ); +} + +/** Start the real filesystem watcher with bounded, serialized refreshes. */ +export async function startWatchFileLoop( + repoPath: string, + debounceMs: number, + refresh: (paths: readonly string[]) => Promise, + onError: WatchRefreshError, + onWatcherError: (error: unknown) => void = (error) => onError(error, []), +): Promise { + let ignorePath = await createWatchIgnorePredicate(repoPath); + let ignoreControlValid = true; + const queue = new WatchRefreshQueue( + async (paths) => { + if (paths.some(isIgnoreControlPath) || !ignoreControlValid) { + const retryingInvalidControls = !ignoreControlValid; + try { + ignorePath = await createWatchIgnorePredicate(repoPath); + ignoreControlValid = true; + watcher.add(repoPath); + } catch (error) { + ignoreControlValid = false; + throw new WatchControlReloadError( + retryingInvalidControls + ? new Error( + 'Ignore controls remain invalid; fix them before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + await refresh(paths); + }, + onError, + debounceMs, + { + maxWaitMs: Math.max(2_000, debounceMs * 10), + maxPendingPaths: 1_000, + holdEventsUntilInitialRefresh: true, + isPriorityPath: (filePath) => isIgnoreControlPath(filePath) || isConfigControlPath(filePath), + }, + ); + + const watcher: FSWatcher = watch(repoPath, { + ignoreInitial: true, + atomic: true, + followSymlinks: false, + awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 20 }, + ignored: (candidate, stats) => { + const relative = repoRelativeWatchPath(repoPath, candidate); + if (relative !== null && isAnalyzerOwnedWatchPath(relative)) return true; + if (relative !== null && (isIgnoreControlPath(relative) || isConfigControlPath(relative))) { + return false; + } + return ignorePath(candidate, stats?.isDirectory() ?? false); + }, + }); + watcher.on('all', (event, changedPath) => { + if (event !== 'add' && event !== 'change' && event !== 'unlink') return; + const relative = repoRelativeWatchPath(repoPath, changedPath); + if (relative && isRelevantWatchPath(relative) && !isAnalyzerOwnedWatchPath(relative)) { + queue.enqueue(relative); + } + }); + watcher.on('error', (error) => { + // Chokidar can surface a transient EPERM on Windows while an ignored + // analyzer-owned path is replaced. Re-arm the root and force one bounded + // catch-up refresh so a missed event cannot leave the graph stale. Other + // watcher errors may mean coverage was lost and remain fatal. + if (TRANSIENT_WATCH_ERROR_CODES.has((error as NodeJS.ErrnoException).code ?? '')) { + watcher.add(repoPath); + queue.enqueue(WATCH_FULL_REFRESH_PATH); + return; + } + onWatcherError(error); + }); + + try { + await waitUntilReady(watcher); + await queue.runInitial(); + } catch (error) { + await watcher.close(); + await queue.close(); + throw error; + } + + return { + waitForIdle: () => queue.waitForIdle(), + close: async () => { + await watcher.close(); + await queue.close(); + }, + }; +} + +export async function watchCommandWithRunnerIdentity( + runnerIdentityAtBootstrap: AnalyzerRunnerIdentity, + inputPath?: string, + cliOptions: WatchCliOptions = {}, +): Promise { + if (await ensureHeap({ cleanForwardedTermination: true })) return; + + const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { + cliError(' gitnexus analyze --watch requires a Git repository.'); + process.exitCode = 1; + return; + } + const repoPath = await fs.realpath(requestedRepoPath); + const baselineEnvironment: WatchEnvironmentBaseline = { + maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, + workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, + verbose: process.env.GITNEXUS_VERBOSE, + }; + try { + let ignoredConfigSignature: string | undefined; + const reportIgnoredConfig = (names: readonly string[]) => { + const signature = [...names].sort().join(','); + if (signature === ignoredConfigSignature) return; + ignoredConfigSignature = signature; + if (names.length > 0) { + cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); + } + }; + let debounceMs: number; + let analyzeOptions: CoreAnalyzeOptions; + try { + debounceMs = + positiveInteger( + cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), + '--debounce', + MAX_TIMER_DELAY_MS, + ) ?? DEFAULT_DEBOUNCE_MS; + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + } catch (error) { + cliError(` ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + return; + } + + let stopWatching!: () => void; + const stopped = new Promise((resolve) => { + stopWatching = resolve; + }); + const stop = () => stopWatching(); + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + try { + let loop: WatchFileLoop; + let fatalRefreshError: unknown; + let configControlValid = true; + let lastSuccessfulRefreshAt: string | undefined; + try { + loop = await startWatchFileLoop( + repoPath, + debounceMs, + async (paths) => { + if (paths.some(isConfigControlPath) || !configControlValid) { + const retryingInvalidConfig = !configControlValid; + try { + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + configControlValid = true; + } catch (error) { + configControlValid = false; + throw new WatchControlReloadError( + retryingInvalidConfig + ? new Error( + 'Configuration remains invalid; fix it before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + const startedAt = Date.now(); + const result = await runFullAnalysis( + repoPath, + analyzeOptions, + { + onProgress: () => {}, + onLog: + process.env.GITNEXUS_VERBOSE === '1' + ? (message) => cliInfo(` ${message}`) + : undefined, + }, + runnerIdentityAtBootstrap, + ); + lastSuccessfulRefreshAt = new Date().toISOString(); + if (paths.length === 0) { + cliInfo( + result.alreadyUpToDate + ? `Watching ${repoPath}; index is up to date.` + : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, + ); + } else { + cliInfo( + refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), + ); + } + }, + (error, paths) => { + const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; + if (shouldStopAfterWatchRefreshFailure(error, paths)) { + fatalRefreshError = error; + cliError( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping because the live index may have been updated in place.', + ); + stopWatching(); + return; + } + const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; + cliWarn( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + `Retry scheduled; last success ${lastSuccess}.`, + ); + }, + (error) => { + fatalRefreshError = error; + cliError( + `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping.', + ); + stopWatching(); + }, + ); + } catch (error) { + cliError( + ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + return; + } + + await stopped; + await loop.close(); + if (fatalRefreshError !== undefined) process.exitCode = 1; + } finally { + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); + } + } finally { + setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); + setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); + } +} diff --git a/gitnexus/src/config/ignore-service.ts b/gitnexus/src/config/ignore-service.ts index f1b9e1150..ac2dc7704 100644 --- a/gitnexus/src/config/ignore-service.ts +++ b/gitnexus/src/config/ignore-service.ts @@ -3,6 +3,7 @@ import { existsSync } from 'fs'; import fs from 'fs/promises'; import nodePath from 'path'; import type { Path } from 'path-scurry'; +import { readRepoControlFile } from './repo-control-file.js'; import { logger } from '../core/logger.js'; import { getCoreExcludesFilePath, getGitInfoExcludePath } from '../storage/git.js'; @@ -401,6 +402,8 @@ export interface IgnoreOptions { noGitignore?: boolean; /** Skip core.excludesFile and $GIT_COMMON_DIR/info/exclude. Defaults to GITNEXUS_NO_GLOBAL_IGNORE env var. */ noGlobalIgnore?: boolean; + /** Fail repository-control reloads closed so long-lived watchers keep their prior predicate. */ + strictRepoControlFiles?: boolean; } export const loadIgnoreRules = async ( @@ -442,20 +445,56 @@ export const loadIgnoreRules = async ( for (const filename of filenames) { try { - const content = await fs.readFile(nodePath.join(repoPath, filename), 'utf-8'); + const content = options?.strictRepoControlFiles + ? await readRepoControlFile(repoPath, filename) + : await fs.readFile(nodePath.join(repoPath, filename), 'utf-8'); + if (content === null) continue; ig.add(content); hasRules = true; } catch (err: unknown) { const code = (err as NodeJS.ErrnoException).code; - if (code !== 'ENOENT') { - logger.warn(` Warning: could not read ${filename}: ${(err as Error).message}`); - } + if (!options?.strictRepoControlFiles && code === 'ENOENT') continue; + if (options?.strictRepoControlFiles) throw err; + logger.warn(` Warning: could not read ${filename}: ${(err as Error).message}`); } } return hasRules ? ig : null; }; +/** + * Build a synchronous predicate for long-lived filesystem watchers. + * + * Unlike {@link createIgnoreFilter}, callers pass ordinary absolute or + * repository-relative paths instead of path-scurry `Path` objects. The rule + * precedence deliberately mirrors the scanner: explicit negations win over + * hardcoded defaults unless a more-specific rule re-ignores the path. + */ +export const createWatchIgnorePredicate = async ( + repoPath: string, + options?: IgnoreOptions, +): Promise<(candidatePath: string, isDirectory?: boolean) => boolean> => { + const ig = await loadIgnoreRules(repoPath, { ...options, strictRepoControlFiles: true }); + const repoRoot = nodePath.resolve(repoPath); + + return (candidatePath: string, isDirectory = false): boolean => { + const absolute = nodePath.isAbsolute(candidatePath) + ? nodePath.resolve(candidatePath) + : nodePath.resolve(repoRoot, candidatePath); + const rel = nodePath.relative(repoRoot, absolute).replace(/\\/g, '/'); + if (!rel) return false; + if (rel === '..' || rel.startsWith('../') || nodePath.isAbsolute(rel)) return true; + + if (ig && hasExplicitUnignore(ig, rel) && !ig.ignores(isDirectory ? `${rel}/` : rel)) { + return false; + } + + if (ig && ig.ignores(isDirectory ? `${rel}/` : rel)) return true; + if (isDirectory && isHardcodedIgnoredDirectoryAtPath(repoRoot, absolute)) return true; + return shouldIgnorePath(rel); + }; +}; + /** * Walk ancestor segments of `rel` and check whether `.gitnexusignore` * (or `.gitignore`) contains an explicit `!pattern` negation that diff --git a/gitnexus/src/config/repo-control-file.ts b/gitnexus/src/config/repo-control-file.ts new file mode 100644 index 000000000..13e08adb0 --- /dev/null +++ b/gitnexus/src/config/repo-control-file.ts @@ -0,0 +1,117 @@ +import fs from 'node:fs'; +import * as path from 'node:path'; + +export const MAX_REPO_CONTROL_FILE_BYTES = 1024 * 1024; + +/** Read a bounded, regular control file owned by the repository root. */ +export async function readRepoControlFile( + repoRoot: string, + filename: string, +): Promise { + const requestedRoot = path.resolve(repoRoot); + const requested = path.resolve(requestedRoot, filename); + const relative = path.relative(requestedRoot, requested); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`${filename} resolves outside the repository root`); + } + + try { + const canonicalRoot = fs.realpathSync(requestedRoot); + const beforeOpen = fs.lstatSync(requested); + if (beforeOpen.isSymbolicLink()) throw new Error(`${filename} must not be a symbolic link`); + if (!beforeOpen.isFile()) throw new Error(`${filename} must be a regular file`); + if (beforeOpen.nlink !== 1) throw new Error(`${filename} must not be a hard link`); + if (beforeOpen.size > MAX_REPO_CONTROL_FILE_BYTES) { + throw new Error(`${filename} exceeds ${MAX_REPO_CONTROL_FILE_BYTES} bytes`); + } + return await new Promise((resolve, reject) => { + const stream = fs.createReadStream(requested, { + flags: 'r', + start: 0, + end: MAX_REPO_CONTROL_FILE_BYTES, + autoClose: true, + }); + const chunks: Buffer[] = []; + let totalBytes = 0; + let validated = false; + let settled = false; + + const finish = (value: string): void => { + if (settled) return; + settled = true; + resolve(value); + }; + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + reject(error); + }; + + stream.pause(); + stream.once('open', (fd) => { + try { + const opened = fs.fstatSync(fd); + if (!opened.isFile()) throw new Error(`${filename} must be a regular file`); + if (opened.nlink !== 1) throw new Error(`${filename} must not be a hard link`); + if (opened.size > MAX_REPO_CONTROL_FILE_BYTES) { + throw new Error(`${filename} exceeds ${MAX_REPO_CONTROL_FILE_BYTES} bytes`); + } + + const entry = fs.lstatSync(requested); + if (entry.isSymbolicLink()) throw new Error(`${filename} must not be a symbolic link`); + if ( + !entry.isFile() || + entry.nlink !== 1 || + entry.dev !== opened.dev || + entry.ino !== opened.ino + ) { + throw new Error(`${filename} moved or was replaced while being opened`); + } + const canonicalFile = fs.realpathSync(requested); + const canonicalRelative = path.relative(canonicalRoot, canonicalFile); + if (canonicalRelative.startsWith('..') || path.isAbsolute(canonicalRelative)) { + throw new Error(`${filename} resolves outside the repository root`); + } + const canonical = fs.statSync(canonicalFile); + if ( + canonical.nlink !== 1 || + canonical.dev !== opened.dev || + canonical.ino !== opened.ino + ) { + throw new Error(`${filename} moved or was replaced while being opened`); + } + + validated = true; + stream.resume(); + } catch (error) { + fail(error); + stream.destroy(); + } + }); + stream.on('data', (chunk: Buffer | string) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += bytes.length; + if (totalBytes > MAX_REPO_CONTROL_FILE_BYTES) { + fail(new Error(`${filename} exceeds ${MAX_REPO_CONTROL_FILE_BYTES} bytes`)); + stream.destroy(); + return; + } + chunks.push(bytes); + }); + stream.once('end', () => { + if (!validated) { + fail(new Error(`${filename} could not be validated`)); + return; + } + finish(Buffer.concat(chunks, totalBytes).toString('utf8')); + }); + stream.once('error', fail); + stream.once('close', () => { + if (!settled) fail(new Error(`${filename} closed before it could be read`)); + }); + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index cd4b8edf4..157e29027 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -477,6 +477,8 @@ export async function runChunkedParseAndResolve( * files. There is no sequential parser — the pool is the sole parse path * whenever a chunk misses the cache. */ usedWorkerPool: boolean; + /** Files dispatched to parser workers after parse-cache lookup. */ + reparsedFileCount: number; /** Worker-produced ParsedFile artifacts aggregated across chunks. * Threaded into scope-resolution as a re-extract cache so the warm- * cache analyze run can skip the dominant `extractParsedFile` cost @@ -783,6 +785,7 @@ export async function runChunkedParseAndResolve( : new Set(); let chunkCacheHits = 0; let chunkCacheMisses = 0; + let reparsedFileCount = 0; try { // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch @@ -1106,6 +1109,7 @@ export async function runChunkedParseAndResolve( // Cache miss: dispatch to workers, capture the raw results, store // them under the chunk hash for the next run. chunkCacheMisses++; + reparsedFileCount += chunkFiles.length; if (durableParsedFileDir !== undefined && chunkHash !== null) { try { await prepareDurableParsedFileChunk(durableParsedFileDir, chunkHash); @@ -1622,6 +1626,11 @@ export async function runChunkedParseAndResolve( // no pool was needed: a warm all-cache-hit run replays cached worker output // without spawning workers, or there were no parseable files. usedWorkerPool: workerPool !== undefined, + // Exact number of files sent through workers on parse-cache misses. A + // changed file can invalidate its whole content-addressed chunk, so this + // is intentionally measured at dispatch time rather than inferred from + // the git/hash diff. + reparsedFileCount, // Per-file ParsedFile artifacts produced by workers' calls to // `extractParsedFile`. Consumed by scope-resolution as a re-extraction // cache: when the file's ParsedFile is here, scope-resolution skips its own diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 38bd4601b..8e151ed73 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -71,6 +71,8 @@ export interface ParseOutput { * is no sequential parser; the pool is the sole parse path on a cache miss. */ readonly usedWorkerPool: boolean; + /** Files actually dispatched to parser workers after parse-cache lookup. */ + readonly reparsedFileCount: number; /** * Per-file `ParsedFile` artifacts produced by workers' calls to * `extractParsedFile`. Threaded through to `scopeResolutionPhase` diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 69858ff28..050822e87 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -370,11 +370,13 @@ export const runPipelineFromRepo = async ( } // Extract final results for the PipelineResult contract - const { totalFiles, usedWorkerPool, unavailableScopeLanguageFiles } = getPhaseOutput<{ - totalFiles: number; - usedWorkerPool: boolean; - unavailableScopeLanguageFiles: number; - }>(results, 'parse'); + const { totalFiles, usedWorkerPool, reparsedFileCount, unavailableScopeLanguageFiles } = + getPhaseOutput<{ + totalFiles: number; + usedWorkerPool: boolean; + reparsedFileCount: number; + unavailableScopeLanguageFiles: number; + }>(results, 'parse'); let communityResult: CommunitiesOutput['communityResult'] | undefined; let processResult: ProcessesOutput['processResult'] | undefined; @@ -426,6 +428,7 @@ export const runPipelineFromRepo = async ( resolutionOutcomes, undecidedSatisfaction, usedWorkerPool, + reparsedFileCount, scopeExtractionFailures, unavailableScopeLanguageFiles, pdgEmitManifest, diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index f3889d444..94d53de16 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -13,6 +13,7 @@ import { detectGraphWriteCollapse, type GraphWriteCollapseVerdict } from './inde import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js'; import path from 'path'; import fs from 'fs/promises'; +import { constants as fsConstants } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { retryRename } from '../storage/fs-atomic.js'; import { acquireIndexLock } from '../storage/index-lock.js'; @@ -471,6 +472,29 @@ export interface AnalyzeOptions { * Process exit reclaims the handles. Long-lived callers (MCP server, tests) * leave this unset so they get a real close. See `closeLbug`. */ skipNativeCloseOnExit?: boolean; + /** + * Stage an incremental write in a copy of the live index before publishing + * it. Used by long-lived watch mode so a failed refresh leaves the previous + * graph readable. Currently supported on POSIX, where an open DB can be + * atomically renamed; Windows retains the established in-place path. + */ + atomicIncremental?: boolean; +} + +const liveIndexMutationRisks = new WeakSet(); + +function recordLiveIndexMutationRisk(error: unknown): void { + if ((typeof error === 'object' && error !== null) || typeof error === 'function') { + liveIndexMutationRisks.add(error); + } +} + +/** Whether a failed analyze may already have changed the live DB. */ +export function analyzeFailureMayHaveMutatedLiveIndex(error: unknown): boolean { + return ( + ((typeof error === 'object' && error !== null) || typeof error === 'function') && + liveIndexMutationRisks.has(error) + ); } export interface AnalyzeResult { @@ -522,6 +546,14 @@ export interface AnalyzeResult { * (The historical "primary" name is kept — it is public API surface.) */ isPrimaryBranch?: boolean; + /** Measured work performed by a successful incremental refresh. */ + incrementalStats?: { + changedFiles: number; + reparsedFiles: number; + affectedDependents: number; + deletedFiles: number; + writeMode: 'incremental' | 'full'; + }; } /** @@ -1944,12 +1976,15 @@ async function runFullAnalysisInner( process.platform === 'win32' && options.pdg !== true && process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1'; - // Incremental atomicity copies the whole index into the temp before mutating - // it, which negates incremental's speed premise — so it is opt-in - // (GITNEXUS_ATOMIC_INCREMENTAL=1) pending a benchmark. Full rebuilds always - // swap where the platform allows. + // Incremental atomicity stages the whole index before mutation. It remains + // opt-in for ordinary analyze runs; watch mode requests it for failure + // preservation. The copy requests a filesystem clone and records its actual + // duration, while Node falls back to a normal copy where reflinks are absent. const wantAtomicIncremental = - isIncremental && !!hashDiff && process.env.GITNEXUS_ATOMIC_INCREMENTAL === '1'; + isIncremental && + !!hashDiff && + process.platform !== 'win32' && + (options.atomicIncremental === true || process.env.GITNEXUS_ATOMIC_INCREMENTAL === '1'); // #2614 F3: the copy-then-swap stages ONLY the main lbug file, so a live index // carrying an orphan .wal/.shadow (a silently-failed prior checkpoint) would // be copied incompletely and lose that delta. Only take the atomic path when @@ -1967,6 +2002,10 @@ async function runFullAnalysisInner( // valve. Nothing between here and there reads either binding except // `initLbug(buildPath)`, which the upgrade re-runs against the staging path. let useAtomicSwap = (isFullRebuild || atomicIncremental) && (posixSwap || windowsSwapOk); + // Set only at the first operation that can mutate the live graph store. + // Pre-write failures (config, lock, parsing, metadata, importer expansion) + // remain retryable even when this platform cannot use an atomic swap. + let liveIndexMutationStarted = false; // #2658: a per-run staging name (was the fixed `lbug.new`). Even under the // single-writer lock, a unique name means a crashed run's half-built staging // file can never be mistaken for — or clobber — a live run's; the lock's @@ -1999,10 +2038,15 @@ async function runFullAnalysisInner( if (atomicIncremental) { // Stage the live index into the temp so the in-place delete/writeback // below mutates the COPY, and the end-of-run swap publishes it atomically. - // Clear any stale temp first (a crashed run), then copy the (consolidated, - // single-file) live index. Whole-file copy — hence opt-in. + // Clear any stale temp first (a crashed run), then clone/copy the + // consolidated single-file live index. await wipeLbugDbFiles(buildPath); - await fs.copyFile(lbugPath, buildPath); + const copyStartedAt = Date.now(); + await fs.copyFile(lbugPath, buildPath, fsConstants.COPYFILE_FICLONE); + log( + `atomic-incremental: staged ${lbugPath} in ${Date.now() - copyStartedAt}ms ` + + '(copy-on-write requested; filesystem fallback is allowed)', + ); } } else { // Full rebuild path: wipe DB files first. @@ -2038,7 +2082,13 @@ async function runFullAnalysisInner( // (`buildPath` = `.new`, clearing any stragglers from a crashed // run) and leaves the live index untouched until the end-of-run swap. On // Windows buildPath === lbugPath, so this is the original in-place wipe. - await wipeLbugDbFiles(buildPath); + if (buildPath === lbugPath) liveIndexMutationStarted = true; + try { + await wipeLbugDbFiles(buildPath); + } catch (error) { + if (liveIndexMutationStarted) recordLiveIndexMutationRisk(error); + throw error; + } } // Size the buffer pool to the graph just built by the pipeline (a page cache @@ -2061,7 +2111,12 @@ async function runFullAnalysisInner( // Full rebuild (POSIX) builds into the temp `buildPath`; incremental and // Windows use `buildPath === lbugPath` in place. - await initLbug(buildPath); + try { + await initLbug(buildPath); + } catch (error) { + if (liveIndexMutationStarted) recordLiveIndexMutationRisk(error); + throw error; + } // Manual WAL checkpoint driver (#1741): periodically drain the WAL // from JS so the un-retriable native auto-checkpoint almost never @@ -2086,6 +2141,7 @@ async function runFullAnalysisInner( // "escalated full write" (DB wiped, index destroyed) — tri-review // 4669518496 P1. let escalatedFullWrite = false; + let incrementalStats: AnalyzeResult['incrementalStats']; // Phase 3.5's restore scope (FIX 3 of this shipping review): on the // SURGICAL write plan this is the exact file set whose rows // deleteNodesForFiles just removed — only THOSE files' cached embedding @@ -2211,6 +2267,13 @@ async function runFullAnalysisInner( } } const importerExpansion = writableFiles.size - directlyChangedCount; + incrementalStats = { + changedFiles: hashDiff.changed.length + hashDiff.added.length + hashDiff.deleted.length, + reparsedFiles: pipelineResult.reparsedFileCount, + affectedDependents: importerExpansion, + deletedFiles: hashDiff.deleted.length, + writeMode: 'incremental', + }; await saveIncrementalDirtyState('importer-bfs', { importerExpansion, shadowSeedCount: shadowSeed.length, @@ -2572,6 +2635,7 @@ async function runFullAnalysisInner( } await walCheckpointDriver.stop(); await closeLbug(); + if (buildPath === lbugPath) liveIndexMutationStarted = true; await wipeLbugDbFiles(buildPath); await initLbug(buildPath); walCheckpointDriver = startWalCheckpointDriver(); @@ -2597,6 +2661,7 @@ async function runFullAnalysisInner( // same connection, and nothing on this branch creates or drops an index // in between — so re-reading would only weaken the one-read invariant // the snapshot type exists to enforce. + if (buildPath === lbugPath) liveIndexMutationStarted = true; await dropSearchFTSIndexes(indexCatalogRows); // 1b. Remove the write set's existing rows — batched (#2409): one // DETACH DELETE per table per 200-file chunk. The former per-file @@ -3773,6 +3838,7 @@ async function runFullAnalysisInner( : false; if (useAtomicSwap && builtDbExists) { await retryRename(buildPath, lbugPath); + liveIndexMutationStarted = true; // Clear any sidecars orphaned beside the replaced file. A cleanly-closed // prior index has none; a crashed one could, and it would be replay // poison next to the freshly published index. Best-effort. @@ -3807,6 +3873,12 @@ async function runFullAnalysisInner( ftsSkipped: !ftsReady, ftsSkipReason: ftsReady ? undefined : ftsSkipReason, isPrimaryBranch: !placement.branch, + incrementalStats: incrementalStats + ? { + ...incrementalStats, + writeMode: escalatedFullWrite ? 'full' : 'incremental', + } + : undefined, }; } catch (err) { // Ensure LadybugDB is closed even on error. Stop the driver first @@ -3845,6 +3917,11 @@ async function runFullAnalysisInner( /* swallow — orphan reclamation must never mask the real failure */ } } + if (liveIndexMutationStarted) { + // Preserve the original error identity/prototype: callers distinguish + // IndexLockTimeoutError and other domain failures with `instanceof`. + recordLiveIndexMutationRisk(err); + } throw err; } } diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 636b73486..bcbb4c189 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -917,7 +917,17 @@ export const persistParseCacheChunk = async ( createdCacheDirs.add(cacheDir); } const payload = JSON.stringify(slim, mapReplacer); - await fs.writeFile(getCacheChunkPath(cache.storagePath, chunkHash), payload, 'utf-8'); + const chunkPath = getCacheChunkPath(cache.storagePath, chunkHash); + try { + await fs.writeFile(chunkPath, payload, 'utf-8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + // Long-lived analyze --watch processes can replace the sharded cache + // directory after this process-local memo recorded it as created. + await fs.mkdir(cacheDir, { recursive: true }); + createdCacheDirs.add(cacheDir); + await fs.writeFile(chunkPath, payload, 'utf-8'); + } cache.onDiskKeys ??= new Set(); cache.onDiskKeys.add(chunkHash); cache.entries.delete(chunkHash); diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index 950cd3f31..5c11f800b 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -40,6 +40,8 @@ export interface PipelineResult { * affordance so regression suites can prove the pool engaged. */ usedWorkerPool: boolean; + /** Files actually dispatched to parser workers after parse-cache lookup. */ + reparsedFileCount: number; /** Files omitted from scope-resolution while the rest of analysis continued. */ scopeExtractionFailures: readonly string[]; /** Files scope resolution could not inspect because their parser was unavailable. */ diff --git a/gitnexus/test/integration/analyze-atomic-swap.test.ts b/gitnexus/test/integration/analyze-atomic-swap.test.ts index 8bee4aba0..f664a6bb6 100644 --- a/gitnexus/test/integration/analyze-atomic-swap.test.ts +++ b/gitnexus/test/integration/analyze-atomic-swap.test.ts @@ -22,17 +22,28 @@ type LbugAdapter = typeof import('../../src/core/lbug/lbug-adapter.js'); const ctx = vi.hoisted(() => ({ loadMock: vi.fn(), realLoad: null as LbugAdapter['loadGraphToLbug'] | null, + deleteMock: vi.fn(), + realDelete: null as LbugAdapter['deleteNodesForFiles'] | null, })); // Delegating mock: overrides only loadGraphToLbug so a rebuild can be made to // fail on demand (mirrors run-analyze-adopt-failure.test.ts). vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => { const actual = await importOriginal(); ctx.realLoad = actual.loadGraphToLbug; + ctx.realDelete = actual.deleteNodesForFiles; ctx.loadMock.mockImplementation(actual.loadGraphToLbug); - return { ...actual, loadGraphToLbug: ctx.loadMock }; + ctx.deleteMock.mockImplementation(actual.deleteNodesForFiles); + return { + ...actual, + loadGraphToLbug: ctx.loadMock, + deleteNodesForFiles: ctx.deleteMock, + }; }); -import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { + analyzeFailureMayHaveMutatedLiveIndex, + runFullAnalysis, +} from '../../src/core/run-analyze.js'; import { getStoragePaths } from '../../src/storage/repo-manager.js'; import { initLbug as poolInit, @@ -68,6 +79,10 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { ctx.loadMock.mockImplementation((...a: Parameters) => ctx.realLoad!(...a), ); + ctx.deleteMock.mockReset(); + ctx.deleteMock.mockImplementation((...a: Parameters) => + ctx.realDelete!(...a), + ); }); afterEach(async () => { @@ -129,6 +144,29 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { } }, 180_000); + it('marks a failure after an atomic publish as potentially live-mutating', async () => { + const { repo, cleanup } = await makeRepo(); + try { + const failure = await runFullAnalysis( + repo, + {}, + { + onProgress: (phase, percent) => { + if (phase === 'done' && percent === 100) { + throw new Error('injected post-publish failure'); + } + }, + }, + ).catch((error: unknown) => error); + + expect(failure).toMatchObject({ message: 'injected post-publish failure' }); + expect(analyzeFailureMayHaveMutatedLiveIndex(failure)).toBe(true); + await expect(fs.stat(getStoragePaths(repo).lbugPath)).resolves.toBeTruthy(); + } finally { + await cleanup(); + } + }, 180_000); + it('the read pool serves the freshly-swapped index after a rebuild (#1 + #2 end-to-end)', async () => { const { repo, cleanup } = await makeRepo(); const repoId = 'atomic-swap-e2e'; @@ -202,6 +240,76 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { } }, 180_000); + it('keeps the live graph unchanged when atomic incremental writeback fails', async () => { + const { repo, cleanup } = await makeRepo(); + const repoId = 'atomic-incr-failure'; + try { + await runFullAnalysis(repo, {}, { onProgress: () => {} }); + const { lbugPath } = getStoragePaths(repo); + const before = await identity(lbugPath); + + await fs.writeFile( + path.join(repo, 'a.ts'), + 'export function greet(n: string) { return `hi ${n}`; }\nexport function caller() { return greet("x"); }\nexport function addedAfterRetry() { return 1; }\n', + ); + execSync('git -c user.name=t -c user.email=t@t commit -am change', { + cwd: repo, + stdio: 'pipe', + }); + + ctx.deleteMock.mockRejectedValueOnce(new Error('injected incremental write failure')); + const failure = await runFullAnalysis( + repo, + { atomicIncremental: true }, + { onProgress: () => {} }, + ).catch((error: unknown) => error); + expect(failure).toMatchObject({ message: 'injected incremental write failure' }); + expect(analyzeFailureMayHaveMutatedLiveIndex(failure)).toBe(false); + expect(await identity(lbugPath)).toBe(before); + expect(await lingeringTemp(lbugPath)).toEqual([]); + + await poolInit(repoId, lbugPath); + const beforeRetry = ( + await poolQuery(repoId, 'MATCH (f:Function) RETURN f.name AS n') + ).flatMap((row) => Object.values(row as Record).map(String)); + expect(beforeRetry).toContain('greet'); + expect(beforeRetry).not.toContain('addedAfterRetry'); + await poolClose(repoId); + + await runFullAnalysis(repo, { atomicIncremental: true }, { onProgress: () => {} }); + await poolInit(repoId, lbugPath); + const afterRetry = (await poolQuery(repoId, 'MATCH (f:Function) RETURN f.name AS n')).flatMap( + (row) => Object.values(row as Record).map(String), + ); + expect(afterRetry).toContain('addedAfterRetry'); + } finally { + await poolClose(repoId); + await cleanup(); + } + }, 180_000); + + it('marks failed in-place incremental writes as potentially live-mutating', async () => { + const { repo, cleanup } = await makeRepo(); + try { + await runFullAnalysis(repo, {}, { onProgress: () => {} }); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function changed() { return 1; }\n'); + execSync('git -c user.name=t -c user.email=t@t commit -am change', { + cwd: repo, + stdio: 'pipe', + }); + + ctx.deleteMock.mockRejectedValueOnce(new Error('injected in-place failure')); + const failure = await runFullAnalysis(repo, {}, { onProgress: () => {} }).catch( + (error: unknown) => error, + ); + expect(failure).toBeInstanceOf(Error); + expect(failure).toMatchObject({ message: 'injected in-place failure' }); + expect(analyzeFailureMayHaveMutatedLiveIndex(failure)).toBe(true); + } finally { + await cleanup(); + } + }, 180_000); + it('publishes cleanly on the production close path (skipNativeCloseOnExit) (#2614 F5)', async () => { const { repo, cleanup } = await makeRepo(); try { diff --git a/gitnexus/test/integration/cli-e2e.test.ts b/gitnexus/test/integration/cli-e2e.test.ts index d44b5c4a8..ffff7dcfd 100644 --- a/gitnexus/test/integration/cli-e2e.test.ts +++ b/gitnexus/test/integration/cli-e2e.test.ts @@ -1048,6 +1048,198 @@ describe('CLI end-to-end', () => { expect(result.stdout).toMatch(/analyze|status|serve/i); }); + it('shows the analyze watch mode and its debounce controls', () => { + const result = runCliRaw(['analyze', '--help'], MINI_REPO); + expect(result.status).toBe(0); + expect(result.stdout).toContain('--watch'); + expect(result.stdout).toContain('--debounce'); + expect(result.stdout).toContain('--workers'); + }); + + it('rejects --debounce without --watch', () => { + const result = runCliRaw(['analyze', '--debounce', '25'], MINI_REPO); + expect(result.status).toBe(1); + expect(result.stderr).toContain('--debounce requires --watch'); + }); + + it('runs production analyze --watch with exact telemetry and transactional config reloads', async () => { + const repo = makeMiniRepoCopy('watch-repo', 'gn-watch-cli-'); + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-watch-cli-home-')); + try { + fs.writeFileSync( + path.join(repo, '.gitnexusrc'), + JSON.stringify({ workers: '1', maxFileSize: '1' }), + 'utf8', + ); + await new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [...CLI_SPAWN_PREFIX, 'analyze', repo, '--watch', '--debounce', '25', '--workers', '1'], + { + cwd: repo, + stdio: ['ignore', 'pipe', 'pipe'], + env: cliEnv({ GITNEXUS_HOME: home }), + }, + ); + let stdout = ''; + let stderr = ''; + let transcript = ''; + let baselineNodes: number | undefined; + let stage = 'ready'; + let stageOffset = 0; + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + child.kill('SIGTERM'); + reject(new Error(`watch CLI timed out\nstdout:\n${stdout}\nstderr:\n${stderr}`)); + }, 480_000); + + const advance = (nextStage: string, action: () => void) => { + stage = nextStage; + stageOffset = transcript.length; + setTimeout(action, 200); + }; + + const writeLargeSource = (fileName: string, functionName: string) => { + fs.writeFileSync( + path.join(repo, fileName), + `const padding = '${'x'.repeat(1_500)}';\n` + + `export function ${functionName}(): number { return padding.length; }\n`, + 'utf8', + ); + }; + + const handleOutput = () => { + const output = transcript.slice(stageOffset); + if (stage === 'ready' && /Watching .*index (?:is up to date|ready)/.test(output)) { + const meta = JSON.parse( + fs.readFileSync(path.join(repo, '.gitnexus', 'gitnexus.json'), 'utf8'), + ); + baselineNodes = meta.stats.nodes; + advance('proof', () => { + fs.writeFileSync( + path.join(repo, 'watch-proof.ts'), + 'export function watchProof(): number { return 1; }\n', + 'utf8', + ); + }); + return; + } + if (stage === 'proof' && /Refresh complete: 1 changed, 8 re-parsed,/.test(output)) { + const meta = JSON.parse( + fs.readFileSync(path.join(repo, '.gitnexus', 'gitnexus.json'), 'utf8'), + ); + expect(meta.stats.nodes).toBeGreaterThan(baselineNodes!); + advance('first-large-file', () => + writeLargeSource('oversized-before.ts', 'skippedByLimit'), + ); + return; + } + if ( + stage === 'first-large-file' && + output.includes('Skipped 1 large files (>1KB)') && + output.includes('- oversized-before.ts') && + /Refresh complete: 0 changed,/.test(output) + ) { + advance('invalid-config', () => { + fs.writeFileSync( + path.join(repo, '.gitnexusrc'), + JSON.stringify({ workers: '1', maxFileSize: '0' }), + 'utf8', + ); + }); + return; + } + if ( + stage === 'invalid-config' && + /Refresh failed.*maxFileSize must be a positive integer/.test(output) + ) { + advance('second-large-file', () => + writeLargeSource('oversized-after-invalid.ts', 'stillSkipped'), + ); + return; + } + if ( + stage === 'second-large-file' && + /Refresh failed.*Configuration remains invalid/.test(output) + ) { + advance('recovered-config', () => { + fs.writeFileSync( + path.join(repo, '.gitnexusrc'), + JSON.stringify({ workers: '1', maxFileSize: '4096' }), + 'utf8', + ); + }); + return; + } + if ( + stage === 'recovered-config' && + /Refresh complete: [2-9][0-9]* changed, [1-9][0-9]* re-parsed,/.test(output) + ) { + stage = 'stopping'; + setTimeout(() => child.kill('SIGTERM'), 100); + } + }; + child.stderr.on('data', (chunk: Buffer) => { + const text = chunk.toString(); + stderr += text; + transcript += text; + handleOutput(); + }); + child.stdout.on('data', (chunk: Buffer) => { + const text = chunk.toString(); + stdout += text; + transcript += text; + handleOutput(); + }); + child.once('error', (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(error); + }); + child.once('close', (code, signal) => { + if (settled) return; + settled = true; + clearTimeout(timer); + const expectedWindowsTermination = + process.platform === 'win32' && code === null && signal === 'SIGTERM'; + if (code !== 0 && !expectedWindowsTermination) { + reject( + new Error( + `watch CLI exited ${code ?? signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`, + ), + ); + return; + } + expect(stage).toBe('stopping'); + expect(transcript).toContain('Refresh complete: 1 changed, 8 re-parsed,'); + resolve(); + }); + }); + + for (const [symbol, file] of [ + ['watchProof', 'watch-proof.ts'], + ['skippedByLimit', 'oversized-before.ts'], + ['stillSkipped', 'oversized-after-invalid.ts'], + ]) { + const result = runCliWithEnv( + ['context', symbol, '--file', file], + repo, + { GITNEXUS_HOME: home }, + 30_000, + ); + expect(result.status).toBe(0); + expect(result.stdout).toContain(symbol); + expect(result.stdout).toContain(file); + } + } finally { + cleanupTempDirSync(path.dirname(repo)); + cleanupTempDirSync(home); + } + }, 540_000); + it('fails with unknown command', () => { const result = runCliRaw(['nonexistent'], MINI_REPO); diff --git a/gitnexus/test/integration/watch-filesystem.test.ts b/gitnexus/test/integration/watch-filesystem.test.ts new file mode 100644 index 000000000..84fa4f8c3 --- /dev/null +++ b/gitnexus/test/integration/watch-filesystem.test.ts @@ -0,0 +1,230 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { startWatchFileLoop, type WatchFileLoop } from '../../src/cli/watch.js'; +import { cleanupTempDir } from '../helpers/test-db.js'; + +const tempDirs: string[] = []; +const loops: WatchFileLoop[] = []; + +async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for watcher event'); + await new Promise((resolve) => setTimeout(resolve, 25)); + } +} + +async function makeRepo(): Promise { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-watch-fs-')); + tempDirs.push(repo); + execFileSync('git', ['init', '-q'], { cwd: repo }); + return repo; +} + +afterEach(async () => { + await Promise.all(loops.splice(0).map((loop) => loop.close())); + await Promise.all(tempDirs.splice(0).map((dir) => cleanupTempDir(dir))); +}); + +describe('watch filesystem integration', () => { + it('fails startup and closes the watcher when the initial analysis fails', async () => { + const repo = await makeRepo(); + const onError = vi.fn(); + + await expect( + startWatchFileLoop( + repo, + 25, + async () => { + throw new Error('initial analysis failed'); + }, + onError, + ), + ).rejects.toThrow('initial analysis failed'); + expect(onError).not.toHaveBeenCalled(); + }); + + it('never enqueues analyzer-owned .gitnexus writes created by the initial refresh', async () => { + const repo = await makeRepo(); + const batches: string[][] = []; + const loop = await startWatchFileLoop( + repo, + 25, + async (paths) => { + batches.push([...paths]); + if (paths.length === 0) { + await fs.mkdir(path.join(repo, '.gitnexus'), { recursive: true }); + await fs.writeFile(path.join(repo, '.gitnexus', 'gitnexus.json'), '{}\n', 'utf8'); + await fs.writeFile(path.join(repo, '.gitnexus', 'lbug'), 'index bytes', 'utf8'); + } + }, + (error) => { + throw error; + }, + ); + loops.push(loop); + + await new Promise((resolve) => setTimeout(resolve, 200)); + await loop.waitForIdle(); + + expect(batches).toEqual([[]]); + }); + + it('coalesces indexed add/change/rename/delete events and stops cleanly', async () => { + const repo = await makeRepo(); + const batches: string[][] = []; + const loop = await startWatchFileLoop( + repo, + 30, + async (paths) => batches.push([...paths]), + (error) => { + throw error; + }, + ); + loops.push(loop); + expect(batches).toEqual([[]]); + + await fs.writeFile(path.join(repo, 'README.md'), '# One', 'utf8'); + await fs.writeFile(path.join(repo, 'src.ts'), 'export const one = 1;', 'utf8'); + await fs.writeFile(path.join(repo, 'src.ts'), 'export const one = 2;', 'utf8'); + await waitFor(() => batches.flat().includes('README.md') && batches.flat().includes('src.ts')); + + await fs.rename(path.join(repo, 'src.ts'), path.join(repo, 'renamed.ts')); + await waitFor(() => batches.flat().includes('renamed.ts')); + await fs.rm(path.join(repo, 'renamed.ts')); + await waitFor(() => batches.flat().filter((entry) => entry === 'renamed.ts').length >= 2); + + expect(batches.flat()).toEqual(expect.arrayContaining(['README.md', 'src.ts', 'renamed.ts'])); + await loop.close(); + loops.pop(); + const countAfterClose = batches.length; + await fs.writeFile(path.join(repo, 'after-close.ts'), 'export {};', 'utf8'); + await new Promise((resolve) => setTimeout(resolve, 150)); + expect(batches).toHaveLength(countAfterClose); + }); + + it('queues edits during refresh, recovers after failure, and ignores external symlinks', async () => { + const repo = await makeRepo(); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-watch-outside-')); + tempDirs.push(outside); + await fs.symlink( + outside, + path.join(repo, 'external'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + const successful: string[][] = []; + const errors: string[][] = []; + let failNext = false; + let releaseRefresh: (() => void) | undefined; + const loop = await startWatchFileLoop( + repo, + 25, + async (paths) => { + if (failNext) { + failNext = false; + throw new Error('injected refresh failure'); + } + successful.push([...paths]); + if (paths.includes('first.ts')) { + await new Promise((resolve) => { + releaseRefresh = resolve; + }); + } + }, + (_error, paths) => errors.push([...paths]), + ); + loops.push(loop); + + await fs.writeFile(path.join(repo, 'first.ts'), 'export const first = 1;', 'utf8'); + await waitFor(() => releaseRefresh !== undefined); + await fs.writeFile(path.join(repo, 'during.ts'), 'export const during = 1;', 'utf8'); + releaseRefresh!(); + await waitFor(() => successful.flat().includes('during.ts')); + + failNext = true; + await fs.writeFile(path.join(repo, 'fails.ts'), 'export const fail = 1;', 'utf8'); + await waitFor(() => errors.length === 1); + await waitFor(() => successful.flat().includes('fails.ts')); + await fs.writeFile(path.join(repo, 'retry.ts'), 'export const retry = 1;', 'utf8'); + await waitFor(() => successful.flat().includes('retry.ts')); + + const beforeExternal = successful.length + errors.length; + await fs.writeFile(path.join(outside, 'outside.ts'), 'export const outside = 1;', 'utf8'); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(successful.length + errors.length).toBe(beforeExternal); + }); + + it('reloads gitignore rules before processing subsequent file events', async () => { + const repo = await makeRepo(); + await fs.writeFile(path.join(repo, '.gitignore'), 'blocked.ts\n', 'utf8'); + const batches: string[][] = []; + const loop = await startWatchFileLoop( + repo, + 25, + async (paths) => batches.push([...paths]), + (error) => { + throw error; + }, + ); + loops.push(loop); + + await fs.writeFile(path.join(repo, 'blocked.ts'), 'export const blocked = 1;', 'utf8'); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(batches.flat()).not.toContain('blocked.ts'); + + await fs.writeFile(path.join(repo, '.gitignore'), '', 'utf8'); + await waitFor(() => batches.flat().includes('.gitignore')); + await fs.writeFile(path.join(repo, 'blocked.ts'), 'export const blocked = 2;', 'utf8'); + await waitFor(() => batches.flat().includes('blocked.ts')); + }); + + it('keeps the last valid ignore predicate after an oversized reload and later recovers', async () => { + const repo = await makeRepo(); + await fs.writeFile(path.join(repo, '.gitignore'), 'blocked.ts\n', 'utf8'); + const batches: string[][] = []; + const errors: string[][] = []; + const loop = await startWatchFileLoop( + repo, + 25, + async (paths) => batches.push([...paths]), + (_error, paths) => errors.push([...paths]), + ); + loops.push(loop); + + await fs.writeFile(path.join(repo, '.gitignore'), 'x'.repeat(1024 * 1024 + 1), 'utf8'); + await waitFor(() => errors.flat().includes('.gitignore')); + await waitFor(() => errors.length >= 2); + await fs.writeFile(path.join(repo, 'other.ts'), 'export const other = 1;', 'utf8'); + await waitFor(() => errors.flat().includes('other.ts')); + expect(batches.flat()).not.toContain('other.ts'); + await fs.writeFile(path.join(repo, 'blocked.ts'), 'export const blocked = 1;', 'utf8'); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(batches.flat()).not.toContain('blocked.ts'); + + await fs.writeFile(path.join(repo, '.gitignore'), '', 'utf8'); + await waitFor(() => batches.flat().includes('.gitignore')); + await fs.writeFile(path.join(repo, 'blocked.ts'), 'export const blocked = 2;', 'utf8'); + await waitFor(() => batches.flat().includes('blocked.ts')); + }); + + it('observes root control files even when gitignore excludes them', async () => { + const repo = await makeRepo(); + await fs.writeFile(path.join(repo, '.gitignore'), '.gitnexusrc\n', 'utf8'); + const batches: string[][] = []; + const loop = await startWatchFileLoop( + repo, + 25, + async (paths) => batches.push([...paths]), + (error) => { + throw error; + }, + ); + loops.push(loop); + + await fs.writeFile(path.join(repo, '.gitnexusrc'), '{}\n', 'utf8'); + await waitFor(() => batches.flat().includes('.gitnexusrc')); + }); +}); diff --git a/gitnexus/test/unit/analyze-config.test.ts b/gitnexus/test/unit/analyze-config.test.ts index 988c9cb87..60d4c545f 100644 --- a/gitnexus/test/unit/analyze-config.test.ts +++ b/gitnexus/test/unit/analyze-config.test.ts @@ -1,9 +1,12 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import fsSync from 'node:fs'; import fs from 'fs/promises'; import path from 'path'; import os from 'os'; import { loadAnalyzeConfig, + loadAnalyzeConfigStrict, mergeAnalyzeOptions, resolveDefaultBranch, validateBranchName, @@ -13,6 +16,10 @@ import { DEFAULT_BRANCH_FALLBACK, } from '../../src/cli/analyze-config.js'; import type { AnalyzeOptions } from '../../src/cli/analyze.js'; +import { + MAX_REPO_CONTROL_FILE_BYTES, + readRepoControlFile, +} from '../../src/config/repo-control-file.js'; describe('analyze-config (.gitnexusrc support, #243)', () => { let dir: string; @@ -34,6 +41,77 @@ describe('analyze-config (.gitnexusrc support, #243)', () => { expect(loadAnalyzeConfig(dir)).toBeUndefined(); }); + it('rejects an oversized repository config before parsing', async () => { + await writeRc(' '.repeat(MAX_REPO_CONTROL_FILE_BYTES + 1)); + await expect(loadAnalyzeConfigStrict(dir)).rejects.toThrow(/exceeds/); + }); + + it('keeps the read bounded if a control file grows after its size check', async () => { + await writeRc('{}'); + const fstatSync = fsSync.fstatSync; + const stat = vi.spyOn(fsSync, 'fstatSync').mockImplementation((fd) => { + const opened = fstatSync(fd); + Object.defineProperty(opened, 'size', { value: MAX_REPO_CONTROL_FILE_BYTES + 1 }); + return opened; + }); + + try { + await expect(readRepoControlFile(dir, GITNEXUS_RC_FILENAME)).rejects.toThrow(/exceeds/); + expect(stat).toHaveBeenCalledOnce(); + } finally { + stat.mockRestore(); + } + }); + + it('rejects a hardlinked repository config', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-rc-hardlink-')); + try { + const target = path.join(outside, 'config.json'); + await fs.writeFile(target, JSON.stringify({ workers: '8' })); + await fs.link(target, path.join(dir, GITNEXUS_RC_FILENAME)); + await expect(loadAnalyzeConfigStrict(dir)).rejects.toThrow(/hard link/); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it.skipIf(process.platform === 'win32')( + 'rejects a FIFO before opening it for reading', + async () => { + const fifo = path.join(dir, GITNEXUS_RC_FILENAME); + execFileSync('mkfifo', [fifo]); + let timeout: ReturnType | undefined; + + try { + await expect( + Promise.race([ + readRepoControlFile(dir, GITNEXUS_RC_FILENAME), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error('FIFO read did not fail promptly')), 500); + }), + ]), + ).rejects.toThrow(/regular file/); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } + }, + ); + + it.skipIf(process.platform === 'win32')( + 'rejects a final-file symlink for repository config', + async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-rc-outside-')); + try { + const target = path.join(outside, 'config.json'); + await fs.writeFile(target, JSON.stringify({ workers: '8' })); + await fs.symlink(target, path.join(dir, GITNEXUS_RC_FILENAME), 'file'); + await expect(loadAnalyzeConfigStrict(dir)).rejects.toThrow(/symbolic link/); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }, + ); + it('throws an actionable error on invalid JSON, naming the file', async () => { await writeRc('{ not valid json '); expect(() => loadAnalyzeConfig(dir)).toThrow(GitNexusRcError); diff --git a/gitnexus/test/unit/analyze-heap-respawn.test.ts b/gitnexus/test/unit/analyze-heap-respawn.test.ts index 53b684f9b..7176fb46d 100644 --- a/gitnexus/test/unit/analyze-heap-respawn.test.ts +++ b/gitnexus/test/unit/analyze-heap-respawn.test.ts @@ -220,6 +220,15 @@ describe('analyzeCommand heap respawn', () => { expect(parseMaxOldSpaceMb('--max-old-space-size --other-flag')).toBeNull(); }); + it('preserves conventional signal exits for analyze but treats watch shutdown as clean', async () => { + const { forwardedSignalExitCode } = await import('../../src/cli/analyze.js'); + expect(forwardedSignalExitCode('SIGINT', false)).toBe(130); + expect(forwardedSignalExitCode('SIGTERM', false)).toBe(143); + expect(forwardedSignalExitCode('SIGINT', true)).toBe(0); + expect(forwardedSignalExitCode('SIGTERM', true)).toBe(0); + expect(forwardedSignalExitCode('SIGABRT', false)).toBe(1); + }); + it('GITNEXUS_MEMORY=off also disables the default (unpinned) respawn (#2649 review)', async () => { delete process.env.NODE_OPTIONS; process.env.GITNEXUS_MEMORY = 'off'; diff --git a/gitnexus/test/unit/cross-platform-shard.test.ts b/gitnexus/test/unit/cross-platform-shard.test.ts index da88151f9..d5e83552e 100644 --- a/gitnexus/test/unit/cross-platform-shard.test.ts +++ b/gitnexus/test/unit/cross-platform-shard.test.ts @@ -3,7 +3,7 @@ * * The regression this guards is specific and was expensive: three CHEAP files * were registered in `SPAWN_CLI`, vitest re-partitioned the list by file COUNT, - * and the reshuffle clustered `cli-e2e` (361 s on Windows) with `cli-limit-e2e` + * and the reshuffle clustered `cli-e2e` (now 621 s on Windows) with `cli-limit-e2e` * (75 s) and `analyze-heap-oom-e2e` (23 s) on one shard, which then blew the * 20-minute watchdog. The added files cost nothing; the COUNT-split did it. * @@ -49,7 +49,7 @@ describe('cross-platform shard partition', () => { }); it('never puts the two heaviest suites on the same shard', () => { - // The exact shape of the outage: cli-e2e and worker-pool are 361 s and + // The exact shape of the outage: cli-e2e and worker-pool are 621 s and // 222 s, so together they are most of a shard's budget before anything else // is scheduled. const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL); diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index ff315faa6..eb792b701 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -842,6 +842,13 @@ describe('runFullAnalysis — incremental orchestration', () => { { onProgress: () => {} }, ); expect(incremental.alreadyUpToDate).toBeUndefined(); + expect(incremental.incrementalStats).toMatchObject({ + changedFiles: 1, + affectedDependents: 2, + deletedFiles: 0, + writeMode: 'incremental', + }); + expect(incremental.incrementalStats?.reparsedFiles).toBe(7); expect( querySpy.mock.calls.some( ([query]) => diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index b476f9743..8097c8ca5 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -748,4 +748,31 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { await rm(dir, { recursive: true, force: true }); } }); + + it('recreates a memoized shard directory after a long-lived process replaces it', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const firstKey = 'd'.repeat(64); + const secondKey = 'e'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set([firstKey]), + storagePath: dir, + onDiskKeys: new Set(), + }; + + await persistParseCacheChunk(cache, firstKey, [minimalResult({ fileCount: 1 })]); + await rm(path.join(dir, 'parse-cache'), { recursive: true, force: true }); + + cache.usedKeys = new Set([secondKey]); + await persistParseCacheChunk(cache, secondKey, [minimalResult({ fileCount: 2 })]); + await saveParseCache(dir, cache); + + const loaded = await loadParseCache(dir); + expect((await loadParseCacheChunk(loaded, secondKey))?.[0]?.fileCount).toBe(2); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/watch-failure-policy.test.ts b/gitnexus/test/unit/watch-failure-policy.test.ts new file mode 100644 index 000000000..aa32f0454 --- /dev/null +++ b/gitnexus/test/unit/watch-failure-policy.test.ts @@ -0,0 +1,29 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const analyzeFailureMayHaveMutatedLiveIndex = vi.hoisted(() => vi.fn()); + +vi.mock('../../src/core/run-analyze.js', () => ({ + analyzeFailureMayHaveMutatedLiveIndex, + runFullAnalysis: vi.fn(), +})); + +import { shouldStopAfterWatchRefreshFailure } from '../../src/cli/watch.js'; + +describe('watch refresh failure policy', () => { + beforeEach(() => analyzeFailureMayHaveMutatedLiveIndex.mockReset()); + + it('retries a queued pre-write failure even when incremental writes are in-place', () => { + const error = new Error('failed before live graph mutation'); + analyzeFailureMayHaveMutatedLiveIndex.mockReturnValue(false); + + expect(shouldStopAfterWatchRefreshFailure(error, ['src/a.ts'])).toBe(false); + }); + + it('stops only when a queued failure may have mutated the live graph', () => { + const error = new Error('failed during live graph mutation'); + analyzeFailureMayHaveMutatedLiveIndex.mockReturnValue(true); + + expect(shouldStopAfterWatchRefreshFailure(error, ['src/a.ts'])).toBe(true); + expect(shouldStopAfterWatchRefreshFailure(error, [])).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/watch-paths.test.ts b/gitnexus/test/unit/watch-paths.test.ts new file mode 100644 index 000000000..8accbb5ee --- /dev/null +++ b/gitnexus/test/unit/watch-paths.test.ts @@ -0,0 +1,180 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createWatchIgnorePredicate } from '../../src/config/ignore-service.js'; +import { isRelevantWatchPath, resolveWatchOptions } from '../../src/cli/watch.js'; +import * as git from '../../src/storage/git.js'; + +vi.mock('../../src/storage/git.js', () => ({ + getCoreExcludesFilePath: vi.fn(), + getGitInfoExcludePath: vi.fn(), +})); + +let repoPath: string; + +beforeEach(async () => { + repoPath = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-watch-')); + vi.mocked(git.getCoreExcludesFilePath).mockReturnValue(null); + vi.mocked(git.getGitInfoExcludePath).mockReturnValue(null); +}); + +afterEach(async () => { + await fs.rm(repoPath, { recursive: true, force: true }); +}); + +describe('watch path selection', () => { + it('accepts every scanner-admitted file instead of maintaining a second allow-list', () => { + expect(isRelevantWatchPath('src/service.ts')).toBe(true); + expect(isRelevantWatchPath('server/app.py')).toBe(true); + expect(isRelevantWatchPath('backend/project.csproj')).toBe(true); + expect(isRelevantWatchPath('.gitnexusrc')).toBe(true); + expect(isRelevantWatchPath('README.md')).toBe(true); + expect(isRelevantWatchPath('docs/guide.mdx')).toBe(true); + expect(isRelevantWatchPath('config/application-prod.yml')).toBe(true); + expect(isRelevantWatchPath('src/main/resources/application.properties')).toBe(true); + expect(isRelevantWatchPath('templates/page.html')).toBe(true); + expect(isRelevantWatchPath('templates/page.htm')).toBe(true); + expect(isRelevantWatchPath('views/page.ejs')).toBe(true); + expect(isRelevantWatchPath('views/page.hbs')).toBe(true); + expect(isRelevantWatchPath('views/page.blade.php')).toBe(true); + expect( + isRelevantWatchPath( + 'src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports', + ), + ).toBe(true); + expect(isRelevantWatchPath('src/main/resources/META-INF/spring.factories')).toBe(true); + expect(isRelevantWatchPath('tsconfig.base.json')).toBe(true); + expect(isRelevantWatchPath('packages/api/tsconfig.build.json')).toBe(true); + expect(isRelevantWatchPath('schema.sql')).toBe(true); + expect(isRelevantWatchPath('Dockerfile')).toBe(true); + expect(isRelevantWatchPath('assets/logo.png')).toBe(true); + expect(isRelevantWatchPath('../outside.ts')).toBe(false); + expect(isRelevantWatchPath('C:\\outside.ts')).toBe(false); + }); + + it('honors hardcoded, gitignore, and explicit-unignore rules', async () => { + await fs.writeFile( + path.join(repoPath, '.gitignore'), + ['generated/*', '!generated/', '!generated/keep.ts'].join('\n'), + ); + const ignored = await createWatchIgnorePredicate(repoPath); + + expect(ignored(path.join(repoPath, 'node_modules', 'pkg', 'index.ts'))).toBe(true); + expect(ignored(path.join(repoPath, 'generated'), true)).toBe(false); + expect(ignored(path.join(repoPath, 'generated', 'drop.ts'))).toBe(true); + expect(ignored(path.join(repoPath, 'generated', 'keep.ts'))).toBe(false); + expect(ignored(path.join(repoPath, 'src', 'keep.ts'))).toBe(false); + expect(ignored(path.resolve(repoPath, '..', 'outside.ts'))).toBe(true); + }); + + it('does not partially mutate environment state when a reloaded config is invalid', async () => { + const names = [ + 'GITNEXUS_MAX_FILE_SIZE', + 'GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', + 'GITNEXUS_VERBOSE', + ] as const; + const original = Object.fromEntries(names.map((name) => [name, process.env[name]])); + try { + await fs.writeFile( + path.join(repoPath, '.gitnexusrc'), + JSON.stringify({ maxFileSize: '2048', workerTimeout: '90', workers: '2' }), + ); + const baseline = { maxFileSize: '512', workerTimeout: '30000', verbose: undefined }; + await resolveWatchOptions(repoPath, {}, baseline); + expect(process.env.GITNEXUS_MAX_FILE_SIZE).toBe('2048'); + expect(process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS).toBe('90000'); + + await fs.writeFile( + path.join(repoPath, '.gitnexusrc'), + JSON.stringify({ maxFileSize: '4096', workerTimeout: '120', workers: '0' }), + ); + await expect(resolveWatchOptions(repoPath, {}, baseline)).rejects.toThrow( + '--workers must be a positive integer', + ); + expect(process.env.GITNEXUS_MAX_FILE_SIZE).toBe('2048'); + expect(process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS).toBe('90000'); + } finally { + for (const name of names) { + const value = original[name]; + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + }); + + it('ignores unsupported repository defaults but rejects explicit unsupported CLI flags', async () => { + await fs.writeFile( + path.join(repoPath, '.gitnexusrc'), + JSON.stringify({ + embeddings: true, + defaultBranch: 'develop', + skipAgentsMd: false, + skipSkills: false, + stats: true, + }), + ); + const ignored: string[][] = []; + await expect( + resolveWatchOptions( + repoPath, + {}, + { + maxFileSize: undefined, + workerTimeout: undefined, + verbose: undefined, + }, + (names) => ignored.push([...names]), + ), + ).resolves.toMatchObject({ skipAgentsMd: true, skipSkills: true }); + expect(ignored).toEqual([ + ['embeddings', 'defaultBranch', 'skipAgentsMd', 'skipSkills', 'stats'], + ]); + + const unsupportedCliOptions: Array<[Parameters[1], string]> = [ + [{ embeddings: true }, '--embeddings'], + [{ defaultBranch: 'develop' }, '--default-branch'], + [{ skipAgentsMd: true }, '--skip-agents-md'], + [{ skipSkills: true }, '--skip-skills'], + [{ stats: false }, '--no-stats'], + ]; + for (const [options, flag] of unsupportedCliOptions) { + await expect( + resolveWatchOptions(repoPath, options, { + maxFileSize: undefined, + workerTimeout: undefined, + verbose: undefined, + }), + ).rejects.toThrow(`analyze --watch does not support ${flag}`); + } + }); + + it('rejects a watch file-size threshold above the parser ceiling', async () => { + await expect( + resolveWatchOptions( + repoPath, + { maxFileSize: '32769' }, + { + maxFileSize: undefined, + workerTimeout: undefined, + verbose: undefined, + }, + ), + ).rejects.toThrow('maxFileSize must not exceed 32768'); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects repository ignore files that are final-file symlinks', + async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-watch-outside-')); + try { + const target = path.join(outside, 'ignore'); + await fs.writeFile(target, 'secret.ts\n'); + await fs.symlink(target, path.join(repoPath, '.gitignore'), 'file'); + await expect(createWatchIgnorePredicate(repoPath)).rejects.toThrow(/symbolic link/); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/gitnexus/test/unit/watch-queue.test.ts b/gitnexus/test/unit/watch-queue.test.ts new file mode 100644 index 000000000..260008a96 --- /dev/null +++ b/gitnexus/test/unit/watch-queue.test.ts @@ -0,0 +1,348 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { WATCH_FULL_REFRESH_PATH, WatchRefreshQueue } from '../../src/cli/watch-queue.js'; + +afterEach(() => vi.useRealTimers()); + +describe('WatchRefreshQueue', () => { + it('propagates an initial refresh failure without reporting it as retryable', async () => { + const onError = vi.fn(); + const queue = new WatchRefreshQueue( + async () => { + throw new Error('initial analyze failed'); + }, + onError, + 10, + ); + + await expect(queue.runInitial()).rejects.toThrow('initial analyze failed'); + expect(onError).not.toHaveBeenCalled(); + await queue.close(); + }); + + it('debounces and deduplicates rapid writes', async () => { + vi.useFakeTimers(); + const batches: readonly string[][] = []; + const mutable = batches as string[][]; + const queue = new WatchRefreshQueue( + async (paths) => mutable.push([...paths]), + () => {}, + 100, + ); + + queue.enqueue('src/a.ts'); + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(50); + queue.enqueue('src/b.ts'); + await vi.advanceTimersByTimeAsync(99); + expect(batches).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + await queue.waitForIdle(); + + expect(batches).toEqual([['src/a.ts', 'src/b.ts']]); + }); + + it('queues edits made during a refresh and never overlaps writers', async () => { + vi.useFakeTimers(); + let releaseFirst!: () => void; + let active = 0; + let peak = 0; + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => { + active++; + peak = Math.max(peak, active); + batches.push([...paths]); + if (batches.length === 1) await new Promise((resolve) => (releaseFirst = resolve)); + active--; + }, + () => {}, + 100, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(100); + queue.enqueue('src/b.ts'); + queue.enqueue('src/c.ts'); + releaseFirst(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(100); + await queue.waitForIdle(); + + expect(peak).toBe(1); + expect(batches).toEqual([['src/a.ts'], ['src/b.ts', 'src/c.ts']]); + }); + + it('retries a failed refresh without dropping its batch', async () => { + vi.useFakeTimers(); + const errors: string[][] = []; + const successful: string[][] = []; + let attempts = 0; + const queue = new WatchRefreshQueue( + async (paths) => { + attempts++; + if (attempts === 1) throw new Error('failed'); + successful.push([...paths]); + }, + (_error, paths) => errors.push([...paths]), + 10, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + expect(errors).toEqual([['src/a.ts']]); + expect(successful).toEqual([]); + await vi.advanceTimersByTimeAsync(250); + await queue.waitForIdle(); + + expect(errors).toEqual([['src/a.ts']]); + expect(successful).toEqual([['src/a.ts']]); + }); + + it('parks pre-ready events until the initial refresh completes', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => batches.push([...paths]), + () => {}, + 50, + { holdEventsUntilInitialRefresh: true }, + ); + + queue.enqueue('src/during-walk.ts'); + await vi.advanceTimersByTimeAsync(80); + expect(batches).toEqual([]); + + await queue.runInitial(); + expect(batches).toEqual([[]]); + await vi.advanceTimersByTimeAsync(50); + await queue.waitForIdle(); + expect(batches).toEqual([[], ['src/during-walk.ts']]); + }); + + it('backs off repeated failures instead of spinning at the debounce interval', async () => { + vi.useFakeTimers(); + let attempts = 0; + const queue = new WatchRefreshQueue( + async () => { + attempts++; + if (attempts < 4) throw new Error('still unavailable'); + }, + () => {}, + 10, + { retryBaseDelayMs: 100, retryMaxDelayMs: 400 }, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + expect(attempts).toBe(1); + await vi.advanceTimersByTimeAsync(99); + expect(attempts).toBe(1); + await vi.advanceTimersByTimeAsync(1); + expect(attempts).toBe(2); + await vi.advanceTimersByTimeAsync(199); + expect(attempts).toBe(2); + await vi.advanceTimersByTimeAsync(1); + expect(attempts).toBe(3); + await vi.advanceTimersByTimeAsync(399); + expect(attempts).toBe(3); + await vi.advanceTimersByTimeAsync(1); + await queue.waitForIdle(); + expect(attempts).toBe(4); + }); + + it('merges an event during retry backoff without shortening the retry delay', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + let attempts = 0; + const queue = new WatchRefreshQueue( + async (paths) => { + attempts++; + if (attempts === 1) throw new Error('failed'); + batches.push([...paths]); + }, + () => {}, + 10, + { retryBaseDelayMs: 1_000 }, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + expect(attempts).toBe(1); + + await vi.advanceTimersByTimeAsync(100); + queue.enqueue('src/b.ts'); + await vi.advanceTimersByTimeAsync(899); + expect(attempts).toBe(1); + + await vi.advanceTimersByTimeAsync(1); + await queue.waitForIdle(); + + expect(attempts).toBe(2); + expect(batches).toEqual([['src/a.ts', 'src/b.ts']]); + }); + + it('contains a throwing error reporter for a detached refresh', async () => { + vi.useFakeTimers(); + const queue = new WatchRefreshQueue( + async () => { + throw new Error('refresh failed'); + }, + async () => { + throw new Error('reporting failed'); + }, + 10, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + + await queue.close(); + await expect(queue.waitForIdle()).resolves.toBeUndefined(); + }); + + it('contains a synchronously throwing refresh and retries its batch', async () => { + vi.useFakeTimers(); + const errors: string[][] = []; + const successful: string[][] = []; + let attempts = 0; + const queue = new WatchRefreshQueue( + (paths) => { + attempts++; + if (attempts === 1) throw new Error('synchronous refresh failure'); + successful.push([...paths]); + return Promise.resolve(); + }, + (_error, paths) => errors.push([...paths]), + 10, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + expect(errors).toEqual([['src/a.ts']]); + await vi.advanceTimersByTimeAsync(250); + await queue.waitForIdle(); + + expect(attempts).toBe(2); + expect(successful).toEqual([['src/a.ts']]); + }); + + it('closes cleanly when a refresh failure triggers shutdown', async () => { + vi.useFakeTimers(); + let closePromise: Promise | undefined; + const queue = new WatchRefreshQueue( + async () => { + throw new Error('stop after failure'); + }, + () => { + closePromise = queue.close(); + }, + 10, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + + expect(closePromise).toBeDefined(); + await expect(closePromise).resolves.toBeUndefined(); + }); + + it('flushes by max wait even when writes never become quiet', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => batches.push([...paths]), + () => {}, + 100, + { maxWaitMs: 250 }, + ); + + queue.enqueue('src/0.ts'); + await vi.advanceTimersByTimeAsync(90); + queue.enqueue('src/1.ts'); + await vi.advanceTimersByTimeAsync(90); + queue.enqueue('src/2.ts'); + await vi.advanceTimersByTimeAsync(70); + await queue.waitForIdle(); + + expect(batches).toEqual([['src/0.ts', 'src/1.ts', 'src/2.ts']]); + }); + + it('bounds high-cardinality paths while retaining priority control files', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => batches.push([...paths]), + () => {}, + 10, + { + maxPendingPaths: 2, + isPriorityPath: (filePath) => filePath === '.gitnexusrc', + }, + ); + + for (let index = 0; index < 20; index++) queue.enqueue(`src/${index}.ts`); + queue.enqueue('.gitnexusrc'); + await vi.advanceTimersByTimeAsync(10); + await queue.waitForIdle(); + + expect(batches).toEqual([[WATCH_FULL_REFRESH_PATH, '.gitnexusrc', 'src/1.ts']]); + }); + + it('bounds a flood of distinct priority paths', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => batches.push([...paths]), + () => {}, + 10, + { + maxPendingPaths: 2, + isPriorityPath: (filePath) => filePath.endsWith('/.gitignore'), + }, + ); + + for (let index = 0; index < 20; index++) queue.enqueue(`packages/${index}/.gitignore`); + await vi.advanceTimersByTimeAsync(10); + await queue.waitForIdle(); + + expect(batches).toEqual([ + [WATCH_FULL_REFRESH_PATH, 'packages/0/.gitignore', 'packages/1/.gitignore'], + ]); + }); + + it('does not report overflow for a duplicate at capacity', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => batches.push([...paths]), + () => {}, + 10, + { maxPendingPaths: 2 }, + ); + + queue.enqueue('src/a.ts'); + queue.enqueue('src/b.ts'); + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + await queue.waitForIdle(); + + expect(batches).toEqual([['src/a.ts', 'src/b.ts']]); + }); + + it('runs a full refresh when the pending-path limit is zero', async () => { + vi.useFakeTimers(); + const batches: string[][] = []; + const queue = new WatchRefreshQueue( + async (paths) => batches.push([...paths]), + () => {}, + 10, + { maxPendingPaths: 0 }, + ); + + queue.enqueue('src/a.ts'); + await vi.advanceTimersByTimeAsync(10); + await queue.waitForIdle(); + + expect(batches).toEqual([[WATCH_FULL_REFRESH_PATH]]); + }); +}); From b7850e669546a79d66e8ab1da85a67e6f64f1fe9 Mon Sep 17 00:00:00 2001 From: guyua9 Date: Sat, 29 Aug 2026 19:33:22 +0800 Subject: [PATCH 19/28] fix(cursor): preserve quoted shell search patterns (#2938) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cursor): preserve quoted shell search patterns * fix: preserve backslashes in quoted shell patterns * fix(cursor): parse attached regexp options * fix(cursor): parse attached regexp options * fix(cursor): honor rg end-of-options marker * fix(cursor): scan repeated regexp options (#2938) Keep parsing after short explicit patterns so later eligible regexps are selected without mistaking path operands for search terms. Co-authored-by: Cursor * fix(cursor): harden shell search pattern parsing (#2938) Keep unquoted Windows backslashes so rg.exe paths still parse, skip pattern-file operands, and treat grep -r as recursive rather than a valued replace flag. Co-authored-by: Cursor --------- Co-authored-by: luyua9 Co-authored-by: Gergő Magyar Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- .../hooks/gitnexus-hook.cjs | 221 +++++++++++++++++- gitnexus/test/unit/cursor-hook.test.ts | 103 +++++--- 2 files changed, 284 insertions(+), 40 deletions(-) diff --git a/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs b/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs index e68aca1de..564384f83 100644 --- a/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs +++ b/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs @@ -85,40 +85,241 @@ function findGitNexusDir(startDir) { return null; } +function tokenizeShellWords(command) { + const tokens = []; + let current = ''; + let quote = null; + let escaped = false; + let hasToken = false; + + for (let index = 0; index < command.length; index += 1) { + const char = command[index]; + if (escaped) { + current += char; + escaped = false; + hasToken = true; + continue; + } + + if (quote === "'") { + if (char === "'") quote = null; + else current += char; + hasToken = true; + continue; + } + + if (quote === '"') { + if (char === '"') { + quote = null; + } else if (char === '\\') { + const next = command[index + 1]; + if (next === '$' || next === '`' || next === '"' || next === '\\') { + escaped = true; + } else { + current += '\\'; + } + } else { + current += char; + } + hasToken = true; + continue; + } + + if (char === '\\') { + const next = command[index + 1]; + if (next === undefined || /\s/.test(next) || next === "'" || next === '"' || next === '\\') { + escaped = true; + } else { + current += '\\' + next; + index += 1; + } + hasToken = true; + } else if (char === "'" || char === '"') { + quote = char; + hasToken = true; + } else if (/\s/.test(char)) { + if (hasToken) tokens.push(current); + current = ''; + hasToken = false; + } else if (char === ';' || char === '|' || char === '&') { + if (hasToken) tokens.push(current); + current = ''; + hasToken = false; + const next = command[index + 1]; + if ((char === '|' || char === '&') && next === char) { + tokens.push(char + char); + index += 1; + } else { + tokens.push(char); + } + } else { + current += char; + hasToken = true; + } + } + + if (escaped) current += '\\'; + if (hasToken) tokens.push(current); + return tokens; +} + function parseRgGrepPattern(cmd) { - const tokens = cmd.split(/\s+/); + const tokens = tokenizeShellWords(cmd); let foundCmd = false; let skipNext = false; + let skipNextAsPattern = false; + let endOfOptions = false; + let explicitPatternSeen = false; + let patternFileSeen = false; const flagsWithValues = new Set([ '-e', '-f', + '--file', '-m', + '--max-count', '-A', '-B', '-C', '-g', '--glob', + '--iglob', '-t', '--type', '--include', '--exclude', + '--encoding', + '--path', ]); + const rgValueFlags = new Set(['-r', '--replace']); + const patternFlags = new Set(['-e', '--regexp']); + const connectors = new Set(['&&', '||', ';', '|', '&']); + const wrappers = new Set([ + 'npx', + 'bunx', + 'pnpm', + 'yarn', + 'npm', + 'sudo', + 'env', + 'command', + 'time', + 'nice', + 'xargs', + 'dlx', + 'exec', + 'run', + 'git', + ]); + const wrapperFlagsWithValues = new Set([ + '--package', + '-p', + '--call', + '--prefix', + '--shell', + '--filter', + '--workspace', + '--dir', + '--cwd', + ]); + const basename = (token) => + token + .split(/[\\/]/) + .pop() + ?.replace(/\.(exe|cmd|bat)$/i, ''); + let previousToken; + let seenWrapper = false; + let searchCommand = null; for (const token of tokens) { if (skipNext) { skipNext = false; + if (skipNextAsPattern) { + skipNextAsPattern = false; + if (token.length >= 3) return token; + } + previousToken = token; continue; } if (!foundCmd) { - if (/\brg$|\bgrep$/.test(token)) foundCmd = true; + if (connectors.has(token)) { + seenWrapper = false; + previousToken = token; + continue; + } + const commandName = basename(token); + if (wrappers.has(commandName)) { + seenWrapper = true; + previousToken = token; + continue; + } + if (seenWrapper && token.startsWith('-')) { + const flagName = token.split('=', 1)[0]; + if (!token.includes('=') && wrapperFlagsWithValues.has(flagName)) skipNext = true; + previousToken = token; + continue; + } + if (seenWrapper && /^[A-Za-z_][A-Za-z0-9_]*=/.test(token)) { + previousToken = token; + continue; + } + const atCommandPosition = + previousToken === undefined || + connectors.has(previousToken) || + wrappers.has(basename(previousToken)) || + seenWrapper; + if (atCommandPosition && (commandName === 'rg' || commandName === 'grep')) { + foundCmd = true; + searchCommand = commandName; + } else if (seenWrapper) { + seenWrapper = false; + } + previousToken = token; + continue; + } + previousToken = token; + if (endOfOptions) { + if (explicitPatternSeen || patternFileSeen) continue; + return token.length >= 3 ? token : null; + } + if (token === '--') { + endOfOptions = true; continue; } if (token.startsWith('-')) { - if (flagsWithValues.has(token)) skipNext = true; + if (token === '-f' || token === '--file') { + skipNext = true; + patternFileSeen = true; + continue; + } + if (token.startsWith('--file=')) { + patternFileSeen = true; + continue; + } + if (token.startsWith('--regexp=')) { + explicitPatternSeen = true; + const value = token.slice('--regexp='.length); + if (value.length >= 3) return value; + continue; + } + const attachedPattern = token.match(/^-e(.+)$/); + if (attachedPattern) { + explicitPatternSeen = true; + if (attachedPattern[1].length >= 3) return attachedPattern[1]; + continue; + } + if ( + flagsWithValues.has(token) || + patternFlags.has(token) || + (searchCommand === 'rg' && rgValueFlags.has(token)) + ) { + skipNext = true; + skipNextAsPattern = patternFlags.has(token); + if (skipNextAsPattern) explicitPatternSeen = true; + } continue; } - const cleaned = token.replace(/['"]/g, ''); - return cleaned.length >= 3 ? cleaned : null; + if (explicitPatternSeen || patternFileSeen) continue; + return token.length >= 3 ? token : null; } return null; } @@ -179,12 +380,6 @@ function extractPattern(toolName, toolInput) { if (t === 'shell') { const cmd = toolInput.command || ''; if (!/\brg\b|\bgrep\b/.test(cmd)) return null; - // NOTE: parseRgGrepPattern uses split(/\s+/) and cannot handle shell - // quoting. `rg "User Service" src/` returns "User" (the first token - // after the rg/grep arg, with surrounding quotes stripped) — the - // multi-word pattern is intentionally not reconstructed since BM25 is - // already token-tolerant. Quoted single tokens (`rg "validateUser"`) - // work fine. return parseRgGrepPattern(cmd); } @@ -282,4 +477,6 @@ function main() { } } -main(); +if (require.main === module) main(); + +module.exports = { parseRgGrepPattern, tokenizeShellWords }; diff --git a/gitnexus/test/unit/cursor-hook.test.ts b/gitnexus/test/unit/cursor-hook.test.ts index 54b5c1f84..dc9bc423b 100644 --- a/gitnexus/test/unit/cursor-hook.test.ts +++ b/gitnexus/test/unit/cursor-hook.test.ts @@ -19,6 +19,7 @@ */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { spawnSync } from 'child_process'; +import { createRequire } from 'module'; import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -55,6 +56,12 @@ const CURSOR_HOOKS_JSON = path.resolve( 'hooks.json', ); +const require = createRequire(import.meta.url); +const { parseRgGrepPattern, tokenizeShellWords } = require(CURSOR_HOOK) as { + parseRgGrepPattern: (command: string) => string | null; + tokenizeShellWords: (command: string) => string[]; +}; + // ─── Cursor-specific output parser ────────────────────────────────── // Cursor postToolUse output shape: { "additional_context": "..." } @@ -549,37 +556,77 @@ describe('Cursor hook concurrency guard (integration)', () => { }); }); -// ─── Documented contract behavior (extractPattern via the live hook) ─ +// ─── Shell pattern parsing ────────────────────────────────────────── -describe('Shell quoted-pattern parser limitations (documented)', () => { - // The Shell parser cannot reconstruct shell quoting. These tests pin the - // current behavior so a future "fix" doesn't silently change extraction - // — and so users diagnosing a noisy/missed pattern can find the behavior - // documented in tests. - // - // We can't observe the extracted pattern directly without an indexed - // repo, but we *can* confirm the hook reaches the augment-call path - // (vs. early-exiting) by checking exit status + clean stdout for cases - // where parseRgGrepPattern would yield a >=3-char token. - - it('quoted multi-word `rg "User Service"` extracts the first word only', () => { - const result = runHook(CURSOR_HOOK, { - tool_name: 'Shell', - tool_input: { command: 'rg "User Service" src/' }, - cwd: tmpDir, // no .gitnexus → exits early after extract - }); - expect(result.status).toBe(0); - expect(result.stdout.trim()).toBe(''); +describe('Shell quoted-pattern parser', () => { + it.each([ + ['rg "User Service" src/', 'User Service'], + ["grep 'error boundary' -- src/", 'error boundary'], + ['rg User\\ Service src/', 'User Service'], + [String.raw`rg "C:\Users" src/`, String.raw`C:\Users`], + ['rg -e "User Service" src/', 'User Service'], + ['rg --regexp=UserService src/', 'UserService'], + ['grep -eUserService src/', 'UserService'], + ['rg -e x -e LongPattern src/', 'LongPattern'], + ['rg -ex -eLongPattern src/', 'LongPattern'], + ['rg --regexp=x --regexp=LongPattern src/', 'LongPattern'], + ['/usr/bin/rg -- "User Service" src/', 'User Service'], + ['rg -- -error src/', '-error'], + [String.raw`C:\Users\me\bin\rg.exe UserService src/`, 'UserService'], + ['rg.exe "validateUser" src/', 'validateUser'], + ['grep.cmd -e LongPattern src/', 'LongPattern'], + ['cd grep && rg LongPattern src/', 'LongPattern'], + ['npx rg "User Service" src/', 'User Service'], + ['npx --yes rg UserService src/', 'UserService'], + ['npx --package rg grep LongPattern src/', 'LongPattern'], + ['rg UserService; echo done', 'UserService'], + ['rg UserService&& echo done', 'UserService'], + ['rg --max-count 100 UserService src/', 'UserService'], + ['grep -r UserService src/', 'UserService'], + ['rg --replace x UserService src/', 'UserService'], + ['git grep UserService src/', 'UserService'], + ])('extracts %j from %j', (command, expected) => { + expect(parseRgGrepPattern(command)).toBe(expected); }); - it('single-token quoted `rg "validateUser"` works as expected', () => { - const result = runHook(CURSOR_HOOK, { - tool_name: 'Shell', - tool_input: { command: 'rg "validateUser"' }, - cwd: tmpDir, - }); - expect(result.status).toBe(0); - expect(result.stdout.trim()).toBe(''); + it.each([ + ['rg --regexp= src/'], + ['rg --regexp="" src/'], + ['rg -e x -- LongPattern src/'], + ['rg -f patterns.txt src/'], + ['rg --file=patterns.txt src/'], + ['rg -eab src/'], + ['sudo echo rg UserService src/'], + ])('extracts no pattern from %j', (command) => { + expect(parseRgGrepPattern(command)).toBeNull(); + }); + + it('does not treat a path after a short explicit pattern as the pattern', () => { + expect(parseRgGrepPattern('rg -e x src/')).toBeNull(); + }); + + it('keeps single-token quoted patterns intact', () => { + expect(parseRgGrepPattern('rg "validateUser"')).toBe('validateUser'); + }); + + it('keeps backslashes in unquoted Windows paths but honours escaped spaces', () => { + expect(tokenizeShellWords(String.raw`C:\foo\bar`)).toEqual([String.raw`C:\foo\bar`]); + expect(tokenizeShellWords('User\\ Service')).toEqual(['User Service']); + expect(tokenizeShellWords('trailing\\')).toEqual(['trailing\\']); + }); + + it('splits unquoted shell operators from adjacent arguments', () => { + expect(tokenizeShellWords('rg UserService; echo done')).toEqual([ + 'rg', + 'UserService', + ';', + 'echo', + 'done', + ]); + expect(tokenizeShellWords("rg 'UserService; echo done'")).toEqual([ + 'rg', + 'UserService; echo done', + ]); }); }); From 54f97c86c7d938e7a7bbf3652645561b66dc38cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 29 Aug 2026 12:58:15 +0100 Subject: [PATCH 20/28] fix(impact): make File risk comparable via shared axes (#3075) (#3082) * docs(plans): add impact file risk plan Capture the evidence, constraints, and verification path for fixing incomparable File and symbol impact risk. Co-authored-by: Cursor * refactor(impact): centralize risk scoring Keep the existing thresholds in one shared scorer and expose a common-axis comparison for targets with unavailable enrichment axes. Co-authored-by: Cursor * fix(impact): expose incomparable file risk scale Mark File impact results when process and module axes are unavailable, and provide a common-axis score for honest cross-kind comparisons. Co-authored-by: Cursor * fix(impact): explain cross-kind risk comparisons Surface the common-axis score in CLI and agent guidance while reusing the shared threshold ladder in the web impact tool. Co-authored-by: Cursor * fix(impact): fail closed when enrichment is incomplete Preserve proved HIGH/CRITICAL process counts, treat failed queries as UNKNOWN, and surface riskScale metadata on MCP, group, CLI, and Graph-RAG File walks. Co-authored-by: Cursor --------- Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- .../skills/gitnexus-impact-analysis/SKILL.md | 9 + AGENTS.md | 6 +- CLAUDE.md | 6 +- ...26-08-28-gitnexus-plan-impact-file-risk.md | 312 +++++++++++++++++ .../skills/gitnexus-impact-analysis/SKILL.md | 9 + .../skills/gitnexus-impact-analysis/SKILL.md | 9 + gitnexus-shared/src/impact-risk.ts | 66 ++-- gitnexus-web/src/core/llm/tools.ts | 111 ++++-- gitnexus-web/test/unit/impact-tool.test.ts | 209 +++++++++++ gitnexus/skills/gitnexus-impact-analysis.md | 9 + gitnexus/src/cli/ai-context.ts | 8 +- gitnexus/src/cli/eval-server.ts | 30 +- gitnexus/src/core/group/cross-impact.ts | 36 +- gitnexus/src/core/group/types.ts | 14 +- gitnexus/src/mcp/local/local-backend.ts | 127 ++++--- gitnexus/src/mcp/tools.ts | 6 +- .../impact-file-risk-scale.test.ts | 143 ++++++++ .../impact-zero-caller-risk.test.ts | 7 + .../ai-context-unknown-risk-policy.test.ts | 3 + gitnexus/test/unit/ai-context.test.ts | 10 + .../test/unit/cli-impact-pdg-format.test.ts | 7 +- gitnexus/test/unit/eval-formatters.test.ts | 46 ++- .../group/cross-impact-fanout-cap.test.ts | 26 ++ gitnexus/test/unit/group/cross-impact.test.ts | 29 ++ gitnexus/test/unit/group/types.test.ts | 17 + .../unit/impact-batching-grouping.test.ts | 327 +++++++++++++++++- gitnexus/test/unit/impact-risk.test.ts | 237 +++++++++++++ .../test/unit/shipped-skills-sync.test.ts | 19 + gitnexus/test/unit/tools.test.ts | 11 + 29 files changed, 1725 insertions(+), 124 deletions(-) create mode 100644 docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md create mode 100644 gitnexus-web/test/unit/impact-tool.test.ts create mode 100644 gitnexus/test/integration/impact-file-risk-scale.test.ts create mode 100644 gitnexus/test/unit/impact-risk.test.ts diff --git a/.claude/skills/gitnexus-impact-analysis/SKILL.md b/.claude/skills/gitnexus-impact-analysis/SKILL.md index 4fb73f3e6..85d90c90d 100644 --- a/.claude/skills/gitnexus-impact-analysis/SKILL.md +++ b/.claude/skills/gitnexus-impact-analysis/SKILL.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/AGENTS.md b/AGENTS.md index c83a2e909..05be52af2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,10 +119,10 @@ This project is indexed by GitNexus as **GitNexus** (248612 symbols, 565510 rela - **MUST run impact analysis before editing.** Use `impact({target: "symbolName", direction: "upstream"})` (MCP) or `node .gitnexus/run.cjs impact "symbolName" --direction upstream --repo .` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis. For unified PDG impact, add `mode: "pdg"` with optional `line: ` — it returns statement-level `affectedStatements` over CDG + REACHING_DEF and inter-procedural symbols in `interproceduralByDepth`/`byDepth`; no-layer/degraded PDG results are UNKNOWN-risk notes (`--pdg` layer). CLI equivalent: `node .gitnexus/run.cjs impact "symbolName" --direction upstream --mode pdg --line --repo .`. - **MUST analyze graph changes before committing.** Use `detect_changes({scope: "all"})` (MCP) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` (CLI fallback). `partial: true` or `truncated: true` is not a clean check — a zero means unseen, not unaffected; re-run it. For regression review: `detect_changes({scope: "compare", base_ref: "main"})` or `node .gitnexus/run.cjs detect-changes --scope compare --base-ref "main" --repo .`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- MUST warn on HIGH/CRITICAL `risk` pre-edit; never use `riskSharedAxes` to waive a HIGH/CRITICAL `risk` warning. Compare File/symbol: MCP File omits axes; Graph-RAG expands File. - **MUST treat `risk: UNKNOWN` as unresolved, not as low.** An empty caller set is not evidence the symbol is unused — it can also mean the callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls). `impact` pairs `UNKNOWN` with a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete; do not proceed on the strength of a zero. -- When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. +- Explore with `query({search_query: "concept"})` for process-grouped flows. +- Use `context({name: "symbolName"})` for callers, callees, and flows. - For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). - For control/data dependence, `pdg_query({mode: "controls", target: "fileOrSymbol"})` answers "under what condition does X run?" (CDG, incl. guard clauses) and `pdg_query({mode: "flows", target, variable})` traces "where does variable Y flow?" (REACHING_DEF). `--pdg` layer. diff --git a/CLAUDE.md b/CLAUDE.md index 55b84d583..10681d819 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,10 +70,10 @@ This project is indexed by GitNexus as **GitNexus** (248612 symbols, 565510 rela - **MUST run impact analysis before editing.** Use `impact({target: "symbolName", direction: "upstream"})` (MCP) or `node .gitnexus/run.cjs impact "symbolName" --direction upstream --repo .` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis. For unified PDG impact, add `mode: "pdg"` with optional `line: ` — it returns statement-level `affectedStatements` over CDG + REACHING_DEF and inter-procedural symbols in `interproceduralByDepth`/`byDepth`; no-layer/degraded PDG results are UNKNOWN-risk notes (`--pdg` layer). CLI equivalent: `node .gitnexus/run.cjs impact "symbolName" --direction upstream --mode pdg --line --repo .`. - **MUST analyze graph changes before committing.** Use `detect_changes({scope: "all"})` (MCP) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` (CLI fallback). `partial: true` or `truncated: true` is not a clean check — a zero means unseen, not unaffected; re-run it. For regression review: `detect_changes({scope: "compare", base_ref: "main"})` or `node .gitnexus/run.cjs detect-changes --scope compare --base-ref "main" --repo .`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- MUST warn on HIGH/CRITICAL `risk` pre-edit; never use `riskSharedAxes` to waive a HIGH/CRITICAL `risk` warning. Compare File/symbol: MCP File omits axes; Graph-RAG expands File. - **MUST treat `risk: UNKNOWN` as unresolved, not as low.** An empty caller set is not evidence the symbol is unused — it can also mean the callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls). `impact` pairs `UNKNOWN` with a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete; do not proceed on the strength of a zero. -- When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. +- Explore with `query({search_query: "concept"})` for process-grouped flows. +- Use `context({name: "symbolName"})` for callers, callees, and flows. - For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). - For control/data dependence, `pdg_query({mode: "controls", target: "fileOrSymbol"})` answers "under what condition does X run?" (CDG, incl. guard clauses) and `pdg_query({mode: "flows", target, variable})` traces "where does variable Y flow?" (REACHING_DEF). `--pdg` layer. diff --git a/docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md b/docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md new file mode 100644 index 000000000..7e14f2174 --- /dev/null +++ b/docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md @@ -0,0 +1,312 @@ +# GitNexus Engineering Plan + +> Task: Fix #3075 — File `impact` risk is not comparable to Function/Method risk. +> Evidence verified at commit `6bff33d14cbfe1e7b4f04bca51507e9f64ef579c` (`feat/kotlin-const-resolver`); GitNexus index 129 commits behind, refresh skipped: full-repo `--index-only --pdg` rebuild is impractical this session. Scorer and schema claims are `[verified]` from source; live inversion numbers are `[graph]` on the stale index. + +## 1. Objective + +Make File vs symbol `impact.risk` honest for consumers: either they can tell the scales differ, or they can compare on a shared two-axis score. Do **not** DEFINES-bridge processes/modules onto File targets (issue reporter sampled 8/10 one-importer files jumping to HIGH/CRITICAL). Do **not** retune Function HIGH/CRITICAL thresholds (agent warn-before-edit). + +Acceptance: + +- A File with a wider blast radius than a Function in the same file no longer looks “safer” when a consumer only reads `risk`, **or** the result states that `risk` is not comparable across kinds and offers `riskSharedAxes` for comparison. +- File targets still cannot trip HIGH/CRITICAL via `processes_affected` / `modules_affected` unless those axes become real in the index (they are not today). +- Existing Function/Method labels under the current four-axis ladder stay the same for the same inputs. +- MCP `riskNote` remains UNKNOWN-only (`tools.ts` contract). + +## 2. Current Behaviour + +Callgraph `impact` ends in `LocalBackend._runImpactBFS` (`gitnexus/src/mcp/local/local-backend.ts`). After BFS it enriches impacted ids with `STEP_IN_PROCESS` and `MEMBER_OF`, then scores: + +```7720:7738:gitnexus/src/mcp/local/local-backend.ts + } else if ( + directCount >= 30 || + processCount >= 5 || + moduleCount >= 5 || + impacted.length >= 200 + ) { + risk = 'CRITICAL'; + } else if ( + directCount >= 15 || + processCount >= 3 || + moduleCount >= 3 || + impacted.length >= 100 + ) { + risk = 'HIGH'; + } else if (directCount >= 5 || impacted.length >= 30) { + risk = 'MEDIUM'; + } else { + risk = 'LOW'; + } +``` + +Empty upstream → `UNKNOWN` + `riskNote`. Downstream empty stays LOW. `skipEnrichment` (ambiguous probes) already scores on direct+total only. PDG mode forces `risk: UNKNOWN` (`composeUnifiedPdgImpactResult`) — out of scope. + +File BFS walk is mostly File←IMPORTS File. Enrichment queries those File ids. Processes are CALLS traces (`process-processor.ts`); communities admit only Function/Class/Method/Interface (`isCommunitySymbol` in `community-processor.ts:412-416`). File is not in that set. `enrichCandidateLabels` UNION also **omits File**, so File `target.type` is often `""`; detect File via `id` prefix `File:`. + +Web Graph RAG (`gitnexus-web/src/core/llm/tools.ts` ~1331–1346) duplicates the same ladder. + +## 3. Relevant Architecture + +| Layer | Role | +|---|---| +| Index | File never sources `STEP_IN_PROCESS` / `MEMBER_OF` by construction | +| MCP `_runImpactBFS` | Blast radius + four-axis `risk` | +| Ambiguous probes | `skipEnrichment` → 2-axis `risk` already | +| `mergeRisk` | Group overlay; monotone in crossings; does not know target kind | +| CLI `formatImpactResult` | Prints counts; **does not print `risk`** on the resolved callgraph path; JSON `impactCommand` still ships `risk` | +| `ai-context.ts` / `tools.ts` | Agent contract: warn on HIGH/CRITICAL; `riskNote` UNKNOWN-only | +| Web LLM `impact` | Same formula, prose `RISK:` line | + +Modules: Local (MCP), Cli (format/docs), Group (`mergeRisk`), gitnexus-web LLM tools. Shared package `gitnexus-shared` is already a dependency of both CLI and web. + +## 4. GitNexus Findings + +- Primary: `_runImpactBFS` — d=1 `[graph]` `impact(target:_runImpactBFS, maxDepth:1, includeTests:true)`: `_impactImpl`, `impactByUid`. Production chain `[verified]`: `impact` → `_impactImpl` → `_runImpactBFS`; `impactByUid` skips per-symbol process lists but **not** aggregation (`skipPerSymbolEnrichment` only). +- `LocalBackend.impact` d=1 `[graph]` `context`: `callTool`. +- Duplicate scorer `[verified]` grep: `gitnexus-web/src/core/llm/tools.ts`. +- `mergeRisk` `[verified]` callers in `src/`: only `runGroupImpact` (`cross-impact.ts:907`). Graph d=1 listed a test File (`impact-pdg-shape.test.ts`) and missed `runGroupImpact` — trust source. +- Schema `[verified]`: `isCommunitySymbol` excludes File; `schema.ts` documents MEMBER_OF as Function/Class/Method/Interface only. +- Live inversion `[graph]` stale index, `impact summaryOnly` on GitNexus: + +| target | kind | impacted | direct | processes | modules | risk | +|---|---|---|---|---|---|---| +| `lbug-config.ts` | File | 54 | 12 | 0 | 0 | MEDIUM | +| `openLbugConnection` | Function | 16 | 9 | 3 | 2 | HIGH | +| `local-backend.ts` | File | 12 | 10 | 0 | 0 | MEDIUM | +| `refreshRepos` | Method | 50 | 5 | 4 | 7 | CRITICAL | + +- Clusters/processes resources `[graph]`: Local/Cli/Group sit in the impact path; process traces are function-stepped, not File-stepped. +- Related tests `[verified]`: `test/unit/impact-pagination.test.ts` (CRITICAL from `direct=400`); `test/integration/impact-zero-caller-risk.test.ts` (`withTestLbugDB` seed — pattern to extend); `test/unit/eval-formatters.test.ts` (`formatImpactResult`); group `mergeRisk` tests. + +## 5. Statement-Level PDG Findings + +PDG unavailable (`pdg_query` on `_runImpactBFS`: “no PDG layer”). Recommend `node .gitnexus/run.cjs analyze --index-only --pdg` before any future statement-slice work. Control flow of the scorer is a straight if/else after enrichment; no hidden guards. `skipEnrichment` is the only branch that structurally zeros process/module counts besides File ids. + +## 6. Proposed Changes + +### 6.1 Extract `scoreImpactRisk` — `gitnexus-shared/src/impact-risk.ts` (new) + +- **Responsibility:** Pure function: `{ direction, directCount, processCount, moduleCount, impactedCount, unusedAxes }` → `{ risk, riskSharedAxes, riskScale }`. +- **Behaviour:** Existing UNKNOWN/CRITICAL/HIGH/MEDIUM/LOW thresholds unchanged when `unusedAxes` is empty. `riskSharedAxes` always scores as if `processCount=0` and `moduleCount=0` (UNKNOWN rule still applies). `riskScale.comparableAcrossKinds` is false iff `unusedAxes` is non-empty. `riskScale.unusedAxes` lists `{ axis, reason }`. +- **Constraints:** Zero deps. Export from `gitnexus-shared/src/index.ts`. Do not put MCP types here. +- **File detection:** caller passes unused axes; helper does not parse UIDs. + +### 6.2 Wire MCP — `_runImpactBFS` in `local-backend.ts` + +- After computing `processCount`/`moduleCount`, set `unusedAxes`: + - target `id` starts with `File:` **or** `symType === 'File'` → processes + modules, reason `file-nodes-have-no-process-or-community-membership`; + - `skipEnrichment` → same axes, reason `enrichment-skipped` (ambiguous probes). +- Replace inline ladder with `scoreImpactRisk`. +- Spread `riskScale` and `riskSharedAxes` on the result next to `risk`. Do **not** set `riskNote` for File. +- Ambiguous candidate summaries: forward the new fields (probes already skip enrichment). +- `target.type` for File: if still `""`, prefer `'File'` when `id` starts with `File:` (display-only; helps CLI). + +### 6.3 Web duplicate — `gitnexus-web/src/core/llm/tools.ts` + +- Import `scoreImpactRisk` from `gitnexus-shared`. Print `RISK:` from `risk`; if `!comparableAcrossKinds`, one extra line: not comparable to Function risk; shared-axes label is `riskSharedAxes`. + +### 6.4 Agent/MCP contract copy + +- `gitnexus/src/mcp/tools.ts` impact description: document `riskScale` / `riskSharedAxes`; keep `riskNote` UNKNOWN-only; say File `risk` is not comparable to symbol `risk`. +- `gitnexus/src/cli/ai-context.ts`: HIGH/CRITICAL warning still applies; add: do not rank a File `MEDIUM` below a contained Function `HIGH` without `riskSharedAxes`. +- `formatImpactResult`: on resolved callgraph results with `risk`, print `Risk: {risk}` and, when incomparable, `Shared-axes risk: {riskSharedAxes} (File/process axes unused)`. + +### 6.5 Explicitly not changing + +- DEFINES-bridge, community/process indexers, `mergeRisk` formula, PDG `UNKNOWN`, `detectChanges` `risk_level`, Function thresholds. + +## 7. Implementation Sequence + +1. Add `gitnexus-shared` helper + unit table (issue-shaped inputs + UNKNOWN + skipEnrichment). Shared package tests if present; otherwise `gitnexus/test/unit/impact-risk.test.ts` importing the helper. +2. Switch `_runImpactBFS` + candidate probe payload. Tree still coherent: old `risk` values identical for Function fixtures. +3. Integration seed in `impact-zero-caller-risk.test.ts` **or** new `impact-file-risk-scale.test.ts`: File with ≥5 File IMPORTS (MEDIUM on direct) vs Function with 3 process-member callers (HIGH); assert File `riskScale.comparableAcrossKinds === false`, Function true, File `riskSharedAxes === risk`, Function `riskSharedAxes` is LOW/MEDIUM while `risk` is HIGH. +4. CLI formatter + `eval-formatters.test.ts`. +5. `tools.ts` + `ai-context.ts` wording. +6. Web import + a unit assertion on the printed RISK block if a test already covers that tool. +7. `npx tsc --noEmit` in `gitnexus/` and `gitnexus-web/`; `cd gitnexus && npm run test:unit -- test/unit/impact-risk.test.ts test/unit/eval-formatters.test.ts`; integration file from step 3. + +## 8. Test Strategy + +| File | Scenarios | +|---|---| +| `gitnexus/test/unit/impact-risk.test.ts` (new) | Issue table: File(25,13,0,0)→MEDIUM; Function(15,2,4,2)→HIGH; shared-axes File MEDIUM vs Function LOW; empty upstream UNKNOWN; downstream empty LOW; skipEnrichment unused axes; CRITICAL via direct≥30 still works with unused process axes | +| `gitnexus/test/integration/impact-file-risk-scale.test.ts` (new) | `withTestLbugDB` seed: `File:src/crypto.ts` ← 13 File IMPORTS, no File STEP_IN_PROCESS; `getEncryptionKey` with 2 CALLS from functions that have STEP_IN_PROCESS to 4 distinct Process nodes — reproduce inversion; assert new fields | +| `gitnexus/test/integration/impact-zero-caller-risk.test.ts` | Unchanged UNKNOWN/`riskNote`; candidates may grow `riskScale` — assert still present only when UNKNOWN for `riskNote` | +| `gitnexus/test/unit/impact-pagination.test.ts` | Hub CRITICAL unchanged | +| `gitnexus/test/unit/eval-formatters.test.ts` | Resolved result prints Risk + shared-axes line for File-shaped `riskScale` | +| Web | Only if an existing Graph RAG impact test snapshots `RISK:` | + +Commands (exist in `gitnexus/package.json`): `npm run test:unit`, `npm test` (full vitest), `npx tsc --noEmit`. Web: `npm test`, `npx tsc -b --noEmit`. Integration needs `pretest:integration` / `npm run test:integration` (runs `scripts/build.js`). + +## 9. Risk and Impact Analysis + +Direct dependents of `_runImpactBFS` `[graph]`: `_impactImpl`, `impactByUid`. `_impactImpl` is the only d=1 of `impact` besides the method’s own class. Any JSON consumer of `impact` (MCP, CLI `output(result)`, group local leg) sees additive fields — compatible if they ignore unknowns. + +- **HIGH workflow:** Function HIGH/CRITICAL unchanged. File still cannot reach HIGH via processes; a File with `direct≥15` or `total≥100` still can. Agents that compare File MEDIUM vs Function HIGH must start using `riskSharedAxes` or `riskScale`. +- **Ambiguous `maxRisk`:** probes skip enrichment, so File vs Function candidates are already 2-axis there — inversion is weaker on that path. +- **Group `mergeRisk`:** still compares incomparable File local `risk` to crossing count. Do not retune this PR; if a group File target is common, follow-up. +- **Web:** browser bundle picks up `gitnexus-shared` export — confirm `gitnexus-shared` build/exports include the new file. +- **Performance:** none (pure arithmetic after existing enrichment). +- **Ladybug empty labels:** File detection must not rely on `symType` alone. + +## 10. Files Expected to Change + +| File | Symbols | Reason | +|---|---|---| +| `gitnexus-shared/src/impact-risk.ts` | `scoreImpactRisk` | New shared scorer | +| `gitnexus-shared/src/index.ts` | exports | Public helper | +| `gitnexus/src/mcp/local/local-backend.ts` | `_runImpactBFS`, ambiguous candidate map | Wire scorer + File unused axes | +| `gitnexus/src/mcp/tools.ts` | `impact` description | Contract | +| `gitnexus/src/cli/ai-context.ts` | generated Always Do | Agent warning | +| `gitnexus/src/cli/eval-server.ts` | `formatImpactResult` | Print scale | +| `gitnexus-web/src/core/llm/tools.ts` | web `impact` | Same formula | +| `gitnexus/test/unit/impact-risk.test.ts` | — | Table tests | +| `gitnexus/test/integration/impact-file-risk-scale.test.ts` | — | Seeded inversion | +| `gitnexus/test/unit/eval-formatters.test.ts` | `formatImpactResult` | Formatter | + +## 11. Reusable Implementation Context + +```yaml +implementation_context: + task_summary: "Fix #3075: File impact.risk is a 2-axis score silently labelled on a 4-axis scale. Extract scoreImpactRisk; mark File/skipEnrichment axes unused; add riskScale + riskSharedAxes; do not DEFINES-bridge or retune Function thresholds." + acceptance_criteria: + - "File vs Function comparison is either labelled incomparable (riskScale) or done via riskSharedAxes" + - "Function/Method risk for identical four-axis inputs unchanged" + - "riskNote still UNKNOWN-only" + - "Integration seed reproduces crypto.ts-style inversion and asserts the new fields" + primary_symbols: + - symbol: "_runImpactBFS" + file: "gitnexus/src/mcp/local/local-backend.ts" + lines: "6991-7888" + role: "BFS + enrichment + inline risk ladder (replace ladder only)" + - symbol: "scoreImpactRisk" + file: "gitnexus-shared/src/impact-risk.ts" + lines: "new" + role: "Pure scorer + shared-axes + riskScale" + - symbol: "formatImpactResult" + file: "gitnexus/src/cli/eval-server.ts" + lines: "305-641" + role: "Human/LLM text surface for impact JSON" + related_symbols: + - symbol: "_impactImpl" + relationship: "CALLS" + relevance: "Resolves target, PDG vs callgraph, ambiguous skipEnrichment probes" + - symbol: "impactByUid" + relationship: "CALLS" + relevance: "Group fan-out; keep skipPerSymbolEnrichment; still run aggregation" + - symbol: "mergeRisk" + relationship: "consumes risk string" + relevance: "Do not change this PR" + - symbol: "isCommunitySymbol" + relationship: "index gate" + relevance: "Why File modules_affected is always 0" + - symbol: "composeUnifiedPdgImpactResult" + relationship: "separate path" + relevance: "PDG risk stays UNKNOWN" + execution_path: + - "impact / callTool → _impactImpl (resolve symbol, File id prefix File:)" + - "_runImpactBFS: IMPORTS-heavy walk for File; CALLS walk for Function" + - "Enrich STEP_IN_PROCESS / MEMBER_OF on impacted ids (empty for File ids)" + - "scoreImpactRisk with unusedAxes for File or skipEnrichment" + - "JSON to MCP/CLI; formatImpactResult for eval text; web LLM tools parallel path" + pdg_constraints: + - description: "No PDG layer on the planning index; scorer is post-enrichment arithmetic" + affected_statements: [] + implementation_consequence: "Do not wait on PDG; do not change pdg impact risk" + architectural_patterns: + - pattern: "Additive optional JSON fields on impact (riskNote, epistemic, partial)" + example_location: "gitnexus/src/mcp/local/local-backend.ts _runImpactBFS base object ~7754" + usage_guidance: "Add riskScale/riskSharedAxes the same way; never overload riskNote" + - pattern: "withTestLbugDB CREATE seed for impact contract" + example_location: "gitnexus/test/integration/impact-zero-caller-risk.test.ts" + usage_guidance: "Seed File IMPORTS + Function CALLS + Process membership separately" + files_to_modify: + - file: "gitnexus-shared/src/impact-risk.ts" + symbols: ["scoreImpactRisk"] + intended_change: "new pure scorer" + - file: "gitnexus-shared/src/index.ts" + symbols: [] + intended_change: "re-export" + - file: "gitnexus/src/mcp/local/local-backend.ts" + symbols: ["_runImpactBFS"] + intended_change: "unusedAxes + helper; File type display" + - file: "gitnexus/src/mcp/tools.ts" + symbols: [] + intended_change: "document fields" + - file: "gitnexus/src/cli/ai-context.ts" + symbols: [] + intended_change: "agent comparability note" + - file: "gitnexus/src/cli/eval-server.ts" + symbols: ["formatImpactResult"] + intended_change: "print risk + shared-axes when incomparable" + - file: "gitnexus-web/src/core/llm/tools.ts" + symbols: [] + intended_change: "import helper; extra prose line" + tests: + - file: "gitnexus/test/unit/impact-risk.test.ts" + scenarios: + - "File(25,13,0,0)+unused process/module → risk MEDIUM, comparableAcrossKinds false, riskSharedAxes MEDIUM" + - "Function(15,2,4,2) → HIGH, riskSharedAxes LOW (direct 2, total 15)" + - "upstream impactedCount 0 → UNKNOWN both fields" + - "direct 400 → CRITICAL even with unused process axes" + - file: "gitnexus/test/integration/impact-file-risk-scale.test.ts" + scenarios: + - "Seed File crypto.ts with 13 File importers vs getEncryptionKey with process-rich callers → inversion on risk, File incomparable, Function comparable" + - file: "gitnexus/test/unit/eval-formatters.test.ts" + scenarios: + - "formatImpactResult includes Shared-axes risk when riskScale.comparableAcrossKinds is false" + verification_commands: + - "cd gitnexus && npx tsc --noEmit" + - "cd gitnexus && npm run test:unit -- test/unit/impact-risk.test.ts test/unit/eval-formatters.test.ts test/unit/impact-pagination.test.ts" + - "cd gitnexus && npm run test:integration -- test/integration/impact-file-risk-scale.test.ts test/integration/impact-zero-caller-risk.test.ts" + - "cd gitnexus-web && npx tsc -b --noEmit" + risks: + - "Consumers that only read risk still see the inversion unless they adopt riskScale/riskSharedAxes — that is the chosen (explicit-scale) fix" + - "File type often empty; must key unusedAxes off File: id prefix" + - "gitnexus-shared export must reach the web bundle" + assumptions: + - "WHAT: File nodes never gain STEP_IN_PROCESS/MEMBER_OF without an indexer change. HOW: keep isCommunitySymbol and process traces as-is; tests seed File with zero such edges" + - "WHAT: Additive JSON fields are backward compatible. HOW: existing tests that exact-match the full impact object may need to allow extra keys — grep expect(res).toEqual on impact results before landing" + - "WHAT: HEAD 6bff33d is the pin; scorer line numbers ~7720. HOW: re-read the ladder if that hunk moved" + open_questions: + - "Whether GroupImpactResult should copy riskScale from local File targets (deferred unless tests already snapshot the full group object)" + avoid: + - "Do not DEFINES-bridge File→symbol processes/modules" + - "Do not lower Function process/module HIGH/CRITICAL thresholds" + - "Do not reuse riskNote for File incomparability" + - "Do not change PDG impact risk or detectChanges risk_level" + - "Do not treat labels(n)[0] or empty target.type as proof the node is not a File" + - "Do not repeat full repository discovery" +``` + +## 12. Assumptions and Open Questions + +**Assumptions** + +- Indexer will not start attaching File→Process/Community in this change (`isCommunitySymbol` stays). `[verified]` source; `[assumed]` future indexers. +- Ignoring unknown JSON keys is safe for MCP clients; any `toEqual` goldens in-repo must be updated. `[assumed]` — grep during implement. +- Stale-index inversion (`lbug-config.ts` vs `openLbugConnection`) is illustrative; the integration seed is the regression lock. `[graph]` vs `[verified]` seed. + +**Open questions** + +- Group `mergeRisk` + File local risk: copy `riskScale` onto `GroupImpactResult`? Default **no** unless a test breaks. +- Class/Interface STEP_IN_PROCESS sparsity: out of scope (#3075 is File). +- Printing `risk` on CLI formatted output is new (JSON already has it). Keep the extra lines short. + +**Deferred** + +- Recalibrated File-only HIGH thresholds. +- Indexing File community membership. +- DEFINES-bridge after a threshold RFC. +- Related #2975 (docs vs scorer wording) except as touched by `tools.ts`. + +## 13. Definition of Done + +- [ ] `scoreImpactRisk` is the only callgraph ladder in MCP and web. +- [ ] File (and skipEnrichment) results include `riskScale.comparableAcrossKinds === false` and `riskSharedAxes`. +- [ ] Function four-axis HIGH/CRITICAL cases in unit tests still pass with the same labels. +- [ ] Integration seed proves wider File blast + lower `risk` than a contained Function, and `riskSharedAxes` orders them without pretending processes existed on the File. +- [ ] `riskNote` still absent unless `risk === 'UNKNOWN'`. +- [ ] `tools.ts` + `ai-context.ts` state that File `risk` is not comparable to symbol `risk`. +- [ ] `cd gitnexus && npx tsc --noEmit` and the named unit/integration commands pass; web typecheck passes. diff --git a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md index 4fb73f3e6..85d90c90d 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md index 4fb73f3e6..85d90c90d 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/gitnexus-shared/src/impact-risk.ts b/gitnexus-shared/src/impact-risk.ts index 6c18614f6..413d02f76 100644 --- a/gitnexus-shared/src/impact-risk.ts +++ b/gitnexus-shared/src/impact-risk.ts @@ -6,6 +6,7 @@ export type UnusedImpactRiskReason = | 'file-nodes-have-no-process-or-community-membership' | 'enrichment-skipped' | 'enrichment-budget-exhausted' + | 'enrichment-truncated' | 'enrichment-query-failed'; export interface UnusedImpactRiskAxis { @@ -50,7 +51,20 @@ function score( return 'LOW'; } -function countsWithUnusedAxesZeroed( +const UNMEASURED_REASONS: ReadonlySet = new Set([ + 'file-nodes-have-no-process-or-community-membership', + 'enrichment-skipped', + 'enrichment-budget-exhausted', +]); + +function unusedPair(reason: UnusedImpactRiskReason): UnusedImpactRiskAxis[] { + return [ + { axis: 'processes', reason }, + { axis: 'modules', reason }, + ]; +} + +function countsWithUnmeasuredAxesZeroed( input: ImpactRiskInput, ): Pick< ImpactRiskInput, @@ -59,6 +73,7 @@ function countsWithUnusedAxesZeroed( let processCount = input.processCount; let moduleCount = input.moduleCount; for (const unused of input.unusedAxes ?? []) { + if (!UNMEASURED_REASONS.has(unused.reason)) continue; if (unused.axis === 'processes') processCount = 0; if (unused.axis === 'modules') moduleCount = 0; } @@ -79,33 +94,23 @@ export function unusedAxesForImpactWalk(input: { processQueryFailed: boolean; moduleQueryFailed: boolean; /** When 0, a zero chunk budget is not an unused-axis event — there was nothing to enrich. */ - impactedCount?: number; + impactedCount: number; + /** True when process/module queries ran on a strict subset of impacted symbols. */ + enrichmentTruncated?: boolean; }): UnusedImpactRiskAxis[] { if (input.isFileTarget) { - return [ - { - axis: 'processes', - reason: 'file-nodes-have-no-process-or-community-membership', - }, - { - axis: 'modules', - reason: 'file-nodes-have-no-process-or-community-membership', - }, - ]; + return unusedPair('file-nodes-have-no-process-or-community-membership'); } if (input.skipEnrichment) { - return [ - { axis: 'processes', reason: 'enrichment-skipped' }, - { axis: 'modules', reason: 'enrichment-skipped' }, - ]; + return unusedPair('enrichment-skipped'); } - if (input.maxChunks === 0 && (input.impactedCount ?? 1) > 0) { - return [ - { axis: 'processes', reason: 'enrichment-budget-exhausted' }, - { axis: 'modules', reason: 'enrichment-budget-exhausted' }, - ]; + if (input.maxChunks === 0 && input.impactedCount > 0) { + return unusedPair('enrichment-budget-exhausted'); } const unused: UnusedImpactRiskAxis[] = []; + if (input.enrichmentTruncated) { + unused.push(...unusedPair('enrichment-truncated')); + } if (input.processQueryFailed) { unused.push({ axis: 'processes', reason: 'enrichment-query-failed' }); } @@ -115,11 +120,28 @@ export function unusedAxesForImpactWalk(input: { return unused; } +const INCOMPLETE_SAMPLE_REASONS: ReadonlySet = new Set([ + 'enrichment-query-failed', + 'enrichment-truncated', +]); + export function scoreImpactRisk(input: ImpactRiskInput): ImpactRiskResult { const unusedAxes = input.unusedAxes ?? []; + const observedRisk = score(countsWithUnmeasuredAxesZeroed(input)); + const incompleteSample = unusedAxes.some((unused) => + INCOMPLETE_SAMPLE_REASONS.has(unused.reason), + ); + // Failed queries and truncated samples make observed process/module counts + // lower bounds. Preserve any HIGH/CRITICAL warning already proved by those + // counts, but never emit a confident LOW/MEDIUM edit gate from an incomplete + // enrichment pass. + const risk = + incompleteSample && (observedRisk === 'LOW' || observedRisk === 'MEDIUM') + ? 'UNKNOWN' + : observedRisk; return { - risk: score(countsWithUnusedAxesZeroed(input)), + risk, riskSharedAxes: score({ ...input, processCount: 0, moduleCount: 0 }), riskScale: { comparableAcrossKinds: unusedAxes.length === 0, diff --git a/gitnexus-web/src/core/llm/tools.ts b/gitnexus-web/src/core/llm/tools.ts index a702e7db8..407018678 100644 --- a/gitnexus-web/src/core/llm/tools.ts +++ b/gitnexus-web/src/core/llm/tools.ts @@ -13,7 +13,7 @@ import { tool } from '@langchain/core/tools'; import { z } from 'zod'; -import { NODE_TABLES, REL_TYPES } from 'gitnexus-shared'; +import { NODE_TABLES, REL_TYPES, scoreImpactRisk, unusedAxesForImpactWalk } from 'gitnexus-shared'; import type { EnrichedSearchResult, GrepResult } from '../../services/backend-client'; /** @@ -1275,6 +1275,9 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, stepCount: number | null; }> = []; let affectedClusters: Array<{ label: string; hits: number; impact: string }> = []; + let processQueryFailed = false; + let clusterQueryFailed = false; + let clusterClassificationFailed = false; if (trimmedIds.length > 0) { const processQuery = ` @@ -1302,9 +1305,23 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, : ''; const [processRes, clusterRes, directClusterRes] = await Promise.all([ - executeQuery(processQuery), - executeQuery(clusterQuery), - directClusterQuery ? executeQuery(directClusterQuery) : Promise.resolve([]), + executeQuery(processQuery).catch((err) => { + processQueryFailed = true; + if (import.meta.env.DEV) console.warn('Impact process enrichment failed:', err); + return []; + }), + executeQuery(clusterQuery).catch((err) => { + clusterQueryFailed = true; + if (import.meta.env.DEV) console.warn('Impact cluster enrichment failed:', err); + return []; + }), + directClusterQuery + ? executeQuery(directClusterQuery).catch((err) => { + clusterClassificationFailed = true; + if (import.meta.env.DEV) console.warn('Impact cluster enrichment failed:', err); + return []; + }) + : Promise.resolve([]), ]); const directClusterSet = new Set(); @@ -1323,7 +1340,11 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, affectedClusters = clusterRes.map((row: any) => { const label = Array.isArray(row) ? row[0] : row.label; const hits = Array.isArray(row) ? row[1] : row.hits; - const impact = directClusterSet.has(label) ? 'direct' : 'indirect'; + const impact = clusterClassificationFailed + ? 'classification-unavailable' + : directClusterSet.has(label) + ? 'direct' + : 'indirect'; return { label, hits, impact }; }); } @@ -1331,19 +1352,25 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, const directCount = depth1.length; const processCount = affectedProcesses.length; const clusterCount = affectedClusters.length; - let risk = 'LOW'; - if (directCount >= 30 || processCount >= 5 || clusterCount >= 5 || totalAffected >= 200) { - risk = 'CRITICAL'; - } else if ( - directCount >= 15 || - processCount >= 3 || - clusterCount >= 3 || - totalAffected >= 100 - ) { - risk = 'HIGH'; - } else if (directCount >= 5 || totalAffected >= 30) { - risk = 'MEDIUM'; - } + const enrichmentCapped = allNodeIds.length > maxIdsForContext; + const unusedAxes = unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: false, + maxChunks: 10, + processQueryFailed, + moduleQueryFailed: clusterQueryFailed, + impactedCount: totalAffected, + enrichmentTruncated: enrichmentCapped, + }); + const scored = scoreImpactRisk({ + direction, + directCount, + processCount, + moduleCount: clusterCount, + impactedCount: totalAffected, + unusedAxes, + }); + const { risk, riskSharedAxes, riskScale } = scored; // ===== COMPACT TABULAR OUTPUT ===== const lines: string[] = [ @@ -1351,22 +1378,42 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, `Confidence: High ${confidenceBuckets.high} | Medium ${confidenceBuckets.medium} | Low ${confidenceBuckets.low}`, ``, `AFFECTED PROCESSES:`, - ...(affectedProcesses.length > 0 - ? affectedProcesses.map( - (p) => - `- ${p.label} - BROKEN at step ${p.minStep ?? '?'} (${p.hits} symbols, ${p.stepCount ?? '?'} steps)`, - ) - : ['- None found']), + ...(processQueryFailed + ? ['- Unavailable (enrichment query failed)'] + : affectedProcesses.length > 0 + ? affectedProcesses.map( + (p) => + `- ${p.label} - BROKEN at step ${p.minStep ?? '?'} (${p.hits} symbols, ${p.stepCount ?? '?'} steps)`, + ) + : ['- None found']), ``, `AFFECTED CLUSTERS:`, - ...(affectedClusters.length > 0 - ? affectedClusters.map((c) => `- ${c.label} (${c.impact}, ${c.hits} symbols)`) - : ['- None found']), + ...(clusterQueryFailed + ? ['- Unavailable (enrichment query failed)'] + : affectedClusters.length > 0 + ? affectedClusters.map((c) => `- ${c.label} (${c.impact}, ${c.hits} symbols)`) + : ['- None found']), ``, - `RISK: ${risk}`, + `RISK: ${risk} (edit gate — warn on HIGH/CRITICAL)`, + `Shared-axes: ${riskSharedAxes} (File vs symbol compare only; do not waive a HIGH risk warning)`, + `Note: this Graph-RAG surface expands File targets to in-file symbols before enrichment, so process/cluster axes are comparable here when enrichment succeeds. MCP File impact does not.`, + ...(riskScale.comparableAcrossKinds + ? [] + : [ + `Note: process/module axes were unused (${riskScale.unusedAxes.map((a) => a.reason).join(', ')}).`, + ]), + ...(risk === 'UNKNOWN' && (processQueryFailed || clusterQueryFailed) + ? ['Note: risk is unresolved because enrichment failed; retry before editing.'] + : []), + ...(enrichmentCapped + ? [`Note: process/cluster enrichment is partial (first ${maxIdsForContext} symbols).`] + : []), + ...(clusterClassificationFailed + ? ['Note: direct/indirect cluster classification is unavailable.'] + : []), `- Direct callers: ${directCount}`, - `- Processes affected: ${processCount}`, - `- Clusters affected: ${clusterCount}`, + `- Processes affected: ${processQueryFailed ? 'unavailable' : processCount}`, + `- Clusters affected: ${clusterQueryFailed ? 'unavailable' : clusterCount}`, ``, ]; @@ -1472,7 +1519,9 @@ relationTypes filter (optional): Additional output sections: - Affected processes (with step impact) - Affected clusters (direct/indirect) -- Risk summary (based on direct callers, processes, clusters)`, +- RISK is the edit gate: warn before edits on HIGH/CRITICAL; UNKNOWN requires retry or corroboration +- Shared-axes risk compares File and symbol targets using direct/total counts only; it never waives the RISK gate +- riskScale notes unavailable process/module axes. This Graph-RAG tool expands File targets to in-file symbols; MCP File impact does not`, schema: z.object({ target: z.string().describe('Name of the function, class, or file to analyze'), direction: z diff --git a/gitnexus-web/test/unit/impact-tool.test.ts b/gitnexus-web/test/unit/impact-tool.test.ts new file mode 100644 index 000000000..f04817ed1 --- /dev/null +++ b/gitnexus-web/test/unit/impact-tool.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createGraphRAGTools, type GraphRAGBackend } from '../../src/core/llm/tools'; + +const noOpBackend: GraphRAGBackend = { + executeQuery: async () => [], + search: async () => [], + grep: async () => [], + readFile: async () => '', +}; + +function impactTool(backend: GraphRAGBackend) { + return createGraphRAGTools(backend).find((candidate) => candidate.name === 'impact')!; +} + +describe('Graph-RAG impact risk contract', () => { + it('advertises the edit gate, shared axes, and MCP File difference', () => { + const description = impactTool(noOpBackend).description; + expect(description).toContain('RISK is the edit gate'); + expect(description).toContain('Shared-axes risk'); + expect(description).toContain('riskScale'); + expect(description).toContain('MCP File impact does not'); + }); + + it('renders failed enrichment as unavailable and fails the risk gate closed', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + if (query.includes('MATCH (affected)-[r:CodeRelation]->(target)')) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + startLine: 4, + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) throw new Error('process query failed'); + if (query.includes('MEMBER_OF')) return []; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('AFFECTED PROCESSES:\n- Unavailable (enrichment query failed)'); + expect(output).not.toContain('AFFECTED PROCESSES:\n- None found'); + expect(output).toContain('RISK: UNKNOWN'); + expect(output).toContain('risk is unresolved because enrichment failed'); + expect(output).toContain('- Processes affected: unavailable'); + }); + + it('preserves proved CRITICAL risk when the cluster query fails', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + if (query.includes('MATCH (affected)-[r:CodeRelation]->(target)')) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) { + return Array.from({ length: 5 }, (_, index) => ({ + label: `process-${index}`, + hits: 1, + minStep: index + 1, + stepCount: 5, + })); + } + if (query.includes('MEMBER_OF') && query.includes('COUNT(DISTINCT s.id)')) { + throw new Error('cluster query failed'); + } + if (query.includes('MEMBER_OF')) return []; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('RISK: CRITICAL'); + expect(output).toContain('AFFECTED CLUSTERS:\n- Unavailable (enrichment query failed)'); + expect(output).toContain('- Processes affected: 5'); + expect(output).toContain('- Clusters affected: unavailable'); + }); + + it('does not invent direct/indirect cluster classification after its query fails', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + if (query.includes('MATCH (affected)-[r:CodeRelation]->(target)')) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) return []; + if (query.includes('MEMBER_OF') && query.includes('RETURN DISTINCT')) { + throw new Error('classification query failed'); + } + if (query.includes('MEMBER_OF')) return [{ label: 'Core', hits: 1 }]; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('- Core (classification-unavailable, 1 symbols)'); + expect(output).toContain('direct/indirect cluster classification is unavailable'); + expect(output).not.toContain('process/module axes were unused'); + }); + + it('treats successful File expansion as comparable because enrichment runs on member symbols', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("n.filePath CONTAINS 'src/target.ts'")) { + return [{ id: 'file-id', nodeType: 'File', filePath: 'src/target.ts' }]; + } + if (query.includes("callee.filePath = 'src/target.ts'")) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) { + return [{ label: 'Build', hits: 1, minStep: 1, stepCount: 1 }]; + } + if (query.includes('MEMBER_OF') && query.includes('RETURN DISTINCT')) { + return [{ label: 'Core' }]; + } + if (query.includes('MEMBER_OF')) return [{ label: 'Core', hits: 1 }]; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'src/target.ts', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('process/cluster axes are comparable here when enrichment succeeds'); + expect(output).toContain('- Processes affected: 1'); + expect(output).toContain('- Clusters affected: 1'); + expect(output).not.toContain('process/module axes were unused'); + }); + + it('surfaces the 500-symbol enrichment cap as partial', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + const depth = query.includes('3 AS depth') ? 3 : query.includes('2 AS depth') ? 2 : 1; + if (query.includes('CodeRelation') && query.includes(` ${depth} AS depth`)) { + return Array.from({ length: 200 }, (_, index) => ({ + id: `d${depth}-${index}`, + name: `node-${depth}-${index}`, + nodeType: 'Function', + filePath: `src/d${depth}-${index}.ts`, + edgeType: 'CALLS', + confidence: 1, + })); + } + if (query.includes('STEP_IN_PROCESS') || query.includes('MEMBER_OF')) return []; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 3, + }); + + expect(output).toContain('process/cluster enrichment is partial (first 500 symbols)'); + expect(output).toContain('enrichment-truncated'); + expect(output).not.toContain('enrichment-budget-exhausted'); + }); +}); diff --git a/gitnexus/skills/gitnexus-impact-analysis.md b/gitnexus/skills/gitnexus-impact-analysis.md index 4fb73f3e6..85d90c90d 100644 --- a/gitnexus/skills/gitnexus-impact-analysis.md +++ b/gitnexus/skills/gitnexus-impact-analysis.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/gitnexus/src/cli/ai-context.ts b/gitnexus/src/cli/ai-context.ts index 258aeb46c..cb7b42c60 100644 --- a/gitnexus/src/cli/ai-context.ts +++ b/gitnexus/src/cli/ai-context.ts @@ -218,16 +218,16 @@ This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${s ## Always Do -- **MUST run impact analysis before editing.** Use \`impact({target: "symbolName", direction: "upstream"})\` (MCP) or \`${runner} impact "symbolName" --direction upstream --repo .\` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis.${ +- **MUST run impact before editing.** Use \`impact({target: "symbolName", direction: "upstream"})\` or \`${runner} impact "symbolName" --direction upstream --repo .\`; report callers, processes, and risk. Never substitute grep for graph analysis.${ hasPdg ? ` For unified PDG impact, add \`mode: "pdg"\` with optional \`line: \` — it returns statement-level \`affectedStatements\` over CDG + REACHING_DEF and inter-procedural symbols in \`interproceduralByDepth\`/\`byDepth\`; no-layer/degraded PDG results are UNKNOWN-risk notes (\`--pdg\` layer). CLI equivalent: \`${runner} impact "symbolName" --direction upstream --mode pdg --line --repo .\`.` : '' } - **MUST analyze graph changes before committing.** Use \`detect_changes({scope: "all"})\` (MCP) or \`${runner} detect-changes --scope all --repo .\` (CLI fallback). \`partial: true\` or \`truncated: true\` is not a clean check — a zero means unseen, not unaffected; re-run it. For regression review: \`detect_changes({scope: "compare", base_ref: ${JSON.stringify(markdownSafeBranch(defaultBranch))}})\` or \`${runner} detect-changes --scope compare --base-ref ${JSON.stringify(markdownSafeBranch(defaultBranch))} --repo .\`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- MUST warn on HIGH/CRITICAL \`risk\` pre-edit; never use \`riskSharedAxes\` to waive a HIGH/CRITICAL \`risk\` warning. Compare File/symbol: MCP File omits axes; Graph-RAG expands File. - **MUST treat \`risk: UNKNOWN\` as unresolved, not as low.** An empty caller set is not evidence the symbol is unused — it can also mean the callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls). \`impact\` pairs \`UNKNOWN\` with a \`riskNote\` saying so. Confirm with a text search before treating the symbol as safe to change or delete; do not proceed on the strength of a zero. -- When exploring unfamiliar code, use \`query({search_query: "concept"})\` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use \`context({name: "symbolName"})\`. +- Explore with \`query({search_query: "concept"})\` for process-grouped flows. +- Use \`context({name: "symbolName"})\` for callers, callees, and flows. - For security review, \`explain({target: "fileOrSymbol"})\` lists taint findings (source→sink flows; needs \`analyze --pdg\`).${ hasPdg ? `\n- For control/data dependence, \`pdg_query({mode: "controls", target: "fileOrSymbol"})\` answers "under what condition does X run?" (CDG, incl. guard clauses) and \`pdg_query({mode: "flows", target, variable})\` traces "where does variable Y flow?" (REACHING_DEF). \`--pdg\` layer.` diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index caf19bc03..b2f656c39 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -302,6 +302,20 @@ function formatTruncationSuffix(result: { return label ? ` (by ${label})` : ''; } +function pushCallgraphRiskLines(lines: string[], result: any): void { + if (result.risk) { + lines.push(`Risk: ${result.risk}`); + } + if (result.riskNote) { + lines.push(String(result.riskNote)); + } + if (result.riskScale?.comparableAcrossKinds === false && result.riskSharedAxes) { + lines.push( + `Shared-axes risk: ${result.riskSharedAxes} (process/module axes are unavailable — compare File vs symbol only; do not use this to waive a HIGH/CRITICAL risk warning)`, + ); + } +} + export function formatImpactResult(result: any): string { if (result.error) { const suggestion = result.suggestion ? `\nSuggestion: ${result.suggestion}` : ''; @@ -567,14 +581,21 @@ export function formatImpactResult(result: any): string { // #1858 — "isolated" is a confident claim. If an interface / indirection // boundary is on the path, the true count is a lower bound, not zero; // callers binding via DI / dynamic dispatch were not traced. Say so instead. + const lines: string[] = []; if (result.epistemic === 'lower-bound') { - const lines = [ + lines.push( `${target?.name || '?'}: no direct ${direction} dependencies traced, but this is a LOWER BOUND — unresolved indirection on the path (actual impact may be higher):`, - ]; + ); for (const b of result.boundaries || []) lines.push(` • ${b}`); - return lines.join('\n'); + } else if (direction === 'upstream') { + lines.push( + `${target?.name || '?'}: No ${direction} callers resolved. This is not evidence the symbol is unused or isolated.`, + ); + } else { + lines.push(`${target?.name || '?'}: No ${direction} dependencies found.`); } - return `${target?.name || '?'}: No ${direction} dependencies found. This symbol appears isolated.`; + pushCallgraphRiskLines(lines, result); + return lines.join('\n'); } const lines: string[] = []; @@ -594,6 +615,7 @@ export function formatImpactResult(result: any): string { ); for (const b of result.boundaries || []) lines.push(` • ${b}`); } + pushCallgraphRiskLines(lines, result); lines.push(''); const depthLabels: Record = { diff --git a/gitnexus/src/core/group/cross-impact.ts b/gitnexus/src/core/group/cross-impact.ts index 19ddd6fee..485b1ee8c 100644 --- a/gitnexus/src/core/group/cross-impact.ts +++ b/gitnexus/src/core/group/cross-impact.ts @@ -5,6 +5,7 @@ import fsp from 'node:fs/promises'; import path from 'node:path'; +import type { ImpactRisk } from 'gitnexus-shared'; import type { BridgeHandle, BridgeMeta, @@ -381,7 +382,17 @@ function extractProcessNames(impact: unknown): string[] { // permanently that a PDG `risk:'UNKNOWN'` never coalesces to a confident `LOW`. // No behavior change — `'UNKNOWN'` was already handled correctly at the // `(localRisk === 'LOW' || localRisk === 'UNKNOWN')` branch below. -export function mergeRisk(localRisk: string, cross: CrossRepoImpact[]): string { +function asImpactRisk(value: unknown, fallback: ImpactRisk = 'LOW'): ImpactRisk { + return value === 'LOW' || + value === 'MEDIUM' || + value === 'HIGH' || + value === 'CRITICAL' || + value === 'UNKNOWN' + ? value + : fallback; +} + +export function mergeRisk(localRisk: ImpactRisk, cross: CrossRepoImpact[]): ImpactRisk { const traversed = cross.filter((c) => c.fanout_status !== 'not_attempted'); const highConf = traversed.some((c) => c.contract.confidence >= 0.85); if (localRisk === 'CRITICAL') return 'CRITICAL'; @@ -391,6 +402,22 @@ export function mergeRisk(localRisk: string, cross: CrossRepoImpact[]): string { return localRisk; } +function liftLocalRiskMeta( + local: unknown, + cross: CrossRepoImpact[], +): Pick { + const { riskSharedAxes, riskScale } = local as { + riskSharedAxes?: unknown; + riskScale?: GroupImpactResult['riskScale']; + }; + return { + ...(riskSharedAxes !== undefined + ? { riskSharedAxes: mergeRisk(asImpactRisk(riskSharedAxes), cross) } + : {}), + ...(riskScale !== undefined ? { riskScale } : {}), + }; +} + /** * Is this bridge's metadata unable to say where its contents came from? * @@ -601,6 +628,7 @@ export async function runGroupImpact( cross_repo_hits: 0, }, risk: 'UNKNOWN', + ...liftLocalRiskMeta(local, []), timeoutMs, crossDepthWarning, }; @@ -656,7 +684,8 @@ export async function runGroupImpact( modules_affected: s.modules_affected ?? 0, cross_repo_hits: 0, }, - risk: String((local as { risk?: string }).risk ?? 'LOW'), + risk: asImpactRisk((local as { risk?: unknown }).risk), + ...liftLocalRiskMeta(local, []), timeoutMs, crossDepthWarning, }; @@ -826,7 +855,7 @@ export async function runGroupImpact( } const localSum = (local as { summary?: Record })?.summary || {}; - const localRisk = String((local as { risk?: string }).risk ?? 'LOW'); + const localRisk = asImpactRisk((local as { risk?: unknown }).risk); const localPartial = Boolean((local as { partial?: boolean }).partial); // The bridge's own incompleteness, in the shared vocabulary, read through // what this query DECLARED. The fan-out above already drops every neighbour @@ -905,6 +934,7 @@ export async function runGroupImpact( cross_repo_hits: cross.length, }, risk: mergeRisk(localRisk, cross), + ...liftLocalRiskMeta(local, cross), timeoutMs, crossDepthWarning, }; diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index beeb6f053..023629506 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,3 +1,5 @@ +import type { ImpactRisk, ImpactRiskResult } from 'gitnexus-shared'; + export type ContractType = | 'http' | 'graphql' @@ -194,7 +196,17 @@ export interface GroupImpactResult { modules_affected: number; cross_repo_hits: number; }; - risk: string; + risk: ImpactRisk; + /** + * Two-axis (direct + total) risk from the local leg, then `mergeRisk` with + * crossings — compare File vs symbol here, not via top-level `risk`. + */ + riskSharedAxes?: ImpactRisk; + /** + * Local-leg scale metadata (File / skipped enrichment). Crossings do not + * invent process/module membership for File nodes. + */ + riskScale?: ImpactRiskResult['riskScale']; /** * `'lower-bound'` when the fan-out was cut short, so `risk` is a FLOOR, not a * verdict. Same vocabulary as single-repo `impact`'s `epistemic` field. diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 6451b0d9b..544f64c12 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -9,6 +9,7 @@ import fs from 'fs/promises'; import path from 'path'; import { createHash } from 'crypto'; +import { scoreImpactRisk, unusedAxesForImpactWalk, type ImpactRiskResult } from 'gitnexus-shared'; import { initLbug, executeQuery, @@ -6412,6 +6413,8 @@ export class LocalBackend { let summary: { impactedCount: number; risk: string; + riskSharedAxes?: string; + riskScale?: ImpactRiskResult['riskScale']; riskNote?: string; summary?: { direct: number }; } | null = null; @@ -6451,6 +6454,10 @@ export class LocalBackend { score: Number(c.score.toFixed(2)), impactedCount: summary?.impactedCount ?? 0, risk: summary?.risk ?? 'UNKNOWN', + ...(summary?.riskSharedAxes !== undefined + ? { riskSharedAxes: summary.riskSharedAxes } + : {}), + ...(summary?.riskScale !== undefined ? { riskScale: summary.riskScale } : {}), direct: summary?.summary?.direct ?? 0, ...(summary?.riskNote !== undefined ? { riskNote: summary.riskNote } : {}), // Carry the explanation with the verdict. The single-symbol path @@ -7504,6 +7511,10 @@ export class LocalBackend { const parsedMaxChunks = rawMaxChunks ? Number(rawMaxChunks) : Number.NaN; const MAX_CHUNKS = Number.isInteger(parsedMaxChunks) && parsedMaxChunks >= 0 ? parsedMaxChunks : 10; + let processQueryFailed = false; + let moduleQueryFailed = false; + let enrichmentDegraded = false; + let moduleClassificationFailed = false; // `skipEnrichment` (ambiguous #2129 per-candidate probes) bypasses the // process/module aggregation passes entirely — those probes need only the @@ -7554,7 +7565,12 @@ export class LocalBackend { ORDER BY pId `, { ids }, - ).catch(() => []); + ).catch((err) => { + processQueryFailed = true; + enrichmentDegraded = true; + logQueryError('impact:process-chunk', err); + return []; + }); for (const row of rows) { const pId = row.pId ?? row[0]; @@ -7605,6 +7621,8 @@ export class LocalBackend { ep.earliest_broken_step = Math.min(ep.earliest_broken_step, minStep ?? Infinity); } } catch (e) { + processQueryFailed = true; + enrichmentDegraded = true; logQueryError('impact:process-chunk', e); } } @@ -7624,7 +7642,11 @@ export class LocalBackend { RETURN p.id AS pid, MIN(r.step) AS minStep `, { pIds, ids: allImpactedIds }, - ).catch(() => []); + ).catch((err) => { + enrichmentDegraded = true; + logQueryError('impact:process-chunk-backfill', err); + return []; + }); for (const mr of missingRows) { const pid = mr.pid ?? mr[0]; @@ -7638,6 +7660,7 @@ export class LocalBackend { } } } catch (e) { + enrichmentDegraded = true; logQueryError('impact:process-chunk-backfill', e); } } @@ -7698,7 +7721,12 @@ export class LocalBackend { LIMIT 20 `, { ids: idsChunk }, - ).catch(() => []); + ).catch((err) => { + moduleQueryFailed = true; + enrichmentDegraded = true; + logQueryError('impact:module-chunk', err); + return []; + }); for (const r of rows) { const name = r.name ?? r[0] ?? null; @@ -7707,6 +7735,8 @@ export class LocalBackend { moduleHitsMap.set(name, (moduleHitsMap.get(name) || 0) + hits); } } catch (e) { + moduleQueryFailed = true; + enrichmentDegraded = true; logQueryError('impact:module-chunk', e); } }; @@ -7732,12 +7762,19 @@ export class LocalBackend { RETURN DISTINCT c.heuristicLabel AS name `, { ids: idsChunk }, - ).catch(() => []); + ).catch((err) => { + enrichmentDegraded = true; + moduleClassificationFailed = true; + logQueryError('impact:direct-module-chunk', err); + return []; + }); for (const r of rows) { const name = r.name ?? r[0] ?? null; if (name) directModuleSet.add(name); } } catch (e) { + enrichmentDegraded = true; + moduleClassificationFailed = true; logQueryError('impact:direct-module-chunk', e); } }; @@ -7762,7 +7799,11 @@ export class LocalBackend { return { name, hits, - impact: directModuleNameSet.has(name) ? 'direct' : 'indirect', + impact: moduleClassificationFailed + ? 'classification-unavailable' + : directModuleNameSet.has(name) + ? 'direct' + : 'indirect', }; }); } @@ -7770,40 +7811,25 @@ export class LocalBackend { // Risk scoring const processCount = affectedProcesses.length; const moduleCount = affectedModules.length; - let risk: string; - if (direction === 'upstream' && impacted.length === 0) { - // An upstream walk that resolved NO callers cannot support `LOW`. "Safe - // to change" is a claim ABOUT callers, and this walk found none to reason - // about: the symbol may be genuinely unused, or reached only through a - // reference class this index does not record — a property access on a - // plain object, or a bare-identifier read of a module-scope `Const`, - // neither of which mints a reference site today. Seeding `LOW` from an - // empty result is the same false-safe signal `anyKnownRisk` refuses to - // emit on the ambiguous-candidate path, and that #2687 removed by making - // an undetermined `impactedCount` `null` instead of `0`. - // - // Downstream is deliberately untouched: an empty downstream walk reports - // that this symbol resolved no callees, which is not a safety verdict. - risk = 'UNKNOWN'; - } else if ( - directCount >= 30 || - processCount >= 5 || - moduleCount >= 5 || - impacted.length >= 200 - ) { - risk = 'CRITICAL'; - } else if ( - directCount >= 15 || - processCount >= 3 || - moduleCount >= 3 || - impacted.length >= 100 - ) { - risk = 'HIGH'; - } else if (directCount >= 5 || impacted.length >= 30) { - risk = 'MEDIUM'; - } else { - risk = 'LOW'; - } + const isFileTarget = symType === 'File' || String(symId).startsWith('File:'); + const unusedAxes = unusedAxesForImpactWalk({ + isFileTarget, + skipEnrichment, + maxChunks: MAX_CHUNKS, + processQueryFailed, + moduleQueryFailed, + impactedCount: impacted.length, + enrichmentTruncated: + !skipEnrichment && MAX_CHUNKS > 0 && impacted.length > MAX_CHUNKS * CHUNK_SIZE, + }); + const { risk, riskSharedAxes, riskScale } = scoreImpactRisk({ + direction, + directCount, + processCount, + moduleCount, + impactedCount: impacted.length, + unusedAxes, + }); // Build per-depth counts (always included, even in summaryOnly mode) const byDepthCounts: Record = {}; @@ -7823,7 +7849,7 @@ export class LocalBackend { target: { id: symId, name: sym.name || sym[1], - type: symType, + type: isFileTarget ? symType || 'File' : symType, filePath: sym.filePath || sym[2], ...(beanMetadata ? { bean: beanMetadata } : {}), ...(aopMetadata ? { aop: aopMetadata } : {}), @@ -7831,18 +7857,27 @@ export class LocalBackend { direction, impactedCount: impacted.length, risk, + riskSharedAxes, + riskScale, ...(risk === 'UNKNOWN' ? { riskNote: - 'No callers resolved. Absence of edges is not evidence the symbol is unused: ' + - 'a caller reaching it through a reference class this index does not record — ' + - 'plain-object property access, a bare-identifier read of a module-scope const — ' + - 'produces no edge to find. Confirm with a text search before treating the ' + - 'change as safe.', + processQueryFailed || moduleQueryFailed + ? 'Risk is unresolved because process/module enrichment failed. Observed counts ' + + 'are lower bounds; retry impact before treating the change as safe.' + : unusedAxes.some((axis) => axis.reason === 'enrichment-truncated') + ? 'Risk is unresolved because process/module enrichment was truncated. Observed ' + + 'counts are lower bounds; retry with a higher IMPACT_MAX_CHUNKS before ' + + 'treating the change as safe.' + : 'No callers resolved. Absence of edges is not evidence the symbol is unused: ' + + 'a caller reaching it through a reference class this index does not record — ' + + 'plain-object property access, a bare-identifier read of a module-scope const — ' + + 'produces no edge to find. Confirm with a text search before treating the ' + + 'change as safe.', } : {}), ...epistemic, - ...(!traversalComplete && { partial: true }), + ...((!traversalComplete || enrichmentDegraded) && { partial: true }), summary: { direct: directCount, processes_affected: processCount, diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 53700a7a2..4244a95bf 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -475,11 +475,13 @@ WHEN TO USE: Before making code changes — especially refactoring, renaming, or AFTER THIS: Review d=1 items (WILL BREAK). Use context() on high-risk symbols. Output includes: -- risk: LOW / MEDIUM / HIGH / CRITICAL / UNKNOWN. An upstream walk that resolved ZERO callers reports UNKNOWN, never LOW, and carries riskNote: "safe to change" is a claim about callers and there were none to reason about, so the symbol is either genuinely unused OR reached only through a reference class the index does not record (plain-object property access, a bare-identifier read of a module-scope const). Confirm with a text search before acting on it. Downstream walks are unaffected — an empty downstream result reports resolved callees, not safety. +- risk: LOW / MEDIUM / HIGH / CRITICAL / UNKNOWN. This is the HIGH/CRITICAL edit-gate field. File targets lack process/community membership, so their risk is not directly comparable with symbol risk; use riskSharedAxes to compare the direct/total axes common to both. Group-mode (\`repo: "@…"\`) results lift the same fields to the top-level envelope. The web Graph-RAG impact tool expands File targets to in-file symbols before enrichment, so process/cluster axes remain comparable there. An upstream walk that resolved ZERO callers reports UNKNOWN, never LOW, and carries riskNote: "safe to change" is a claim about callers and there were none to reason about, so the symbol is either genuinely unused OR reached only through a reference class the index does not record (plain-object property access, a bare-identifier read of a module-scope const). Confirm with a text search before acting on it. Downstream walks are unaffected — an empty downstream result reports resolved callees, not safety. +- riskSharedAxes: single-repo risk computed only from direct and total impact. Group mode then applies the cross-repo crossing overlay to that local value. Suitable for comparing File and symbol targets within the same mode. Never substitute it for \`risk\` when deciding whether to warn before edits. +- riskScale: { comparableAcrossKinds, unusedAxes } — names process/module axes that were structurally unavailable, skipped, budget-exhausted (\`IMPACT_MAX_CHUNKS=0\`), truncated (sampled a subset of impacted symbols), or failed at query time. Failed-query and truncated-sample counts are lower bounds: known HIGH/CRITICAL warnings survive, otherwise risk is UNKNOWN. Group impact copies this metadata from the local leg. - riskNote: string — present only when risk is UNKNOWN; states why the verdict is withheld. - summary: direct callers, processes affected, modules affected - affected_processes: which execution flows break and at which step -- affected_modules: which functional areas are hit (direct vs indirect) +- affected_modules: which functional areas are hit (direct vs indirect; classification-unavailable when that secondary query fails) - byDepth: affected symbols grouped by traversal depth (paginated by limit/offset; omitted when summaryOnly:true — use byDepthCounts for totals per depth, pagination object when truncated). Each item includes a processes:[{id,label,processType,step}] field listing the execution flows that symbol participates in. Empty when the symbol has no process membership. Can ALSO be empty when partial:true is set — either the process-aggregation pass hit its cap before detecting affected processes, or per-symbol enrichment was capped on a very large page. When partial:true, do NOT treat processes:[] as proof of no participation; cross-check the top-level affected_processes list. - epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the walk provably missed callers, so the count is a floor. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). - boundaries: string[] — one plain-language sentence per reason the count is short. Prose for humans; branch on causes instead. diff --git a/gitnexus/test/integration/impact-file-risk-scale.test.ts b/gitnexus/test/integration/impact-file-risk-scale.test.ts new file mode 100644 index 000000000..16b2e6a7f --- /dev/null +++ b/gitnexus/test/integration/impact-file-risk-scale.test.ts @@ -0,0 +1,143 @@ +import { beforeAll, expect, it, vi } from 'vitest'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { withTestLbugDB, type IndexedDBHandle } from '../helpers/test-indexed-db.js'; + +vi.mock('../../src/storage/repo-manager.js', () => ({ + listRegisteredRepos: vi.fn().mockResolvedValue([]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), +})); + +const fileImporters = Array.from( + { length: 13 }, + (_, index) => + `CREATE (f:File {id: 'File:src/importer-${index}.ts', name: 'importer-${index}.ts', filePath: 'src/importer-${index}.ts', content: ''})`, +); +const fileImportEdges = Array.from( + { length: 13 }, + (_, index) => + `MATCH (a:File {id:'File:src/importer-${index}.ts'}), (b:File {id:'File:src/crypto.ts'}) CREATE (a)-[:CodeRelation {type:'IMPORTS', confidence:1.0, reason:'import', step:0}]->(b)`, +); +const processNodes = Array.from( + { length: 4 }, + (_, index) => + `CREATE (p:Process {id: 'proc-${index}', label: 'Flow ${index}', heuristicLabel: 'Flow ${index}', processType: 'cross_community', stepCount: 3, communities: [], entryPointId: 'Function:src/entry-${index}.ts:entry${index}', terminalId: 'Function:src/crypto.ts:getEncryptionKey'})`, +); +const processEntryPoints = Array.from( + { length: 4 }, + (_, index) => + `CREATE (ep:Function {id: 'Function:src/entry-${index}.ts:entry${index}', name: 'entry${index}', filePath: 'src/entry-${index}.ts', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, +); +const processEdges = Array.from( + { length: 4 }, + (_, index) => + `MATCH (a:Function {id:'Function:src/caller-${index % 2}.ts:caller${index % 2}'}), (p:Process {id:'proc-${index}'}) CREATE (a)-[:CodeRelation {type:'STEP_IN_PROCESS', confidence:1.0, reason:'trace-detection', step:1}]->(p)`, +); + +const SEED = [ + `CREATE (f:File {id: 'File:src/crypto.ts', name: 'crypto.ts', filePath: 'src/crypto.ts', content: ''})`, + ...fileImporters, + ...fileImportEdges, + `CREATE (fn:Function {id: 'Function:src/crypto.ts:getEncryptionKey', name: 'getEncryptionKey', filePath: 'src/crypto.ts', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, + `CREATE (c0:Function {id: 'Function:src/caller-0.ts:caller0', name: 'caller0', filePath: 'src/caller-0.ts', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, + `CREATE (c1:Function {id: 'Function:src/caller-1.ts:caller1', name: 'caller1', filePath: 'src/caller-1.ts', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, + `MATCH (a:Function {id:'Function:src/caller-0.ts:caller0'}), (b:Function {id:'Function:src/crypto.ts:getEncryptionKey'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:1.0, reason:'direct', step:0}]->(b)`, + `MATCH (a:Function {id:'Function:src/caller-1.ts:caller1'}), (b:Function {id:'Function:src/crypto.ts:getEncryptionKey'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:1.0, reason:'direct', step:0}]->(b)`, + ...processEntryPoints, + ...processNodes, + ...processEdges, +]; + +type BackendHandle = IndexedDBHandle & { _backend?: LocalBackend }; + +withTestLbugDB( + 'impact-file-risk-scale', + (handle) => { + let backend: LocalBackend; + beforeAll(() => { + const ext = handle as BackendHandle; + if (!ext._backend) throw new Error('LocalBackend not initialized'); + backend = ext._backend; + }); + + it('marks the wider File score incomparable with the process-rich Function score', async () => { + const file = await backend.callTool('impact', { + target: 'crypto.ts', + kind: 'File', + direction: 'upstream', + }); + const fn = await backend.callTool('impact', { + target: 'getEncryptionKey', + kind: 'Function', + direction: 'upstream', + }); + + expect(file.impactedCount).toBe(13); + expect(file.risk).toBe('MEDIUM'); + expect(file.riskSharedAxes).toBe('MEDIUM'); + expect(file.target.type).toBe('File'); + expect(file.riskScale).toEqual({ + comparableAcrossKinds: false, + unusedAxes: [ + { + axis: 'processes', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + { + axis: 'modules', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + ], + }); + expect(file.riskNote).toBeUndefined(); + + expect(fn.impactedCount).toBe(2); + expect(fn.risk).toBe('HIGH'); + expect(fn.riskSharedAxes).toBe('LOW'); + expect(fn.summary.processes_affected).toBe(4); + expect(fn.riskScale).toEqual({ + comparableAcrossKinds: true, + unusedAxes: [], + }); + expect(fn.riskNote).toBeUndefined(); + }); + + it('marks downstream File risk incomparable on the same seed', async () => { + const file = await backend.callTool('impact', { + target: 'crypto.ts', + kind: 'File', + direction: 'downstream', + }); + expect(file.target.type).toBe('File'); + expect(file.riskScale.comparableAcrossKinds).toBe(false); + expect(file.riskScale.unusedAxes).toEqual( + expect.arrayContaining([ + { + axis: 'processes', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + ]), + ); + }); + }, + { + seed: SEED, + poolAdapter: true, + afterSetup: async (handle) => { + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc123', + stats: { files: 14, nodes: 21, communities: 0, processes: 4 }, + }, + ]); + const backend = new LocalBackend(); + await backend.init(); + (handle as BackendHandle)._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/impact-zero-caller-risk.test.ts b/gitnexus/test/integration/impact-zero-caller-risk.test.ts index 3438ffc46..ad6653916 100644 --- a/gitnexus/test/integration/impact-zero-caller-risk.test.ts +++ b/gitnexus/test/integration/impact-zero-caller-risk.test.ts @@ -71,6 +71,8 @@ withTestLbugDB( expect(result).not.toHaveProperty('error'); expect(result.impactedCount).toBe(0); expect(result.risk).toBe('UNKNOWN'); + expect(result.riskScale.comparableAcrossKinds).toBe(true); + expect(result.riskNote).toBeDefined(); }); // The ambiguous fan-out narrows candidates into a fresh object, and that @@ -93,6 +95,11 @@ withTestLbugDB( expect(c.risk).toBe('UNKNOWN'); expect(typeof c.riskNote).toBe('string'); expect(c.riskNote).toMatch(/not evidence/i); + expect( + (c as { riskScale?: { unusedAxes?: { reason: string }[] } }).riskScale?.unusedAxes, + ).toEqual( + expect.arrayContaining([expect.objectContaining({ reason: 'enrichment-skipped' })]), + ); } }); diff --git a/gitnexus/test/unit/ai-context-unknown-risk-policy.test.ts b/gitnexus/test/unit/ai-context-unknown-risk-policy.test.ts index b906a1975..37f13de8b 100644 --- a/gitnexus/test/unit/ai-context-unknown-risk-policy.test.ts +++ b/gitnexus/test/unit/ai-context-unknown-risk-policy.test.ts @@ -30,6 +30,9 @@ describe('generateGitNexusContent keeps the risk: UNKNOWN policy unconditional ( const content = generateGitNexusContent('UnknownRiskProject', stats, { hasPdg }); expect(content).toContain('MUST treat `risk: UNKNOWN` as unresolved, not as low.'); + expect(content).toContain( + 'never use `riskSharedAxes` to waive a HIGH/CRITICAL `risk` warning', + ); expect(content).toContain( 'callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls)', ); diff --git a/gitnexus/test/unit/ai-context.test.ts b/gitnexus/test/unit/ai-context.test.ts index d8cf4b208..1e2611df8 100644 --- a/gitnexus/test/unit/ai-context.test.ts +++ b/gitnexus/test/unit/ai-context.test.ts @@ -225,6 +225,16 @@ describe('generateAIContextFiles', () => { expect(withoutPdg).toContain('explain('); }); + it('documents the MCP and Graph-RAG File-risk scale difference', () => { + const content = generateGitNexusContent('RiskScaleProject', { + nodes: 50, + edges: 100, + processes: 5, + }); + expect(content).toContain('MCP File omits axes'); + expect(content).toContain('Graph-RAG expands File'); + }); + it('emits MD060-compatible compact tables in generated docs (#2709)', () => { const content = generateGitNexusContent('MarkdownProject', { nodes: 50, diff --git a/gitnexus/test/unit/cli-impact-pdg-format.test.ts b/gitnexus/test/unit/cli-impact-pdg-format.test.ts index 9e9f4870f..e04f524af 100644 --- a/gitnexus/test/unit/cli-impact-pdg-format.test.ts +++ b/gitnexus/test/unit/cli-impact-pdg-format.test.ts @@ -498,9 +498,10 @@ describe('formatImpactResult — callgraph rendering is UNCHANGED (regression gu }, }; - it('renders the callgraph result with the exact pre-U5 text (byte-identical)', () => { + it('renders the callgraph result with Risk on the callgraph contract (byte-identical for U5+risk)', () => { const expected = [ 'Blast radius for Function computeTotal (upstream): 2 symbol(s) depends on this (will break if changed)', + 'Risk: MEDIUM', '', 'd=1: WILL BREAK (direct) (1)', ' Function callerA → src/a.ts [CALLS]', @@ -532,14 +533,14 @@ describe('formatImpactResult — callgraph rendering is UNCHANGED (regression gu expect(out).not.toContain('PDG-dependent symbols'); }); - it('renders the callgraph isolated / zero case unchanged', () => { + it('renders the callgraph isolated / zero case with risk, without claiming isolation', () => { const out = formatImpactResult({ target: { name: 'lonely' }, direction: 'downstream', impactedCount: 0, risk: 'LOW', }); - expect(out).toBe('lonely: No downstream dependencies found. This symbol appears isolated.'); + expect(out).toBe('lonely: No downstream dependencies found.\nRisk: LOW'); }); it('renders the callgraph lower-bound (DI/dynamic-dispatch) copy unchanged', () => { diff --git a/gitnexus/test/unit/eval-formatters.test.ts b/gitnexus/test/unit/eval-formatters.test.ts index c0349e80d..cdd12b4fd 100644 --- a/gitnexus/test/unit/eval-formatters.test.ts +++ b/gitnexus/test/unit/eval-formatters.test.ts @@ -409,6 +409,35 @@ describe('formatImpactResult', () => { expect(result).toContain('caller2'); }); + it('prints the shared-axes comparison when a target has unavailable risk axes', () => { + const result = formatImpactResult({ + target: { kind: 'File', name: 'crypto.ts' }, + direction: 'upstream', + impactedCount: 13, + risk: 'MEDIUM', + riskSharedAxes: 'MEDIUM', + riskScale: { + comparableAcrossKinds: false, + unusedAxes: [ + { + axis: 'processes', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + { + axis: 'modules', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + ], + }, + byDepthCounts: { 1: 13 }, + }); + + expect(result).toContain('Risk: MEDIUM'); + expect(result).toContain('Shared-axes risk: MEDIUM'); + expect(result).toContain('process/module axes are unavailable'); + expect(result).toContain('do not use this to waive a HIGH/CRITICAL risk warning'); + }); + it('handles zero impact', () => { const result = formatImpactResult({ target: { name: 'foo' }, @@ -416,7 +445,22 @@ describe('formatImpactResult', () => { impactedCount: 0, byDepth: {}, }); - expect(result).toContain('No upstream dependencies'); + expect(result).toContain('No upstream callers resolved'); + expect(result).not.toContain('appears isolated'); + }); + + it('prints UNKNOWN and riskNote for an empty upstream walk', () => { + const result = formatImpactResult({ + target: { name: 'foo' }, + direction: 'upstream', + impactedCount: 0, + risk: 'UNKNOWN', + riskNote: 'safe to change is a claim about callers and there were none to reason about', + byDepth: {}, + }); + expect(result).toContain('Risk: UNKNOWN'); + expect(result).toContain('safe to change is a claim about callers'); + expect(result).not.toContain('appears isolated'); }); it('formats impact by depth', () => { diff --git a/gitnexus/test/unit/group/cross-impact-fanout-cap.test.ts b/gitnexus/test/unit/group/cross-impact-fanout-cap.test.ts index aac1ba794..58b11451c 100644 --- a/gitnexus/test/unit/group/cross-impact-fanout-cap.test.ts +++ b/gitnexus/test/unit/group/cross-impact-fanout-cap.test.ts @@ -175,6 +175,32 @@ describe('group impact fan-out is bounded by a count, not by the clock (#2787)', expect(result).not.toHaveProperty('riskEpistemic'); }); + it('applies crossing risk to shared axes while preserving local scale metadata', async () => { + bridgeRows.value = [crossingRow(0, 0.9)]; + const riskScale = { + comparableAcrossKinds: true, + unusedAxes: [], + } as const; + const port = makePort({ + impact: vi.fn(async () => ({ + target: { id: 'Function:src/api.ts:publish', filePath: 'src/api.ts' }, + byDepth: { 1: [{ id: 'u1', filePath: 'src/a.ts' }] }, + summary: { direct: 1, processes_affected: 4, modules_affected: 0 }, + risk: 'HIGH', + riskSharedAxes: 'LOW', + riskScale, + })) as GroupToolPort['impact'], + }); + + const result = await run(port); + + expect(result).toMatchObject({ + risk: 'HIGH', + riskSharedAxes: 'HIGH', + riskScale, + }); + }); + it('marks risk as a floor when a crossing is dropped, and does NOT clamp the value down', async () => { // Three crossings, one neighbour unresolvable. Two traverse → HIGH (the // >=0.85-confidence gate). Had the third traversed, `traversed.length >= 3` diff --git a/gitnexus/test/unit/group/cross-impact.test.ts b/gitnexus/test/unit/group/cross-impact.test.ts index 9aac679a0..bceaa9195 100644 --- a/gitnexus/test/unit/group/cross-impact.test.ts +++ b/gitnexus/test/unit/group/cross-impact.test.ts @@ -371,6 +371,35 @@ describe('cross-impact', () => { expect(r).not.toHaveProperty('truncationReason'); }); + it('lifts local riskSharedAxes and riskScale when there are no symbol uids to fan out', async () => { + const riskScale = { + comparableAcrossKinds: false, + unusedAxes: [ + { + axis: 'processes' as const, + reason: 'file-nodes-have-no-process-or-community-membership', + }, + { + axis: 'modules', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + ], + }; + const r = await runLocalOnlyImpact(async () => ({ + byDepth: {}, + summary: { direct: 13, processes_affected: 0, modules_affected: 0 }, + risk: 'MEDIUM', + riskSharedAxes: 'MEDIUM', + riskScale, + })); + expect(r).toMatchObject({ + truncated: false, + risk: 'MEDIUM', + riskSharedAxes: 'MEDIUM', + riskScale, + }); + }); + it('test_runGroupImpact_bridge_schema_mismatch_returns_error', async () => { const { tmpDir, groupDir, cleanup } = tmpGroup(); vi.stubEnv('GITNEXUS_HOME', tmpDir); diff --git a/gitnexus/test/unit/group/types.test.ts b/gitnexus/test/unit/group/types.test.ts index 94b0b556c..50eddf921 100644 --- a/gitnexus/test/unit/group/types.test.ts +++ b/gitnexus/test/unit/group/types.test.ts @@ -6,6 +6,7 @@ import type { CrossLink, ContractRegistry, GroupManifestLink, + GroupImpactResult, MatchType, } from '../../../src/core/group/types.js'; @@ -132,4 +133,20 @@ describe('Group types', () => { }; expect(l.contract).toBe('/x'); }); + + it('uses the shared closed union for unused impact-axis reasons', () => { + type UnusedAxis = NonNullable['unusedAxes'][number]; + const valid = { + axis: 'processes', + reason: 'enrichment-query-failed', + } satisfies UnusedAxis; + const invalid = { + axis: 'processes', + // @ts-expect-error unknown reasons must not widen the shared contract + reason: 'not-a-real-reason', + } satisfies UnusedAxis; + + expect(valid.reason).toBe('enrichment-query-failed'); + expect(invalid.reason).toBe('not-a-real-reason'); + }); }); diff --git a/gitnexus/test/unit/impact-batching-grouping.test.ts b/gitnexus/test/unit/impact-batching-grouping.test.ts index 8224cd1d3..e068870f9 100644 --- a/gitnexus/test/unit/impact-batching-grouping.test.ts +++ b/gitnexus/test/unit/impact-batching-grouping.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; // Mock the lbug-adapter module before importing LocalBackend so the class // uses the mocked implementations of executeQuery / executeParameterized. @@ -38,6 +38,10 @@ describe('impact: batching and grouping', () => { vi.clearAllMocks(); }); + afterEach(() => { + delete process.env.IMPACT_MAX_CHUNKS; + }); + it('batches 250 IDs into 3 chunked STEP_IN_PROCESS queries', async () => { // Prepare backend and a fake repo handle const backend = new LocalBackend(); @@ -320,11 +324,328 @@ describe('impact: batching and grouping', () => { expect(Array.isArray(res.affected_modules)).toBe(true); const modNames = res.affected_modules.map((m: any) => m.name); expect(modNames).toContain('ModuleA'); + expect(res.riskScale.comparableAcrossKinds).toBe(false); + expect(res.riskScale.unusedAxes).toEqual( + expect.arrayContaining([expect.objectContaining({ reason: 'enrichment-truncated' })]), + ); // Cleanup env delete process.env.IMPACT_MAX_CHUNKS; }); + it('marks IMPACT_MAX_CHUNKS=0 as unused process/module axes', async () => { + process.env.IMPACT_MAX_CHUNKS = '0'; + const backend = new LocalBackend(); + const repoHandle = { + id: 'repo-zero-budget', + name: 'repo-zero-budget', + repoPath: '/tmp/repo-zero-budget', + storagePath: '/tmp/repo-zero-budget/.gitnexus', + lbugPath: '/tmp/repo-zero-budget/.gitnexus/lbug', + indexedAt: 'now', + lastCommit: 'c', + stats: {}, + } as any; + (backend as any).repos.set(repoHandle.id, repoHandle); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + executeQueryMock.mockImplementation(async () => []); + executeParameterizedMock.mockImplementation(async (...args: any[]) => { + const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { + return [ + { + id: 'node-1', + name: 'n1', + filePath: 'file-1.js', + relType: 'CALLS', + confidence: null, + }, + ]; + } + return [{ id: 'symX', name: 'TargetX', filePath: 'f' }]; + }); + + const res = await (backend as any)._impactImpl(repoHandle, { + target: 'TargetX', + direction: 'downstream', + maxDepth: 1, + } as any); + expect(res.riskScale.comparableAcrossKinds).toBe(false); + expect(res.riskScale.unusedAxes).toEqual( + expect.arrayContaining([expect.objectContaining({ reason: 'enrichment-budget-exhausted' })]), + ); + delete process.env.IMPACT_MAX_CHUNKS; + }); + + it('marks swallowed process enrichment failures as unused process axes', async () => { + const backend = new LocalBackend(); + const repoHandle = { + id: 'repo-enrich-fail', + name: 'repo-enrich-fail', + repoPath: '/tmp/repo-enrich-fail', + storagePath: '/tmp/repo-enrich-fail/.gitnexus', + lbugPath: '/tmp/repo-enrich-fail/.gitnexus/lbug', + indexedAt: 'now', + lastCommit: 'c', + stats: {}, + } as any; + (backend as any).repos.set(repoHandle.id, repoHandle); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + executeQueryMock.mockImplementation(async () => []); + executeParameterizedMock.mockImplementation(async (...args: any[]) => { + const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('STEP_IN_PROCESS')) { + throw new Error('process chunk failed'); + } + if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { + return [ + { + id: 'node-1', + name: 'n1', + filePath: 'file-1.js', + relType: 'CALLS', + confidence: null, + }, + ]; + } + return [{ id: 'symFail', name: 'TargetFail', filePath: 'f' }]; + }); + + const res = await (backend as any)._impactImpl(repoHandle, { + target: 'TargetFail', + direction: 'downstream', + maxDepth: 1, + } as any); + expect(res.riskScale.unusedAxes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ axis: 'processes', reason: 'enrichment-query-failed' }), + ]), + ); + expect(res.risk).toBe('UNKNOWN'); + expect(res.partial).toBe(true); + expect(res.riskNote).toContain('enrichment failed'); + }); + + it('marks module query failure without discarding a successful process axis', async () => { + const backend = new LocalBackend(); + const repoHandle = { + id: 'repo-module-fail', + name: 'repo-module-fail', + repoPath: '/tmp/repo-module-fail', + storagePath: '/tmp/repo-module-fail/.gitnexus', + lbugPath: '/tmp/repo-module-fail/.gitnexus/lbug', + indexedAt: 'now', + lastCommit: 'c', + stats: {}, + } as any; + (backend as any).repos.set(repoHandle.id, repoHandle); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + executeQueryMock.mockImplementation(async () => []); + executeParameterizedMock.mockImplementation(async (...args: any[]) => { + const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('MEMBER_OF')) throw new Error('module chunk failed'); + if (query.includes('STEP_IN_PROCESS') && query.includes('COUNT(DISTINCT s.id)')) { + return [ + { + pId: 'p1', + entryPointId: 'ep1', + epName: 'main', + epType: 'Function', + hits: 1, + minStep: 1, + }, + ]; + } + if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { + return [ + { + id: 'node-1', + name: 'n1', + filePath: 'file-1.js', + relType: 'CALLS', + confidence: null, + }, + ]; + } + return [{ id: 'symModule', name: 'TargetModule', filePath: 'f' }]; + }); + + const res = await (backend as any)._impactImpl(repoHandle, { + target: 'TargetModule', + direction: 'downstream', + maxDepth: 1, + } as any); + expect(res.affected_processes).toHaveLength(1); + expect(res.riskScale.unusedAxes).toEqual([ + { axis: 'modules', reason: 'enrichment-query-failed' }, + ]); + expect(res.risk).toBe('UNKNOWN'); + expect(res.partial).toBe(true); + }); + + it('keeps process risk measured when only minStep backfill fails', async () => { + const backend = new LocalBackend(); + const repoHandle = { + id: 'repo-backfill-fail', + name: 'repo-backfill-fail', + repoPath: '/tmp/repo-backfill-fail', + storagePath: '/tmp/repo-backfill-fail/.gitnexus', + lbugPath: '/tmp/repo-backfill-fail/.gitnexus/lbug', + indexedAt: 'now', + lastCommit: 'c', + stats: {}, + } as any; + (backend as any).repos.set(repoHandle.id, repoHandle); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + executeQueryMock.mockImplementation(async () => []); + executeParameterizedMock.mockImplementation(async (...args: any[]) => { + const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('MIN(r.step) AS minStep') && !query.includes('COUNT(DISTINCT s.id)')) { + throw new Error('minStep backfill failed'); + } + if (query.includes('STEP_IN_PROCESS') && query.includes('COUNT(DISTINCT s.id)')) { + return [ + { + pId: 'p1', + entryPointId: 'ep1', + epName: 'main', + epType: 'Function', + hits: 1, + minStep: null, + }, + ]; + } + if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { + return [ + { + id: 'node-1', + name: 'n1', + filePath: 'file-1.js', + relType: 'CALLS', + confidence: null, + }, + ]; + } + if (query.includes('MEMBER_OF')) return []; + return [{ id: 'symBackfill', name: 'TargetBackfill', filePath: 'f' }]; + }); + + const res = await (backend as any)._impactImpl(repoHandle, { + target: 'TargetBackfill', + direction: 'downstream', + maxDepth: 1, + } as any); + expect(res.affected_processes).toHaveLength(1); + expect(res.riskScale.unusedAxes).toEqual([]); + expect(res.risk).toBe('LOW'); + expect(res.partial).toBe(true); + }); + + it('keeps observed process warnings when a later enrichment chunk fails', async () => { + const backend = new LocalBackend(); + const repoHandle = { + id: 'repo-later-process-fail', + name: 'repo-later-process-fail', + repoPath: '/tmp/repo-later-process-fail', + storagePath: '/tmp/repo-later-process-fail/.gitnexus', + lbugPath: '/tmp/repo-later-process-fail/.gitnexus/lbug', + indexedAt: 'now', + lastCommit: 'c', + stats: {}, + } as any; + (backend as any).repos.set(repoHandle.id, repoHandle); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + executeQueryMock.mockImplementation(async () => []); + let processChunk = 0; + executeParameterizedMock.mockImplementation(async (...args: any[]) => { + const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('STEP_IN_PROCESS') && query.includes('COUNT(DISTINCT s.id)')) { + processChunk += 1; + if (processChunk === 2) throw new Error('later process chunk failed'); + return Array.from({ length: 5 }, (_, index) => ({ + pId: `p${index}`, + entryPointId: `ep${index}`, + epName: `process-${index}`, + epType: 'Function', + hits: 1, + minStep: 1, + })); + } + if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { + return Array.from({ length: 150 }, (_, index) => ({ + id: `node-${index}`, + name: `n${index}`, + filePath: `file-${index}.js`, + relType: 'CALLS', + confidence: null, + })); + } + if (query.includes('MEMBER_OF')) return []; + return [{ id: 'symLaterFail', name: 'TargetLaterFail', filePath: 'f' }]; + }); + + const res = await (backend as any)._impactImpl(repoHandle, { + target: 'TargetLaterFail', + direction: 'downstream', + maxDepth: 1, + } as any); + expect(res.affected_processes).toHaveLength(5); + expect(res.risk).toBe('CRITICAL'); + expect(res.riskScale.unusedAxes).toContainEqual({ + axis: 'processes', + reason: 'enrichment-query-failed', + }); + expect(res.partial).toBe(true); + }); + + it('does not invent direct/indirect module classification after backfill failure', async () => { + const backend = new LocalBackend(); + const repoHandle = { + id: 'repo-module-classification-fail', + name: 'repo-module-classification-fail', + repoPath: '/tmp/repo-module-classification-fail', + storagePath: '/tmp/repo-module-classification-fail/.gitnexus', + lbugPath: '/tmp/repo-module-classification-fail/.gitnexus/lbug', + indexedAt: 'now', + lastCommit: 'c', + stats: {}, + } as any; + (backend as any).repos.set(repoHandle.id, repoHandle); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + executeQueryMock.mockImplementation(async () => []); + executeParameterizedMock.mockImplementation(async (...args: any[]) => { + const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('MEMBER_OF') && query.includes('RETURN DISTINCT c.heuristicLabel')) { + throw new Error('module classification failed'); + } + if (query.includes('MEMBER_OF')) return [{ name: 'ModuleA', hits: 1 }]; + if (query.includes('STEP_IN_PROCESS')) return []; + if (query.includes('r.type IN')) { + return [ + { + id: 'node-1', + name: 'n1', + filePath: 'file-1.js', + relType: 'CALLS', + confidence: null, + }, + ]; + } + return [{ id: 'symClassify', name: 'TargetClassify', filePath: 'f' }]; + }); + + const res = await (backend as any)._impactImpl(repoHandle, { + target: 'TargetClassify', + direction: 'downstream', + maxDepth: 1, + } as any); + expect(res.affected_modules).toEqual([ + expect.objectContaining({ name: 'ModuleA', impact: 'classification-unavailable' }), + ]); + expect(res.riskScale.unusedAxes).toEqual([]); + expect(res.partial).toBe(true); + }); + it('caps implicit object-callable expansion and reports partial impact', async () => { const backend = new LocalBackend(); const repoHandle = { @@ -384,6 +705,10 @@ describe('impact: batching and grouping', () => { expect(traversalCall?.[2]?.frontierIds).toEqual(['owner']); expect(result.byDepth['1']).toHaveLength(5000); expect(result.partial).toBe(true); + expect(result.riskScale.comparableAcrossKinds).toBe(false); + expect(result.riskScale.unusedAxes).toEqual( + expect.arrayContaining([expect.objectContaining({ reason: 'enrichment-skipped' })]), + ); }); it('marks object impact partial when callable seeding fails', async () => { diff --git a/gitnexus/test/unit/impact-risk.test.ts b/gitnexus/test/unit/impact-risk.test.ts new file mode 100644 index 000000000..bd1b2661e --- /dev/null +++ b/gitnexus/test/unit/impact-risk.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from 'vitest'; +import { + scoreImpactRisk, + unusedAxesForImpactWalk, + type ImpactRiskInput, + type UnusedImpactRiskAxis, +} from 'gitnexus-shared'; + +const fileUnusedAxes: readonly UnusedImpactRiskAxis[] = [ + { + axis: 'processes', + reason: 'file-nodes-have-no-process-or-community-membership', + }, + { + axis: 'modules', + reason: 'file-nodes-have-no-process-or-community-membership', + }, +]; + +const base: ImpactRiskInput = { + direction: 'upstream', + directCount: 0, + processCount: 0, + moduleCount: 0, + impactedCount: 1, +}; + +describe('scoreImpactRisk', () => { + it('makes the issue #3075 File and Function scales explicit', () => { + const file = scoreImpactRisk({ + ...base, + directCount: 13, + impactedCount: 25, + unusedAxes: fileUnusedAxes, + }); + const fn = scoreImpactRisk({ + ...base, + directCount: 2, + processCount: 4, + moduleCount: 2, + impactedCount: 15, + }); + + expect(file).toEqual({ + risk: 'MEDIUM', + riskSharedAxes: 'MEDIUM', + riskScale: { + comparableAcrossKinds: false, + unusedAxes: fileUnusedAxes, + }, + }); + expect(fn).toEqual({ + risk: 'HIGH', + riskSharedAxes: 'LOW', + riskScale: { + comparableAcrossKinds: true, + unusedAxes: [], + }, + }); + }); + + it('preserves UNKNOWN only for an empty upstream walk', () => { + expect(scoreImpactRisk({ ...base, impactedCount: 0 }).risk).toBe('UNKNOWN'); + expect(scoreImpactRisk({ ...base, direction: 'downstream', impactedCount: 0 }).risk).toBe( + 'LOW', + ); + }); + + it('marks skipped enrichment as a non-comparable scale', () => { + const skippedAxes: readonly UnusedImpactRiskAxis[] = [ + { axis: 'processes', reason: 'enrichment-skipped' }, + { axis: 'modules', reason: 'enrichment-skipped' }, + ]; + + expect(scoreImpactRisk({ ...base, unusedAxes: skippedAxes }).riskScale).toEqual({ + comparableAcrossKinds: false, + unusedAxes: skippedAxes, + }); + }); + + it('preserves direct and total thresholds when enrichment axes are unused', () => { + expect( + scoreImpactRisk({ + ...base, + directCount: 30, + impactedCount: 30, + unusedAxes: fileUnusedAxes, + }).risk, + ).toBe('CRITICAL'); + expect( + scoreImpactRisk({ + ...base, + directCount: 15, + impactedCount: 15, + unusedAxes: fileUnusedAxes, + }).risk, + ).toBe('HIGH'); + expect( + scoreImpactRisk({ + ...base, + directCount: 2, + processCount: 10, + moduleCount: 10, + unusedAxes: fileUnusedAxes, + }).risk, + ).toBe('LOW'); + }); + + it('zeros unused process/module counts on the primary risk ladder', () => { + const skippedAxes: readonly UnusedImpactRiskAxis[] = [ + { axis: 'processes', reason: 'enrichment-skipped' }, + { axis: 'modules', reason: 'enrichment-skipped' }, + ]; + const scored = scoreImpactRisk({ + ...base, + directCount: 2, + processCount: 4, + moduleCount: 4, + impactedCount: 15, + unusedAxes: skippedAxes, + }); + expect(scored.risk).toBe('LOW'); + expect(scored.riskSharedAxes).toBe('LOW'); + }); + + it('preserves a warning already proved before a later enrichment failure', () => { + const scored = scoreImpactRisk({ + ...base, + directCount: 2, + processCount: 5, + impactedCount: 5, + unusedAxes: [{ axis: 'processes', reason: 'enrichment-query-failed' }], + }); + + expect(scored.risk).toBe('CRITICAL'); + expect(scored.riskSharedAxes).toBe('LOW'); + expect(scored.riskScale.comparableAcrossKinds).toBe(false); + }); + + it('fails closed when query failure leaves only a LOW or MEDIUM observed score', () => { + for (const directCount of [2, 6]) { + const scored = scoreImpactRisk({ + ...base, + direction: 'downstream', + directCount, + processCount: 0, + impactedCount: directCount, + unusedAxes: [{ axis: 'processes', reason: 'enrichment-query-failed' }], + }); + expect(scored.risk).toBe('UNKNOWN'); + } + }); + + it('fails closed when a truncated sample leaves only a LOW or MEDIUM observed score', () => { + const truncatedAxes: readonly UnusedImpactRiskAxis[] = [ + { axis: 'processes', reason: 'enrichment-truncated' }, + { axis: 'modules', reason: 'enrichment-truncated' }, + ]; + const scored = scoreImpactRisk({ + ...base, + direction: 'downstream', + directCount: 2, + processCount: 1, + moduleCount: 1, + impactedCount: 8, + unusedAxes: truncatedAxes, + }); + expect(scored.risk).toBe('UNKNOWN'); + expect(scored.riskScale.comparableAcrossKinds).toBe(false); + }); +}); + +describe('unusedAxesForImpactWalk', () => { + it('marks File, skipEnrichment, zero budget, and query failure distinctly', () => { + expect( + unusedAxesForImpactWalk({ + isFileTarget: true, + skipEnrichment: false, + maxChunks: 10, + processQueryFailed: true, + moduleQueryFailed: true, + impactedCount: 1, + }).every((a) => a.reason === 'file-nodes-have-no-process-or-community-membership'), + ).toBe(true); + expect( + unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: true, + maxChunks: 0, + processQueryFailed: false, + moduleQueryFailed: false, + impactedCount: 1, + }).map((a) => a.reason), + ).toEqual(['enrichment-skipped', 'enrichment-skipped']); + expect( + unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: false, + maxChunks: 0, + processQueryFailed: false, + moduleQueryFailed: false, + impactedCount: 1, + }).map((a) => a.reason), + ).toEqual(['enrichment-budget-exhausted', 'enrichment-budget-exhausted']); + expect( + unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: false, + maxChunks: 10, + processQueryFailed: true, + moduleQueryFailed: false, + impactedCount: 1, + }), + ).toEqual([{ axis: 'processes', reason: 'enrichment-query-failed' }]); + expect( + unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: false, + maxChunks: 0, + processQueryFailed: false, + moduleQueryFailed: false, + impactedCount: 0, + }), + ).toEqual([]); + expect( + unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: false, + maxChunks: 10, + processQueryFailed: false, + moduleQueryFailed: false, + impactedCount: 501, + enrichmentTruncated: true, + }).map((a) => a.reason), + ).toEqual(['enrichment-truncated', 'enrichment-truncated']); + }); +}); diff --git a/gitnexus/test/unit/shipped-skills-sync.test.ts b/gitnexus/test/unit/shipped-skills-sync.test.ts index c82e45321..984037e4c 100644 --- a/gitnexus/test/unit/shipped-skills-sync.test.ts +++ b/gitnexus/test/unit/shipped-skills-sync.test.ts @@ -170,6 +170,21 @@ describe('intended standard-skill improvements stay in every applicable copy', ( } }); + it('keeps the cross-surface risk-scale guidance in every impact-analysis copy', () => { + const required = [ + '`riskSharedAxes`', + 'MCP File walks', + 'web Graph-RAG expands File targets', + 'Within single-repo mode', + 'Within group mode', + 'overlays resolved', + ]; + for (const file of standardSkillCopies('gitnexus-impact-analysis')) { + const content = fs.readFileSync(file, 'utf-8'); + for (const fragment of required) expect(content).toContain(fragment); + } + }); + // Same shape as the UNKNOWN guard above, for the other half of the verdict: // `detect_changes` can come back SHORT — `partial` when a batched graph query // failed, `truncated` when the changed-symbol listing hit its cap — and both @@ -331,6 +346,10 @@ describe('root AGENTS.md / CLAUDE.md managed block keeps the risk: UNKNOWN polic const REQUIRED_FRAGMENTS = [ 'MUST treat `risk: UNKNOWN` as unresolved, not as low.', 'never read `UNKNOWN` as an all-clear', + 'never use `riskSharedAxes` to waive a HIGH/CRITICAL `risk` warning', + 'Compare File/symbol', + 'MCP File omits axes', + 'Graph-RAG expands File', ]; it.each(['AGENTS.md', 'CLAUDE.md'])('%s managed block documents the policy', (file) => { diff --git a/gitnexus/test/unit/tools.test.ts b/gitnexus/test/unit/tools.test.ts index d9a2890d9..700407f78 100644 --- a/gitnexus/test/unit/tools.test.ts +++ b/gitnexus/test/unit/tools.test.ts @@ -193,6 +193,17 @@ describe('GITNEXUS_TOOLS', () => { expect(impactTool.description).toContain('truncatedBy'); }); + it('documents riskSharedAxes as a compare aid, not the edit gate', () => { + const impactTool = GITNEXUS_TOOLS.find((t) => t.name === 'impact')!; + expect(impactTool.description).toContain('riskSharedAxes'); + expect(impactTool.description).toContain('Never substitute it for `risk`'); + expect(impactTool.description).toContain('IMPACT_MAX_CHUNKS=0'); + expect(impactTool.description).toContain('sampled a subset of impacted symbols'); + expect(impactTool.description).toContain('Graph-RAG'); + expect(impactTool.description).toContain('cross-repo crossing overlay'); + expect(impactTool.description).toContain('known HIGH/CRITICAL warnings survive'); + }); + it.each(['query', 'context', 'impact'])( '%s advertises an optional positive maxTokens budget', (name) => { From 170eefd4a0893be9d255dc3581d51fd448ce2e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 29 Aug 2026 13:56:41 +0100 Subject: [PATCH 21/28] fix(status): judge freshness by covered files, not a dirty working tree (#3083) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gitnexus status` reported "stale (re-run gitnexus analyze)" whenever the working tree held any modified or untracked file, including files the index never reads. Because `analyze` cannot commit, stash or delete such a file, the remedy it prescribed could not clear the verdict — the only way back to up-to-date was to remove the file. `meta.fileHashes` already records the exact set of files a run covered, so answer the question directly: compare those hashes against disk, reusing analyze's own scan, hash and diff helpers so the two cannot disagree about what "changed" means. A new coverable file still counts as stale (the index is genuinely incomplete then), but one `analyze` now settles it. The repo-wide dirty flag survives only as the fallback for metadata written before `fileHashes` existed. Both freshness checks now read GitNexus's own analyze output (AGENTS.md, CLAUDE.md, the agent skill mirrors) from one shared list. They previously held separate copies, and since analyze rewrites those files after recording hashes, a per-file comparison that missed them would report a freshly indexed repository as permanently stale. Closes #3077 Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- gitnexus/src/cli/i18n/en.ts | 11 + gitnexus/src/cli/i18n/zh-CN.ts | 9 + gitnexus/src/cli/status.ts | 107 ++++- gitnexus/src/core/index-content-drift.ts | 170 ++++++++ .../src/core/ingestion/filesystem-walker.ts | 37 +- .../core/ingestion/pipeline-phases/scan.ts | 37 +- gitnexus/src/core/run-analyze.ts | 13 + gitnexus/src/storage/file-hash.ts | 22 +- gitnexus/src/storage/git.ts | 116 +++-- .../src/storage/gitnexus-managed-paths.ts | 53 +++ gitnexus/src/storage/repo-meta.ts | 12 + gitnexus/test/unit/git-utils.test.ts | 207 +++++++++ .../test/unit/index-content-drift.test.ts | 404 ++++++++++++++++++ gitnexus/test/unit/list-status-branch.test.ts | 1 + .../test/unit/status-content-drift.test.ts | 279 ++++++++++++ 15 files changed, 1429 insertions(+), 49 deletions(-) create mode 100644 gitnexus/src/core/index-content-drift.ts create mode 100644 gitnexus/src/storage/gitnexus-managed-paths.ts create mode 100644 gitnexus/test/unit/index-content-drift.test.ts create mode 100644 gitnexus/test/unit/status-content-drift.test.ts diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 15e1644e7..698e27d4e 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -33,6 +33,17 @@ export const en = { 'status.workspaceIndexLabel': "Workspace index: last analyzed on '{{primary}}' (re-run gitnexus analyze to follow the current branch)", 'status.status': 'Status', + 'status.indexContentCurrent': 'Index content: matches all {{count}} covered file(s)', + 'status.indexContentDrifted': + 'Index content: {{changed}} changed, {{added}} added, {{deleted}} deleted', + 'status.indexContentMore': ' ...and {{count}} more {{label}}', + 'status.indexContentUnmeasurable': + 'Index content: not comparable ({{reason}}); fell back to the working-tree check', + 'status.indexContentScanFailed': + 'Index content: coverage scan failed; treating the index as stale', + 'status.driftChanged': 'changed', + 'status.driftAdded': 'added', + 'status.driftDeleted': 'deleted', 'status.upToDate': '✅ up-to-date', 'status.stale': '⚠️ stale (re-run gitnexus analyze)', 'clean.deleteAll': 'This will delete GitNexus indexes for {{count}} repo(s):', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 63e290e54..24a6a2ad6 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -37,6 +37,15 @@ export const zhCN = { 'status.workspaceIndexLabel': "工作区索引：最近在 '{{primary}}' 分支上分析（重新运行 gitnexus analyze 以跟随当前分支）", 'status.status': '状态', + 'status.indexContentCurrent': '索引内容：与覆盖的全部 {{count}} 个文件一致', + 'status.indexContentDrifted': + '索引内容：{{changed}} 个已修改，{{added}} 个新增，{{deleted}} 个已删除', + 'status.indexContentMore': ' ……另有 {{count}} 个 {{label}}', + 'status.indexContentUnmeasurable': '索引内容：无法比对（{{reason}}），已回退到工作区检查', + 'status.indexContentScanFailed': '索引内容：覆盖扫描失败，按过期处理', + 'status.driftChanged': '已修改', + 'status.driftAdded': '新增', + 'status.driftDeleted': '已删除', 'status.upToDate': '✅ 已是最新', 'status.stale': '⚠️ 已过期（重新运行 gitnexus analyze）', 'clean.deleteAll': '将删除 {{count}} 个仓库的 GitNexus 索引：', diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index 09eb415d3..e0ee08bcc 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -18,8 +18,69 @@ import { resolveAnalyzerRunnerIdentity, } from '../core/analyzer-identity.js'; import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { detectIndexContentDrift, type IndexContentDrift } from '../core/index-content-drift.js'; import { t } from './i18n/index.js'; +/** How many drifted paths the report names before summarizing the rest. */ +const DRIFT_SAMPLE_LIMIT = 10; + +/** + * Machine-readable form of the per-file comparison. `'not-checked'` is its own + * value rather than a silent omission: it says the index was already stale on + * metadata alone, so the scan was skipped, which is not the same claim as a + * scan that ran and found nothing. + */ +const describeContentDrift = (drift: IndexContentDrift | undefined) => { + if (!drift) return { status: 'not-checked' as const }; + if (drift.kind === 'current') { + return { status: 'current' as const, coveredFiles: drift.coveredFileCount }; + } + if (drift.kind === 'unmeasurable') { + return { status: 'unmeasurable' as const, reason: drift.reason }; + } + return { + status: 'drifted' as const, + counts: { + changed: drift.changed.length, + added: drift.added.length, + deleted: drift.deleted.length, + }, + changed: drift.changed.slice(0, DRIFT_SAMPLE_LIMIT), + added: drift.added.slice(0, DRIFT_SAMPLE_LIMIT), + deleted: drift.deleted.slice(0, DRIFT_SAMPLE_LIMIT), + truncated: { + changed: drift.changed.length > DRIFT_SAMPLE_LIMIT, + added: drift.added.length > DRIFT_SAMPLE_LIMIT, + deleted: drift.deleted.length > DRIFT_SAMPLE_LIMIT, + }, + }; +}; + +/** Escape control characters in repo-relative paths before printing. */ +const formatDriftPath = (rel: string): string => + /[\u0000-\u001f\u007f]/.test(rel) ? JSON.stringify(rel) : rel; +const printDriftDetail = (drift: Extract): void => { + console.log( + t('status.indexContentDrifted', { + changed: drift.changed.length, + added: drift.added.length, + deleted: drift.deleted.length, + }), + ); + const labelled: [string, readonly string[]][] = [ + [t('status.driftChanged'), drift.changed], + [t('status.driftAdded'), drift.added], + [t('status.driftDeleted'), drift.deleted], + ]; + for (const [label, paths] of labelled) { + for (const p of paths.slice(0, DRIFT_SAMPLE_LIMIT)) { + console.log(` ${label}: ${formatDriftPath(p)}`); + } + const remaining = paths.length - DRIFT_SAMPLE_LIMIT; + if (remaining > 0) console.log(t('status.indexContentMore', { count: remaining, label })); + } +}; + export interface StatusOptions { json?: boolean; } @@ -85,14 +146,36 @@ export const statusCommand = async (options: StatusOptions = {}) => { currentRunnerIdentity, ); const incompleteReasons = getIndexIncompleteReasons(activeMeta); - // A matching HEAD is not enough: `analyze` re-indexes a dirty working tree, - // so a repo with uncommitted source changes is stale even at the same commit. - // Skip the check for non-git folders (currentCommit === '') to match analyze. - const isUpToDate = + const metadataIsCurrent = currentCommit === activeMeta.lastCommit && runnerIdentityIsCurrent && - incompleteReasons.length === 0 && - (currentCommit === '' || !isWorkingTreeDirty(repo.repoPath)); + incompleteReasons.length === 0; + + // A matching HEAD is not enough: `analyze` re-indexes changed content at the + // same commit, so the files the index covers must still be compared against + // disk. Only worth the scan once the cheap metadata checks agree, and skipped + // for non-git folders (currentCommit === '') to match analyze. + const contentDrift: IndexContentDrift | undefined = + metadataIsCurrent && currentCommit !== '' + ? await detectIndexContentDrift( + repo.repoPath, + activeMeta.fileHashes, + activeMeta.indexCoverage, + ) + : undefined; + + // The repo-wide dirty flag survives only as the fallback for metadata written + // before `fileHashes` existed. Where the per-file comparison can run it + // decides, so a file the index does not cover no longer pins a byte-current + // index to a "stale" verdict that `analyze` is powerless to clear (#3077). + const contentIsCurrent = + contentDrift === undefined || + contentDrift.kind === 'current' || + (contentDrift.kind === 'unmeasurable' && + contentDrift.reason === 'no-file-hashes' && + !isWorkingTreeDirty(repo.repoPath)); + + const isUpToDate = metadataIsCurrent && contentIsCurrent; if (options.json) { console.log( JSON.stringify({ @@ -111,6 +194,7 @@ export const statusCommand = async (options: StatusOptions = {}) => { commit: currentCommit, runnerIdentity: currentRunnerIdentity, }, + contentDrift: describeContentDrift(contentDrift), status: isUpToDate ? 'up-to-date' : 'stale', }), ); @@ -137,5 +221,16 @@ export const statusCommand = async (options: StatusOptions = {}) => { console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`); } console.log(`${t('status.currentRunnerIdentity')}: ${JSON.stringify(currentRunnerIdentity)}`); + if (contentDrift?.kind === 'current') { + console.log(t('status.indexContentCurrent', { count: contentDrift.coveredFileCount })); + } else if (contentDrift?.kind === 'drifted') { + printDriftDetail(contentDrift); + } else if (contentDrift?.kind === 'unmeasurable') { + if (contentDrift.reason === 'scan-failed') { + console.log(t('status.indexContentScanFailed')); + } else if (!isUpToDate) { + console.log(t('status.indexContentUnmeasurable', { reason: contentDrift.reason })); + } + } console.log(`${t('status.status')}: ${isUpToDate ? t('status.upToDate') : t('status.stale')}`); }; diff --git a/gitnexus/src/core/index-content-drift.ts b/gitnexus/src/core/index-content-drift.ts new file mode 100644 index 000000000..3599db7b1 --- /dev/null +++ b/gitnexus/src/core/index-content-drift.ts @@ -0,0 +1,170 @@ +/** + * Does the index still reflect the files it actually covers? + * + * `status` used to answer this with a repo-wide `git status --porcelain` + * boolean, which says something different: whether the working tree differs + * from HEAD. Those two questions diverge in both directions. A scratch file, + * a build artifact, or a tracked file under a tool directory the indexer + * never reads makes the tree dirty while every indexed file is byte-current — + * and because `analyze` cannot commit or delete that file, the resulting + * "stale (re-run gitnexus analyze)" verdict was unclearable (#3077). It also + * misses the reverse case: reverting a file that was indexed while dirty + * leaves a clean tree over an index holding the pre-revert content. + * + * `meta.fileHashes` already records the exact set of files the last run + * covered, so the question can be answered directly. This module recomputes + * the coverage set with the same `walkRepositoryPaths` scan (ignore rules and + * dotfile handling stay shared) and the large-file cap recorded in + * `meta.indexCoverage`, hashes only the paths that can actually have changed + * since that run, and diffs against what was recorded. + */ + +import { constants as fsConstants } from 'node:fs'; +import { access } from 'node:fs/promises'; +import path from 'node:path'; +import { walkRepositoryPaths } from './ingestion/filesystem-walker.js'; +import { computeFileHashesDetailed } from '../storage/file-hash.js'; +import { listWorkingTreeDirtyPaths } from '../storage/git.js'; +import { isGitNexusManagedPath } from '../storage/gitnexus-managed-paths.js'; +import { chunk } from '../lib/utils.js'; +import { logger } from './logger.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; + +/** Why the recorded coverage set could not be compared against disk at all. */ +export type IndexContentUnmeasurableReason = + /** Metadata predates per-file hashes, or the run recorded none (non-git). */ + | 'no-file-hashes' + /** The repository scan or hashing pass threw. */ + | 'scan-failed'; + +/** + * A three-way verdict. `'unmeasurable'` is kept apart from `'current'` on + * purpose: it means the comparison never ran, which is not evidence the index + * is fresh. Legacy metadata without hashes still falls back to the working-tree + * check; a failed scan must not. + */ +export type IndexContentDrift = + | { kind: 'current'; coveredFileCount: number } + | { kind: 'drifted'; changed: string[]; added: string[]; deleted: string[] } + | { kind: 'unmeasurable'; reason: IndexContentUnmeasurableReason }; + +export type IndexCoveragePolicy = NonNullable; + +const HASH_BATCH = 100; + +const collectUnreadablePaths = async ( + repoPath: string, + relPaths: readonly string[], +): Promise => { + const unreadable: string[] = []; + for (const batch of chunk(relPaths, HASH_BATCH)) { + await Promise.all( + batch.map(async (rel) => { + try { + await access(path.join(repoPath, rel), fsConstants.R_OK); + } catch { + unreadable.push(rel); + } + }), + ); + } + unreadable.sort(); + return unreadable; +}; + +/** + * Compare the files recorded in `fileHashes` against the current working tree. + * + * `added` covers files the index would pick up but has never seen, so a new + * source file still reports stale — the index is genuinely incomplete then, + * and comparing only the recorded entries would wave that through. + */ +export const detectIndexContentDrift = async ( + repoPath: string, + fileHashes: Readonly> | undefined, + coverage?: IndexCoveragePolicy, +): Promise => { + if (!fileHashes || Object.keys(fileHashes).length === 0) { + return { kind: 'unmeasurable', reason: 'no-file-hashes' }; + } + + // Excluded from BOTH sides, or GitNexus's own output guarantees a mismatch: + // analyze rewrites AGENTS.md/CLAUDE.md after recording hashes, so they read + // as `added` on a first run and `changed` on every run after that — a fresh + // index would report itself stale forever. + const recorded = Object.fromEntries( + Object.entries(fileHashes).filter(([rel]) => !isGitNexusManagedPath(rel)), + ); + if (Object.keys(recorded).length === 0) { + return { kind: 'unmeasurable', reason: 'no-file-hashes' }; + } + + try { + const scanned = await walkRepositoryPaths(repoPath, undefined, { + quiet: true, + maxFileSizeBytes: coverage?.maxFileSizeBytes, + }); + const scannedPaths = scanned.map((file) => file.path).filter((p) => !isGitNexusManagedPath(p)); + const scannedSet = new Set(scannedPaths); + const recordedSet = new Set(Object.keys(recorded)); + + // Legacy indexes have `fileHashes` but no `indexCoverage`. A later default + // cap would omit a still-present hashed file and call it deleted. Recorded + // paths that still exist stay in the coverage set even if this walk skipped + // them for size. + const recovered = new Set(); + for (const rel of recordedSet) { + if (scannedSet.has(rel)) continue; + try { + await access(path.join(repoPath, rel), fsConstants.R_OK); + recovered.add(rel); + scannedSet.add(rel); + } catch { + // Missing or unreadable: stays deleted / changed below. + } + } + + const added = scannedPaths.filter((p) => !recordedSet.has(p)).sort(); + const deleted = [...recordedSet].filter((p) => !scannedSet.has(p)).sort(); + const intersection = [...recordedSet].filter((p) => scannedSet.has(p)); + + const dirtyNow = listWorkingTreeDirtyPaths(repoPath); + const dirtyAtIndex = coverage?.dirtyPaths; + const dirtyNowSet = dirtyNow === null ? null : new Set(dirtyNow); + const dirtyAtIndexSet = dirtyAtIndex === undefined ? undefined : new Set(dirtyAtIndex); + const hashCandidates = + dirtyNowSet === null || dirtyAtIndexSet === undefined + ? intersection + : intersection.filter( + (p) => dirtyAtIndexSet.has(p) || dirtyNowSet.has(p) || recovered.has(p), + ); + + const hashCandidateSet = new Set(hashCandidates); + const skipHash = intersection.filter((p) => !hashCandidateSet.has(p)); + const unreadableFromAccess = await collectUnreadablePaths(repoPath, skipHash); + const unreadableSet = new Set(unreadableFromAccess); + const { hashes: hashed, unreadable: unreadableFromHash } = await computeFileHashesDetailed( + repoPath, + hashCandidates, + ); + for (const p of unreadableFromHash) unreadableSet.add(p); + const changed: string[] = []; + for (const p of intersection) { + if (unreadableSet.has(p)) { + changed.push(p); + continue; + } + const currentHash = hashed.get(p) ?? recorded[p]; + if (currentHash !== recorded[p]) changed.push(p); + } + changed.sort(); + + if (changed.length === 0 && added.length === 0 && deleted.length === 0) { + return { kind: 'current', coveredFileCount: scannedSet.size }; + } + return { kind: 'drifted', changed, added, deleted }; + } catch (err) { + logger.warn({ err, repoPath }, 'index content drift scan failed'); + return { kind: 'unmeasurable', reason: 'scan-failed' }; + } +}; diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 804921a62..7e41c07c2 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -57,16 +57,49 @@ const warnLargeFileSkip = (message: string): void => { logger.warn(message); }; +export interface WalkRepositoryOptions { + /** + * Suppress the operator-facing large-file notice. Set by read-only callers + * such as `status`, which reuse this scan purely to learn which files the + * index covers and must not emit analyze's progress commentary. + */ + quiet?: boolean; + /** + * Override the large-file cap. `status` replays the bytes recorded at + * analyze time so `--max-file-size` / `GITNEXUS_MAX_FILE_SIZE` cannot + * silently drop a file that the index actually covers. + */ + maxFileSizeBytes?: number; +} + /** * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded. * Memory: ~10MB for 100K files vs ~1GB+ with content. */ +const assertWalkRootIsDirectory = async (repoPath: string): Promise => { + let st; + try { + st = await fs.stat(repoPath); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT' || code === 'ENOTDIR') { + throw new Error(`walkRepositoryPaths: path does not exist: ${repoPath}`); + } + throw err; + } + if (!st.isDirectory()) { + throw new Error(`walkRepositoryPaths: not a directory: ${repoPath}`); + } +}; + export const walkRepositoryPaths = async ( repoPath: string, onProgress?: (current: number, total: number, filePath: string) => void, + options: WalkRepositoryOptions = {}, ): Promise => { + await assertWalkRootIsDirectory(repoPath); const ignoreFilter = await createIgnoreFilter(repoPath); - const maxFileSizeBytes = getMaxFileSizeBytes(); + const maxFileSizeBytes = options.maxFileSizeBytes ?? getMaxFileSizeBytes(); const filtered = await glob('**/*', { cwd: repoPath, @@ -117,7 +150,7 @@ export const walkRepositoryPaths = async ( left.path < right.path ? -1 : left.path > right.path ? 1 : 0, ); - if (skippedLarge > 0) { + if (skippedLarge > 0 && !options.quiet) { const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; const isOverrideUnset = !process.env.GITNEXUS_MAX_FILE_SIZE; const suffix = isDefault ? ', likely generated/vendored' : ''; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/scan.ts b/gitnexus/src/core/ingestion/pipeline-phases/scan.ts index 5a1353267..f8629ea0d 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/scan.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/scan.ts @@ -30,20 +30,31 @@ export const scanPhase: PipelinePhase = { message: 'Scanning repository...', }); - const scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => { - const scanProgress = Math.round((current / total) * 15); - ctx.onProgress({ - phase: 'extracting', - percent: scanProgress, - message: 'Scanning repository...', - detail: filePath, - stats: { - filesProcessed: current, - totalFiles: total, - nodesCreated: ctx.graph.nodeCount, - }, + let scannedFiles; + try { + scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => { + const scanProgress = Math.round((current / total) * 15); + ctx.onProgress({ + phase: 'extracting', + percent: scanProgress, + message: 'Scanning repository...', + detail: filePath, + stats: { + filesProcessed: current, + totalFiles: total, + nodesCreated: ctx.graph.nodeCount, + }, + }); }); - }); + } catch (err) { + // Missing roots throw so status cannot treat an empty glob as "every + // covered file was deleted". The pipeline still reports an empty scan + // for a path that is not a directory, matching analyze of a bad cwd. + if (err instanceof Error && err.message.startsWith('walkRepositoryPaths:')) { + return { scannedFiles: [], allPaths: [], totalFiles: 0 }; + } + throw err; + } const totalFiles = scannedFiles.length; const allPaths = scannedFiles.map((f) => f.path); diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 94d53de16..018efc42a 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -153,8 +153,11 @@ import { hasGitDir, getInferredRepoName, isWorkingTreeDirty, + listWorkingTreeDirtyPaths, resolveRepoIdentityRoot, } from '../storage/git.js'; +import { isGitNexusManagedPath } from '../storage/gitnexus-managed-paths.js'; +import { getMaxFileSizeBytes } from './ingestion/utils/max-file-size.js'; import type { CachedEmbedding } from './embeddings/types.js'; import { generateAIContextFiles } from '../cli/ai-context.js'; import { sanitizeDetectedBranch } from '../cli/analyze-config.js'; @@ -3642,6 +3645,16 @@ async function runFullAnalysisInner( // absence has exactly one meaning — an index older than the field. embeddingDims: EMBEDDING_DIMS, fileHashes: hasGitDir(repoPath) ? newFileHashesRecord : undefined, + indexCoverage: hasGitDir(repoPath) + ? { + maxFileSizeBytes: getMaxFileSizeBytes(), + dirtyPaths: ( + listWorkingTreeDirtyPaths(repoPath) ?? Object.keys(newFileHashesRecord) + ).filter( + (rel) => newFileHashesRecord[rel] !== undefined && !isGitNexusManagedPath(rel), + ), + } + : undefined, // This branch's full live chunk-key set (#2106 R6). `usedKeys` is every // chunk hash touched in this scan — cache HITS included (see parse-impl // usedKeys.add) — so it's complete even on an incremental run. Persisted diff --git a/gitnexus/src/storage/file-hash.ts b/gitnexus/src/storage/file-hash.ts index 2b2fc9491..2c002a83a 100644 --- a/gitnexus/src/storage/file-hash.ts +++ b/gitnexus/src/storage/file-hash.ts @@ -44,18 +44,32 @@ export const computeFileHashes = async ( repoPath: string, relPaths: readonly string[], ): Promise> => { - const out = new Map(); + const { hashes } = await computeFileHashesDetailed(repoPath, relPaths); + return hashes; +}; + +/** Like {@link computeFileHashes}, but keeps paths whose content could not be read. */ +export const computeFileHashesDetailed = async ( + repoPath: string, + relPaths: readonly string[], +): Promise<{ hashes: Map; unreadable: string[] }> => { + const hashes = new Map(); + const unreadable: string[] = []; const BATCH = 100; for (const batch of chunk(relPaths, BATCH)) { const results = await Promise.all( batch.map(async (rel) => { const h = await computeFileHash(path.join(repoPath, rel)); - return h ? ([rel, h] as const) : null; + return { rel, h }; }), ); - for (const r of results) if (r) out.set(r[0], r[1]); + for (const { rel, h } of results) { + if (h) hashes.set(rel, h); + else unreadable.push(rel); + } } - return out; + unreadable.sort(); + return { hashes, unreadable }; }; /** Result of comparing the current on-disk hashes against stored ones. */ diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index f4bf32366..792c1010b 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -4,42 +4,32 @@ import path from 'path'; import os from 'os'; import { logger } from '../core/logger.js'; import { toZeroBasedLine } from '../core/ingestion/utils/line-base.js'; +import { GITNEXUS_MANAGED_PATH_EXCLUDES, isGitNexusManagedPath } from './gitnexus-managed-paths.js'; // Git utilities for repository detection, commit tracking, and diff analysis const chompGitOutput = (value: Buffer): string => value.toString().replace(/\r?\n$/, ''); +const GIT_PATH_LIST_MAX_BUFFER = 64 * 1024 * 1024; /** * True when the working tree has uncommitted changes that analyze would - * re-index, even at a matching HEAD. Excludes the paths GitNexus writes during - * analyze (.gitnexus/, .claude/, .cursor/, AGENTS.md, CLAUDE.md, and the - * repo-local .agents/ mirror) so its own output never counts as dirty - * (regression vs PR #1233 behavior). The entire .agents/ tree is excluded, - * matching the .claude/ treatment, because the skill mirror writes across - * .agents/skills/ and deeper paths. Conservative on any git failure. Shared - * so `analyze`'s fast-path gate and `status`'s freshness report agree on what - * "dirty" means. + * re-index, even at a matching HEAD. Excludes GITNEXUS_MANAGED_PATHS so + * GitNexus's own analyze output never counts as dirty (regression vs PR #1233 + * behavior); whole directory trees are excluded, not just their root entries, + * because the skill mirror writes across .agents/skills/ and deeper paths. + * Conservative on any git failure. + * + * This drives `analyze`'s up-to-date fast path. It is deliberately coarse: + * a false "dirty" here costs only a hash diff that finds nothing. `status` + * reaches for the per-file comparison in core/index-content-drift.ts instead, + * because there the same false positive is a verdict the user cannot clear + * (#3077), and falls back to this only when that comparison cannot run. */ export const isWorkingTreeDirty = (repoPath: string): boolean => { try { const out = execFileSync( 'git', - [ - 'status', - '--porcelain', - '--', - '.', - ':(exclude).gitnexus', - ':(exclude).gitnexus/**', - ':(exclude).claude', - ':(exclude).claude/**', - ':(exclude).cursor', - ':(exclude).cursor/**', - ':(exclude)AGENTS.md', - ':(exclude)CLAUDE.md', - ':(exclude).agents', - ':(exclude).agents/**', - ], + ['status', '--porcelain', '--', '.', ...GITNEXUS_MANAGED_PATH_EXCLUDES], { cwd: repoPath, stdio: ['ignore', 'pipe', 'ignore'], @@ -53,6 +43,84 @@ export const isWorkingTreeDirty = (repoPath: string): boolean => { } }; +const parsePorcelainPaths = (porcelain: string): string[] => { + const paths = new Set(); + const records = porcelain.split('\0'); + for (let i = 0; i < records.length; i++) { + const record = records[i]; + if (record.length < 4) continue; + + const status = record.slice(0, 2); + paths.add(record.slice(3)); + + // In porcelain v1 `-z` mode, rename/copy source and destination paths are + // separate NUL records (with no human-facing ` -> ` delimiter). Keep both: + // either side may be present in the previous coverage set. + if (status.includes('R') || status.includes('C')) { + const pairedPath = records[++i]; + if (pairedPath) paths.add(pairedPath); + } + } + return [...paths]; +}; + +const gitPathListExec = { + stdio: ['ignore', 'pipe', 'ignore'] as ['ignore', 'pipe', 'ignore'], + encoding: 'utf8' as const, + maxBuffer: GIT_PATH_LIST_MAX_BUFFER, +}; + +const listHiddenIndexPaths = (repoPath: string): string[] => { + const out = execFileSync('git', ['ls-files', '-v', '-z', '--'], { + cwd: repoPath, + windowsHide: true, + ...gitPathListExec, + }); + const paths: string[] = []; + for (const record of out.split('\0')) { + if (record.length < 3 || record[1] !== ' ') continue; + const tag = record[0]; + // `S` marks skip-worktree. With `-v`, an assume-unchanged entry's + // ordinary tag is lower-cased (`H` -> `h`, `S` -> `s`, etc.). + if (tag === 'S' || (tag >= 'a' && tag <= 'z')) paths.push(record.slice(2)); + } + return paths; +}; + +/** + * Repo-relative paths `git status` reports as dirty or untracked, using the + * same managed-path excludes as {@link isWorkingTreeDirty}, plus tracked paths + * whose assume-unchanged or skip-worktree bits can hide content changes from + * porcelain. `null` means either query failed — callers must not treat that as + * a clean tree. + */ +export const listWorkingTreeDirtyPaths = (repoPath: string): string[] | null => { + try { + const out = execFileSync( + 'git', + [ + 'status', + '--porcelain=v1', + '-z', + '--untracked-files=all', + '--', + '.', + ...GITNEXUS_MANAGED_PATH_EXCLUDES, + ], + { cwd: repoPath, windowsHide: true, ...gitPathListExec }, + ); + return [ + ...new Set( + [...parsePorcelainPaths(out), ...listHiddenIndexPaths(repoPath)].filter( + (rel) => !isGitNexusManagedPath(rel), + ), + ), + ]; + } catch { + return null; + } +}; + /** * Snapshot, per candidate file, whether it is safe for `selfCommitContextFiles` * to auto-commit — call this BEFORE `analyze` writes AGENTS.md/CLAUDE.md. diff --git a/gitnexus/src/storage/gitnexus-managed-paths.ts b/gitnexus/src/storage/gitnexus-managed-paths.ts new file mode 100644 index 000000000..cbdd7b81b --- /dev/null +++ b/gitnexus/src/storage/gitnexus-managed-paths.ts @@ -0,0 +1,53 @@ +/** + * The paths GitNexus itself writes during `analyze`. + * + * `analyze` rewrites the stats blocks in AGENTS.md/CLAUDE.md and refreshes the + * agent skill mirrors as its final step — after it has recorded the per-file + * hashes for the run. Counting its own output as a repository change makes + * every completed run look immediately out of date, which is the regression + * PR #1233 introduced and #1233's fix excluded these paths to prevent. + * + * Two freshness checks depend on this list agreeing: `isWorkingTreeDirty` + * (analyze's up-to-date fast-path gate) and the per-file comparison behind + * `status`. They used to hold separate copies of it, so a path added to one + * silently became a permanent "stale" verdict in the other. One list, imported + * by both. + */ + +/** + * Repository-root-relative. A directory entry covers everything beneath it; + * a file entry matches only itself. Prefix collisions are NOT matches — + * `.agentsrc` is an ordinary file, not part of the `.agents` tree. + */ +export const GITNEXUS_MANAGED_PATHS = [ + '.gitnexus', + '.claude', + '.cursor', + '.agents', + 'AGENTS.md', + 'CLAUDE.md', +] as const; + +/** + * Git pathspecs excluding {@link GITNEXUS_MANAGED_PATHS} from a `git status` + * run rooted at the repository. Patterns include `./` so they match only at + * the repo root: a slash-free `:(exclude)AGENTS.md` would also drop + * `docs/AGENTS.md`, which {@link isGitNexusManagedPath} does not treat as + * managed. Both forms are emitted per entry: the root path itself, and `/**` + * for directory contents. + */ +export const GITNEXUS_MANAGED_PATH_EXCLUDES: readonly string[] = GITNEXUS_MANAGED_PATHS.flatMap( + (managed) => [`:(exclude,glob)./${managed}`, `:(exclude,glob)./${managed}/**`], +); + +/** + * True when a repository-relative path is GitNexus's own output. Mirrors the + * pathspec semantics above: root-relative, whole path segments only, so + * neither `.agentsrc` nor a nested `subdir/.agents/` is treated as managed. + */ +export const isGitNexusManagedPath = (relPath: string): boolean => { + const normalized = relPath.replace(/\\/g, '/'); + return GITNEXUS_MANAGED_PATHS.some( + (managed) => normalized === managed || normalized.startsWith(`${managed}/`), + ); +}; diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 3cda63545..b7ab3b871 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -284,6 +284,18 @@ export interface RepoMeta { * Map keys are repo-relative paths. */ fileHashes?: Record; + /** + * Coverage policy used when `fileHashes` was recorded. `status` replays it + * so analyze-time `--max-file-size` / `GITNEXUS_MAX_FILE_SIZE` cannot make + * a later default-cap walk drop a file the index actually covers. + * `dirtyPaths` are covered files that were dirty vs HEAD at that moment — + * status must re-hash those even after Git becomes clean (indexed-dirty then + * restore). Absent on indexes written before this field. + */ + indexCoverage?: { + maxFileSizeBytes: number; + dirtyPaths?: string[]; + }; /** * Set when a run finished but the persisted edge count came back far short * of what the pipeline produced — the B2 "refresh reports SUCCESS while the diff --git a/gitnexus/test/unit/git-utils.test.ts b/gitnexus/test/unit/git-utils.test.ts index feb49effd..a5cb94083 100644 --- a/gitnexus/test/unit/git-utils.test.ts +++ b/gitnexus/test/unit/git-utils.test.ts @@ -928,3 +928,210 @@ describe('isWorkingTreeDirty', () => { } }); }); + +describe('listWorkingTreeDirtyPaths', () => { + it('returns an empty list for a clean repository', async () => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execFileSync(gitExecutable, ['add', '--', 'README.md'], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { + cwd: repo, + stdio: 'ignore', + }); + + expect(listWorkingTreeDirtyPaths(repo)).toEqual([]); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('returns dirty source paths and omits GitNexus-managed writes', async () => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + fs.mkdirSync(path.join(repo, 'src'), { recursive: true }); + fs.writeFileSync(path.join(repo, 'src', 'foo.ts'), 'export const x = 1;'); + fs.mkdirSync(path.join(repo, '.gitnexus'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.gitnexus', 'meta.json'), '{}'); + fs.writeFileSync(path.join(repo, 'AGENTS.md'), 'x'); + + expect(listWorkingTreeDirtyPaths(repo)).toEqual(['src/foo.ts']); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('still reports nested lookalikes that are not GitNexus-managed', async () => { + const { isWorkingTreeDirty, listWorkingTreeDirtyPaths } = + await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.mkdirSync(path.join(repo, 'docs'), { recursive: true }); + fs.writeFileSync(path.join(repo, 'docs', 'AGENTS.md'), 'project notes'); + execFileSync(gitExecutable, ['add', '--', 'docs/AGENTS.md'], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { + cwd: repo, + stdio: 'ignore', + }); + fs.writeFileSync(path.join(repo, 'docs', 'AGENTS.md'), 'edited notes'); + + expect(isWorkingTreeDirty(repo)).toBe(true); + expect(listWorkingTreeDirtyPaths(repo)).toEqual(['docs/AGENTS.md']); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('returns null (not an empty list) outside a git repository', async () => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const dir = makeIsolatedTempDir('gn-nongit-paths-'); + try { + expect(listWorkingTreeDirtyPaths(dir)).toBeNull(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('preserves non-ASCII, newline, and arrow-shaped filenames exactly', async () => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + const names = ['src/ä.ts']; + if (process.platform !== 'win32') { + names.push('src/a -> b.ts', 'src/line\nbreak.ts', 'src/tab\tname.ts', 'src/back\\slash.ts'); + } + try { + for (const name of names) { + fs.mkdirSync(path.dirname(path.join(repo, name)), { recursive: true }); + fs.writeFileSync(path.join(repo, name), 'before'); + } + execFileSync(gitExecutable, ['add', '--', ...names], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { cwd: repo, stdio: 'ignore' }); + for (const name of names) fs.writeFileSync(path.join(repo, name), 'after'); + + expect(listWorkingTreeDirtyPaths(repo)?.sort()).toEqual([...names].sort()); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it.skipIf(process.platform === 'win32')( + 'returns both paths for a rename without parsing filename text', + async () => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + const before = 'src/before -> literal.ts'; + const after = 'src/after -> literal.ts'; + try { + fs.mkdirSync(path.join(repo, 'src'), { recursive: true }); + fs.writeFileSync(path.join(repo, before), 'content'); + execFileSync(gitExecutable, ['add', '--', before], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['mv', '--', before, after], { cwd: repo, stdio: 'ignore' }); + + expect(listWorkingTreeDirtyPaths(repo)?.sort()).toEqual([after, before].sort()); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }, + ); + + it.each(['--assume-unchanged', '--skip-worktree'])( + 'includes paths hidden by git update-index %s', + async (flag) => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'hidden.ts'), 'before'); + execFileSync(gitExecutable, ['add', '--', 'hidden.ts'], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', flag, '--', 'hidden.ts'], { + cwd: repo, + stdio: 'ignore', + }); + fs.writeFileSync(path.join(repo, 'hidden.ts'), 'after'); + + expect(listWorkingTreeDirtyPaths(repo)).toContain('hidden.ts'); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }, + ); + + it.each(['--assume-unchanged', '--skip-worktree'])( + 'omits GitNexus-managed paths hidden by git update-index %s', + async (flag) => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + fs.writeFileSync(path.join(repo, 'AGENTS.md'), 'before'); + execFileSync(gitExecutable, ['add', '--', 'README.md', 'AGENTS.md'], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', flag, '--', 'AGENTS.md'], { + cwd: repo, + stdio: 'ignore', + }); + fs.writeFileSync(path.join(repo, 'AGENTS.md'), 'after'); + + expect(listWorkingTreeDirtyPaths(repo)).toEqual([]); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }, + ); + + it.skipIf(process.platform === 'win32')( + 'preserves exact unusual names hidden by index bits, including both bits', + async () => { + const { listWorkingTreeDirtyPaths } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + const names = ['ä.ts', 'a -> b.ts', 'tab\tname.ts', 'line\nbreak.ts']; + try { + for (const name of names) fs.writeFileSync(path.join(repo, name), 'before'); + execFileSync(gitExecutable, ['add', '--', ...names], { cwd: repo, stdio: 'ignore' }); + execFileSync(gitExecutable, ['commit', '-q', '-m', 'init'], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', '--assume-unchanged', '--', names[0]], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', '--skip-worktree', '--', names[1]], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', '--assume-unchanged', '--', names[2]], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', '--skip-worktree', '--', names[2]], { + cwd: repo, + stdio: 'ignore', + }); + execFileSync(gitExecutable, ['update-index', '--skip-worktree', '--', names[3]], { + cwd: repo, + stdio: 'ignore', + }); + for (const name of names) fs.writeFileSync(path.join(repo, name), 'after'); + + expect(listWorkingTreeDirtyPaths(repo)?.sort()).toEqual([...names].sort()); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/gitnexus/test/unit/index-content-drift.test.ts b/gitnexus/test/unit/index-content-drift.test.ts new file mode 100644 index 000000000..04b26785b --- /dev/null +++ b/gitnexus/test/unit/index-content-drift.test.ts @@ -0,0 +1,404 @@ +/** + * Unit Tests: per-file index freshness (core/index-content-drift.ts) + * + * Issue #3077: `status` answered "is the index fresh?" with a repo-wide + * `git status --porcelain` boolean, so a modified or untracked file the index + * never reads pinned the verdict to "stale" — and because `analyze` cannot + * commit or delete that file, the advice it printed could never clear it. + * + * These tests use real temporary directories rather than mocks: the whole + * point of the helper is that it reuses analyze's own scan, so the ignore + * rules and the large-file cap are exactly what the assertions are about. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import path from 'path'; +import os from 'os'; +import fs from 'fs'; +import { execFileSync } from 'child_process'; + +import { detectIndexContentDrift } from '../../src/core/index-content-drift.js'; +import { walkRepositoryPaths } from '../../src/core/ingestion/filesystem-walker.js'; +import { computeFileHashes } from '../../src/storage/file-hash.js'; +import { listWorkingTreeDirtyPaths } from '../../src/storage/git.js'; +import { + GITNEXUS_MANAGED_PATH_EXCLUDES, + isGitNexusManagedPath, +} from '../../src/storage/gitnexus-managed-paths.js'; + +const gitExecutable = (() => { + if (process.platform !== 'win32') return 'git'; + try { + return ( + execFileSync('where.exe', ['git'], { encoding: 'utf8' }).split(/\r?\n/).find(Boolean) ?? 'git' + ); + } catch { + return 'git'; + } +})(); + +const isolatedTmpRoot = (() => { + const root = + process.platform === 'win32' + ? path.join(path.parse(os.tmpdir()).root, 'gitnexus-drift') + : path.join(os.tmpdir(), 'gitnexus-drift'); + fs.mkdirSync(root, { recursive: true }); + return root; +})(); + +const createdRepos: string[] = []; + +const makeRepo = (files: Record): string => { + const dir = fs.mkdtempSync(path.join(isolatedTmpRoot, 'repo-')); + createdRepos.push(dir); + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(dir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + } + return dir; +}; + +/** Reproduce what `analyze` records in `meta.fileHashes` for a repository. */ +const recordCoverage = async ( + repoPath: string, + walkOptions?: Parameters[2], +): Promise> => { + const scanned = await walkRepositoryPaths(repoPath, undefined, walkOptions); + const hashes = await computeFileHashes( + repoPath, + scanned.map((f) => f.path), + ); + return Object.fromEntries(hashes); +}; + +afterEach(() => { + while (createdRepos.length > 0) { + fs.rmSync(createdRepos.pop()!, { recursive: true, force: true }); + } +}); + +describe('detectIndexContentDrift', () => { + it('reports current when every covered file still matches disk', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const recorded = await recordCoverage(repo); + + const drift = await detectIndexContentDrift(repo, recorded); + + expect(drift).toEqual({ kind: 'current', coveredFileCount: Object.keys(recorded).length }); + }); + + it('stays current when a file the index does not cover is modified (#3077)', async () => { + // `.lock` is an ignored extension, so the indexer never reads this file. + // Under the old repo-wide dirty check its edit forced an unclearable + // "stale" verdict on an index that was byte-current with its own coverage. + const repo = makeRepo({ + 'a.js': 'export const a = 1;\n', + 'toolingdir/state.lock': 'before\n', + }); + const recorded = await recordCoverage(repo); + expect(Object.keys(recorded)).not.toContain('toolingdir/state.lock'); + + fs.writeFileSync(path.join(repo, 'toolingdir/state.lock'), 'after\n'); + + expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ kind: 'current' }); + }); + + it('stays current when an ignored directory changes', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const recorded = await recordCoverage(repo); + + fs.mkdirSync(path.join(repo, 'node_modules', 'left-pad'), { recursive: true }); + fs.writeFileSync( + path.join(repo, 'node_modules', 'left-pad', 'index.js'), + 'module.exports=1;\n', + ); + + expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ kind: 'current' }); + }); + + it('reports the covered file that changed', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n', 'b.js': 'export const b = 2;\n' }); + const recorded = await recordCoverage(repo); + + fs.writeFileSync(path.join(repo, 'b.js'), 'export const b = 3;\n'); + + const drift = await detectIndexContentDrift(repo, recorded); + expect(drift).toMatchObject({ kind: 'drifted', changed: ['b.js'], added: [], deleted: [] }); + }); + + it('reports a new coverable file as added rather than certifying the index', async () => { + // The index is missing a file `analyze` would pick up, so "up-to-date" + // would be a false all-clear even though every recorded hash matches. + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const recorded = await recordCoverage(repo); + + fs.writeFileSync(path.join(repo, 'new-source.js'), 'export const n = 1;\n'); + + expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ + kind: 'drifted', + added: ['new-source.js'], + changed: [], + }); + }); + + it('reports a removed covered file as deleted', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n', 'b.js': 'export const b = 2;\n' }); + const recorded = await recordCoverage(repo); + + fs.rmSync(path.join(repo, 'b.js')); + + expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ + kind: 'drifted', + deleted: ['b.js'], + changed: [], + }); + }); + + it('clears back to current once the coverage set is re-recorded', async () => { + // The loop the issue reports: `analyze` ran, reported success, and the + // verdict did not move. Re-recording coverage must settle the verdict. + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const stale = await recordCoverage(repo); + fs.writeFileSync(path.join(repo, 'notes.txt'), 'scratch\n'); + expect(await detectIndexContentDrift(repo, stale)).toMatchObject({ kind: 'drifted' }); + + const reanalyzed = await recordCoverage(repo); + + expect(await detectIndexContentDrift(repo, reanalyzed)).toMatchObject({ kind: 'current' }); + }); + + it("ignores GitNexus's own analyze output on both sides", async () => { + // analyze rewrites AGENTS.md/CLAUDE.md after recording hashes. Counting + // them made a freshly indexed repo report itself stale: absent from the + // first run's coverage, then rewritten on every run after that. + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const firstRun = await recordCoverage(repo); + fs.writeFileSync(path.join(repo, 'AGENTS.md'), 'stats block\n'); + fs.writeFileSync(path.join(repo, 'CLAUDE.md'), 'stats block\n'); + + expect(await detectIndexContentDrift(repo, firstRun)).toMatchObject({ kind: 'current' }); + + const secondRun = await recordCoverage(repo); + expect(Object.keys(secondRun)).toContain('AGENTS.md'); + fs.writeFileSync(path.join(repo, 'AGENTS.md'), 'refreshed stats block\n'); + + expect(await detectIndexContentDrift(repo, secondRun)).toMatchObject({ kind: 'current' }); + }); + + it('is unmeasurable, not current, when metadata carries no file hashes', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + + expect(await detectIndexContentDrift(repo, undefined)).toEqual({ + kind: 'unmeasurable', + reason: 'no-file-hashes', + }); + expect(await detectIndexContentDrift(repo, {})).toEqual({ + kind: 'unmeasurable', + reason: 'no-file-hashes', + }); + }); + + it('is unmeasurable when the repository scan throws', async () => { + const drift = await detectIndexContentDrift('/no-such-gitnexus-drift-repo', { + 'a.js': 'deadbeef', + }); + expect(drift).toEqual({ kind: 'unmeasurable', reason: 'scan-failed' }); + }); + + it('replays a recorded max-file-size so a later default cap cannot drop coverage', async () => { + // `.bin` is a hardcoded ignore; a large source file is what analyze would + // actually hash once `--max-file-size` / GITNEXUS_MAX_FILE_SIZE is raised. + const raisedCap = 1024 * 1024; + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + fs.writeFileSync(path.join(repo, 'payload.js'), Buffer.alloc(700 * 1024, 1)); + const recorded = await recordCoverage(repo, { maxFileSizeBytes: raisedCap, quiet: true }); + expect(Object.keys(recorded)).toContain('payload.js'); + + const withPolicy = await detectIndexContentDrift(repo, recorded, { + maxFileSizeBytes: raisedCap, + }); + expect(withPolicy).toMatchObject({ kind: 'current' }); + + // No persisted policy (indexes from before `indexCoverage`): the file is + // still on disk and hashed, so a later default cap must not call it deleted. + expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ kind: 'current' }); + }); + + it('treats a covered file that can no longer be read as changed, not current', async () => { + if (typeof process.getuid === 'function' && process.getuid() === 0) { + return; + } + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const recorded = await recordCoverage(repo); + const target = path.join(repo, 'a.js'); + fs.chmodSync(target, 0); + try { + expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ + kind: 'drifted', + changed: ['a.js'], + }); + } finally { + fs.chmodSync(target, 0o644); + } + }); + + it('re-hashes a path that was dirty at index time even after Git is clean', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + execFileSync(gitExecutable, ['init'], { cwd: repo }); + execFileSync(gitExecutable, ['add', '.'], { cwd: repo }); + execFileSync( + gitExecutable, + ['-c', 'user.email=t@t.test', '-c', 'user.name=t', 'commit', '-m', 'i'], + { cwd: repo }, + ); + fs.writeFileSync(path.join(repo, 'a.js'), 'export const a = 2;\n'); + const recorded = await recordCoverage(repo); + execFileSync(gitExecutable, ['checkout', '--', 'a.js'], { cwd: repo }); + + const skipped = await detectIndexContentDrift(repo, recorded, { + maxFileSizeBytes: 512 * 1024, + dirtyPaths: [], + }); + expect(skipped).toMatchObject({ kind: 'current' }); + + const restored = await detectIndexContentDrift(repo, recorded, { + maxFileSizeBytes: 512 * 1024, + dirtyPaths: ['a.js'], + }); + expect(restored).toMatchObject({ kind: 'drifted', changed: ['a.js'] }); + }); + + it.each(['--assume-unchanged', '--skip-worktree'])( + 'does not let git update-index %s hide covered-file drift', + async (flag) => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + execFileSync(gitExecutable, ['init', '-q'], { cwd: repo }); + execFileSync(gitExecutable, ['add', '--', 'a.js'], { cwd: repo }); + execFileSync( + gitExecutable, + ['-c', 'user.email=t@t.test', '-c', 'user.name=t', 'commit', '-q', '-m', 'init'], + { cwd: repo }, + ); + const recorded = await recordCoverage(repo); + execFileSync(gitExecutable, ['update-index', flag, '--', 'a.js'], { cwd: repo }); + fs.writeFileSync(path.join(repo, 'a.js'), 'export const a = 2;\n'); + const listed = listWorkingTreeDirtyPaths(repo); + expect(listed).not.toBeNull(); + expect(listed).toContain('a.js'); + + expect( + await detectIndexContentDrift(repo, recorded, { + maxFileSizeBytes: 512 * 1024, + dirtyPaths: [], + }), + ).toMatchObject({ kind: 'drifted', changed: ['a.js'] }); + }, + ); + + it('hashes the full intersection when the Git path query fails', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + execFileSync(gitExecutable, ['init', '-q'], { cwd: repo }); + execFileSync(gitExecutable, ['add', '--', 'a.js'], { cwd: repo }); + execFileSync( + gitExecutable, + ['-c', 'user.email=t@t.test', '-c', 'user.name=t', 'commit', '-q', '-m', 'init'], + { cwd: repo }, + ); + const recorded = await recordCoverage(repo); + fs.writeFileSync(path.join(repo, 'a.js'), 'export const a = 2;\n'); + + const savedPath = process.env.PATH; + try { + process.env.PATH = ''; + expect(listWorkingTreeDirtyPaths(repo)).toBeNull(); + expect( + await detectIndexContentDrift(repo, recorded, { + maxFileSizeBytes: 512 * 1024, + dirtyPaths: [], + }), + ).toMatchObject({ kind: 'drifted', changed: ['a.js'] }); + } finally { + process.env.PATH = savedPath; + } + }); + + it.each(process.platform === 'win32' ? ['ä.js'] : ['ä.js', 'a -> b.js', 'line\nbreak.js'])( + 'detects drift for porcelain-sensitive filename %j', + async (fileName) => { + const repo = makeRepo({ [fileName]: 'export const a = 1;\n' }); + execFileSync(gitExecutable, ['init', '-q'], { cwd: repo }); + execFileSync(gitExecutable, ['add', '--', fileName], { cwd: repo }); + execFileSync( + gitExecutable, + ['-c', 'user.email=t@t.test', '-c', 'user.name=t', 'commit', '-q', '-m', 'init'], + { cwd: repo }, + ); + const recorded = await recordCoverage(repo); + fs.writeFileSync(path.join(repo, fileName), 'export const a = 2;\n'); + + expect( + await detectIndexContentDrift(repo, recorded, { + maxFileSizeBytes: 512 * 1024, + dirtyPaths: [], + }), + ).toMatchObject({ kind: 'drifted', changed: [fileName] }); + }, + ); +}); + +describe('isGitNexusManagedPath', () => { + it('matches managed files and whole managed trees', () => { + expect(isGitNexusManagedPath('AGENTS.md')).toBe(true); + expect(isGitNexusManagedPath('CLAUDE.md')).toBe(true); + expect(isGitNexusManagedPath('.agents/skills/gitnexus-area-auth/SKILL.md')).toBe(true); + expect(isGitNexusManagedPath('.gitnexus/meta.json')).toBe(true); + }); + + it('does not match prefix collisions or nested lookalikes', () => { + // Same boundaries the `:(exclude)` pathspecs enforce for isWorkingTreeDirty. + expect(isGitNexusManagedPath('.agentsrc')).toBe(false); + expect(isGitNexusManagedPath('.claudefoo')).toBe(false); + expect(isGitNexusManagedPath('subdir/.agents/x')).toBe(false); + expect(isGitNexusManagedPath('docs/AGENTS.md')).toBe(false); + }); + + it('emits root-anchored recursive pathspecs for every managed path', () => { + expect(GITNEXUS_MANAGED_PATH_EXCLUDES).toContain(':(exclude,glob)./AGENTS.md'); + expect(GITNEXUS_MANAGED_PATH_EXCLUDES).toContain(':(exclude,glob)./.agents'); + expect(GITNEXUS_MANAGED_PATH_EXCLUDES).toContain(':(exclude,glob)./.agents/**'); + }); +}); + +describe('walkRepositoryPaths quiet option', () => { + const savedMaxFileSize = process.env.GITNEXUS_MAX_FILE_SIZE; + const savedProgressActive = process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE; + + afterEach(() => { + if (savedMaxFileSize === undefined) delete process.env.GITNEXUS_MAX_FILE_SIZE; + else process.env.GITNEXUS_MAX_FILE_SIZE = savedMaxFileSize; + if (savedProgressActive === undefined) delete process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE; + else process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE = savedProgressActive; + }); + + it('suppresses the large-file notice so read-only callers stay silent', async () => { + const repo = makeRepo({ 'big.js': `// ${'x'.repeat(4096)}\n` }); + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; // 1KB cap — big.js is skipped + process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE = '1'; // routes the notice to console.warn + + const warnings: unknown[][] = []; + const originalWarn = console.warn; + console.warn = (...args: unknown[]) => void warnings.push(args); + try { + const noisy = await walkRepositoryPaths(repo); + const noisyCount = warnings.length; + warnings.length = 0; + const quiet = await walkRepositoryPaths(repo, undefined, { quiet: true }); + + expect(noisyCount).toBeGreaterThan(0); + expect(warnings).toEqual([]); + expect(quiet).toEqual(noisy); + } finally { + console.warn = originalWarn; + } + }); +}); diff --git a/gitnexus/test/unit/list-status-branch.test.ts b/gitnexus/test/unit/list-status-branch.test.ts index f472397f8..2f0368614 100644 --- a/gitnexus/test/unit/list-status-branch.test.ts +++ b/gitnexus/test/unit/list-status-branch.test.ts @@ -66,6 +66,7 @@ vi.mock('../../src/storage/git.js', () => ({ getCurrentBranch: vi.fn().mockReturnValue('main'), getGitRoot: vi.fn((p: string) => p), isWorkingTreeDirty: vi.fn().mockReturnValue(false), + listWorkingTreeDirtyPaths: vi.fn().mockReturnValue([]), })); import { listCommand } from '../../src/cli/list.js'; diff --git a/gitnexus/test/unit/status-content-drift.test.ts b/gitnexus/test/unit/status-content-drift.test.ts new file mode 100644 index 000000000..a773f7b80 --- /dev/null +++ b/gitnexus/test/unit/status-content-drift.test.ts @@ -0,0 +1,279 @@ +/** + * Unit Tests: `status` freshness verdict from per-file drift (#3077) + * + * The reported defect was a verdict nobody could clear: any modified or + * untracked file in the working tree — including files the index never reads — + * made `status` print "stale (re-run gitnexus analyze)", and running `analyze` + * left it unchanged. These tests pin the new decision order: the per-file + * comparison decides when it can run, and the repo-wide dirty flag survives + * only as the fallback for metadata written before `fileHashes` existed. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +const { runnerIdentity } = vi.hoisted(() => ({ + runnerIdentity: { + schemaVersion: 4 as const, + runtime: { + executablePath: '/usr/bin/node', + version: 'v22.0.0', + platform: 'linux', + architecture: 'x64', + modulesAbi: '127', + libc: 'glibc:2.39', + }, + cliVersion: '1.6.10', + invokedArtifact: { path: '/opt/gitnexus/dist/cli/index.js', digest: 'sha256:entry' }, + build: { + kind: 'distribution' as const, + rootPath: '/opt/gitnexus/dist', + canonicalization: 'gitnexus-analyzer-build-v2' as const, + digest: 'sha256:build', + }, + dependencyRuntime: { + manifestPath: '/opt/gitnexus/package.json', + lockfilePath: '/opt/package-lock.json', + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4' as const, + packageCount: 42, + artifactCount: 12, + digest: 'sha256:dependencies', + }, + }, +})); + +vi.mock('../../src/storage/repo-manager.js', () => ({ + listRegisteredRepos: vi.fn(), + findRepo: vi.fn(), + getStoragePaths: vi.fn((repoPath: string) => ({ + storagePath: `${repoPath}/.gitnexus`, + lbugPath: `${repoPath}/.gitnexus/lbug`, + metaPath: `${repoPath}/.gitnexus/meta.json`, + })), + loadMeta: vi.fn(), + hasKuzuIndex: vi.fn().mockResolvedValue(false), +})); + +vi.mock('../../src/core/analyzer-identity.js', () => ({ + resolveAnalyzerRunnerIdentity: vi.fn(() => runnerIdentity), + analyzerRunnerIdentitiesEqual: vi.fn((indexed: unknown, current: unknown) => indexed === current), +})); + +vi.mock('../../src/storage/git.js', () => ({ + isGitRepo: vi.fn().mockReturnValue(true), + getCurrentCommit: vi.fn().mockReturnValue('headsha0'), + getCurrentBranch: vi.fn().mockReturnValue('main'), + getGitRoot: vi.fn((p: string) => p), + isWorkingTreeDirty: vi.fn().mockReturnValue(false), + listWorkingTreeDirtyPaths: vi.fn().mockReturnValue([]), +})); + +vi.mock('../../src/core/index-content-drift.js', () => ({ + detectIndexContentDrift: vi.fn(), +})); + +import { statusCommand } from '../../src/cli/status.js'; +import { setCliLanguage } from '../../src/cli/i18n/index.js'; +import { findRepo } from '../../src/storage/repo-manager.js'; +import { getCurrentCommit, isWorkingTreeDirty } from '../../src/storage/git.js'; +import { detectIndexContentDrift } from '../../src/core/index-content-drift.js'; + +let logSpy: ReturnType; +const output = () => logSpy.mock.calls.map((c) => c.join(' ')).join('\n'); + +const repoWithCoverage = { + repoPath: '/repo', + storagePath: '/repo/.gitnexus', + lbugPath: '/repo/.gitnexus/lbug', + metaPath: '/repo/.gitnexus/meta.json', + meta: { + repoPath: '/repo', + lastCommit: 'headsha0', + indexedAt: '2026-08-28T12:00:00.000Z', + branch: 'main', + runnerIdentity, + fileHashes: { 'a.js': 'sha-a' }, + scopeExtractionReceipt: 1 as const, + }, +}; + +beforeEach(() => { + vi.clearAllMocks(); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + (findRepo as any).mockResolvedValue(repoWithCoverage); + (getCurrentCommit as any).mockReturnValue('headsha0'); + (isWorkingTreeDirty as any).mockReturnValue(false); +}); + +afterEach(() => { + setCliLanguage(null); + logSpy.mockRestore(); +}); + +describe('status freshness from per-file drift (#3077)', () => { + it('is up-to-date when every covered file matches, despite a dirty working tree', async () => { + // The reported case: one modified file outside the index's coverage. The + // old repo-wide check called this stale and `analyze` could not clear it. + (isWorkingTreeDirty as any).mockReturnValue(true); + (detectIndexContentDrift as any).mockResolvedValue({ kind: 'current', coveredFileCount: 210 }); + + await statusCommand({ json: true }); + + expect(JSON.parse(output())).toMatchObject({ + status: 'up-to-date', + contentDrift: { status: 'current', coveredFiles: 210 }, + }); + }); + + it('reports covered-file drift as stale and names the files', async () => { + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'drifted', + changed: ['src/app.ts'], + added: [], + deleted: [], + }); + + await statusCommand(); + + const out = output(); + expect(out).not.toContain('up-to-date'); + expect(out).toContain('1 changed, 0 added, 0 deleted'); + expect(out).toContain('changed: src/app.ts'); + }); + + it('escapes control characters in drifted path names', async () => { + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'drifted', + changed: ['src/\u001b[31mevil.ts'], + added: [], + deleted: [], + }); + + await statusCommand(); + + const out = output(); + expect(out).toContain(JSON.stringify('src/\u001b[31mevil.ts')); + expect(out).not.toContain('\u001b[31m'); + }); + + it('localizes overflow category labels in zh-CN', async () => { + setCliLanguage('zh-CN'); + const changed = Array.from({ length: 12 }, (_, i) => `src/file-${i}.ts`); + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'drifted', + changed, + added: [], + deleted: [], + }); + + await statusCommand(); + + const out = output(); + expect(out).toContain('已修改: src/file-0.ts'); + expect(out).toContain('另有 2 个 已修改'); + expect(out).not.toMatch(/\bchanged\b/); + }); + + it('names a failed coverage scan in human output instead of falling back', async () => { + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'unmeasurable', + reason: 'scan-failed', + }); + + await statusCommand(); + + const out = output(); + expect(out).toContain('coverage scan failed'); + expect(out).toContain('stale'); + expect(out).not.toContain('fell back to the working-tree check'); + }); + + it('exposes drift counts and a capped sample in --json', async () => { + const changed = Array.from({ length: 25 }, (_, i) => `src/file-${i}.ts`); + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'drifted', + changed, + added: [], + deleted: [], + }); + + await statusCommand({ json: true }); + + const parsed = JSON.parse(output()); + expect(parsed.status).toBe('stale'); + expect(parsed.contentDrift.counts).toEqual({ changed: 25, added: 0, deleted: 0 }); + expect(parsed.contentDrift.changed).toHaveLength(10); + expect(parsed.contentDrift.truncated).toEqual({ + changed: true, + added: false, + deleted: false, + }); + }); + + it('falls back to the working-tree check when coverage cannot be compared', async () => { + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'unmeasurable', + reason: 'no-file-hashes', + }); + (isWorkingTreeDirty as any).mockReturnValue(true); + + await statusCommand({ json: true }); + + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + contentDrift: { status: 'unmeasurable', reason: 'no-file-hashes' }, + }); + }); + + it('is stale when coverage cannot be compared because the scan failed', async () => { + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'unmeasurable', + reason: 'scan-failed', + }); + + await statusCommand({ json: true }); + + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + contentDrift: { status: 'unmeasurable', reason: 'scan-failed' }, + }); + }); + + it('is up-to-date on a clean tree when hashes are missing (legacy metadata)', async () => { + (detectIndexContentDrift as any).mockResolvedValue({ + kind: 'unmeasurable', + reason: 'no-file-hashes', + }); + + await statusCommand({ json: true }); + + expect(JSON.parse(output())).toMatchObject({ + status: 'up-to-date', + contentDrift: { status: 'unmeasurable', reason: 'no-file-hashes' }, + }); + }); + + it('skips the scan when the index is already stale on metadata alone', async () => { + // A moved HEAD is decided without paying for a repository-wide hash pass. + (getCurrentCommit as any).mockReturnValue('othersha'); + + await statusCommand({ json: true }); + + expect(detectIndexContentDrift).not.toHaveBeenCalled(); + expect(JSON.parse(output())).toMatchObject({ + status: 'stale', + contentDrift: { status: 'not-checked' }, + }); + }); + + it('replays persisted indexCoverage into the drift check', async () => { + const coverage = { maxFileSizeBytes: 1024 * 1024, dirtyPaths: ['a.js'] }; + (findRepo as any).mockResolvedValue({ + ...repoWithCoverage, + meta: { ...repoWithCoverage.meta, indexCoverage: coverage }, + }); + (detectIndexContentDrift as any).mockResolvedValue({ kind: 'current', coveredFileCount: 1 }); + + await statusCommand({ json: true }); + + expect(detectIndexContentDrift).toHaveBeenCalledWith('/repo', { 'a.js': 'sha-a' }, coverage); + }); +}); From 5bad2d8b0b699a0cbb2ff08db370285f1ac5b79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=AE=80=E5=BE=8B=E7=BA=AF?= Date: Sun, 30 Aug 2026 05:07:43 +0800 Subject: [PATCH 22/28] fix(mcp): resolve omitted repo from cwd (#3085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(mcp): resolve omitted repo from cwd * test(mcp): cover cwd repository routing gaps * fix(mcp): harden cwd repository routing * docs(mcp): clarify cwd repository boundary * fix(mcp): preserve resolver compatibility * fix(mcp): align restricted repository routing --------- Co-authored-by: Gergő Magyar --- GUARDRAILS.md | 4 +- README.md | 4 +- gitnexus/README.md | 2 +- gitnexus/scripts/cross-platform-tests.ts | 3 + gitnexus/src/mcp/local/local-backend.ts | 121 ++++-- gitnexus/src/mcp/repository-policy.ts | 64 +++- gitnexus/src/mcp/resources.ts | 5 +- gitnexus/src/mcp/server.ts | 12 +- gitnexus/src/mcp/tools.ts | 49 ++- gitnexus/test/unit/calltool-dispatch.test.ts | 362 +++++++++++++++++- .../test/unit/mcp-repository-policy.test.ts | 24 ++ gitnexus/test/unit/resources.test.ts | 5 +- gitnexus/test/unit/server.test.ts | 43 ++- gitnexus/test/unit/tools.test.ts | 19 + 14 files changed, 639 insertions(+), 78 deletions(-) diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 72e9c1e59..f34cf79d5 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -73,8 +73,8 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Wrong repo in multi-repo setups - **Trigger:** Query/impact results belong to another project. -- **Do:** Call `list_repos`, then pass `repo` on subsequent tools. -- **Why:** Default target is ambiguous when multiple repos are registered. +- **Do:** Confirm an MCP default is configured or the GitNexus process was launched inside the intended registered path without crossing into an unindexed nested Git checkout. Otherwise call `list_repos`, then pass `repo` on subsequent tools; pass it for mutating tools when multiple repos are registered and no MCP default exists. +- **Why:** Read-only tools derive their default from MCP configuration or a process cwd that stays within one registered Git boundary. Outside those paths the target remains ambiguous, and mutating tools stay explicit unless configuration supplies the target. ### LadybugDB lock / "database busy" diff --git a/README.md b/README.md index 5e0536a48..173cd1d42 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ flowchart TB | `group_list` | List configured repository groups | | `group_sync` | Rebuild a group's Contract Registry and cross-repo links | -> Per-repo tools take an optional `repo` parameter (omit it when only one repo is indexed) and an optional `branch` for indexes pinned with `gitnexus analyze --branch`. Omitting `branch` queries the workspace index, which follows your checked-out working tree — switching branches and re-running `gitnexus analyze` updates it incrementally. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. +> Per-repo read-only tools take an optional `repo` parameter. Omit it when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout; otherwise pass it explicitly. Mutating tools require `repo` when multiple repos are indexed and no MCP default exists. Per-repo tools also take an optional `branch` for indexes pinned with `gitnexus analyze --branch`. Omitting `branch` queries the workspace index, which follows your checked-out working tree — switching branches and re-running `gitnexus analyze` updates it incrementally. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. ### Resources for instant context @@ -612,7 +612,7 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas GitNexus uses a **global registry** so one MCP server can serve multiple indexed repos. No per-project MCP config needed — set it up once and it works everywhere. -Each `gitnexus analyze` stores the index in `.gitnexus/` inside the repo (portable, gitignored) and registers a pointer in `~/.gitnexus/registry.json`. When an AI agent starts, the MCP server reads the registry and can serve any indexed repo. LadybugDB connections are opened lazily on first query and evicted after 5 minutes of inactivity (max 5 concurrent). If only one repo is indexed, the `repo` parameter is optional on all tools — agents don't need to change anything. +Each `gitnexus analyze` stores the index in `.gitnexus/` inside the repo (portable, gitignored) and registers a pointer in `~/.gitnexus/registry.json`. When an AI agent starts, the MCP server reads the registry and can serve any indexed repo. LadybugDB connections are opened lazily on first query and evicted after 5 minutes of inactivity (max 5 concurrent). Read-only tools can omit `repo` when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout. Outside those paths—and for mutating tools with multiple indexed repos and no MCP default—pass `repo` explicitly.
Architecture diagram diff --git a/gitnexus/README.md b/gitnexus/README.md index 93b2dd354..20ea65ef5 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -204,7 +204,7 @@ Your AI agent gets **17 tools** (15 per-repo + 2 group) automatically: | `group_list` | List configured repository groups | | `group_sync` | Rebuild a group's Contract Registry and cross-repo links | -> With one indexed repo, the `repo` param is optional. With multiple, specify which: `query({search_query: "auth", repo: "my-app"})`. Per-repo tools also take an optional `branch` for indexes pinned with `gitnexus analyze --branch`; omitting it queries the workspace index, which follows your checked-out working tree. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. +> Read-only tools can omit `repo` when one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout. Otherwise—and for mutating tools with multiple indexed repos and no MCP default—specify it explicitly: `query({search_query: "auth", repo: "my-app"})`. Per-repo tools also take an optional `branch` for indexes pinned with `gitnexus analyze --branch`; omitting it queries the workspace index, which follows your checked-out working tree. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. ## MCP Resources diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 3f96c5c40..44e45fbbb 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -114,6 +114,9 @@ const PLATFORM_LOGIC = [ // POSIX and Windows — the fail-closed path-claim semantics must hold on the // real windows-latest path implementation (#2419/#2420). 'test/unit/server-api-repo-resolution.test.ts', + // #3073: cwd-based repository selection canonicalizes real paths, compares + // platform separators/case, and rejects nested Git-boundary fallthrough. + 'test/unit/calltool-dispatch.test.ts', // The index write-lock (#2658) selects its backend by process.platform — the // OS socket lock (Windows named pipe / Linux abstract socket) vs the file // fallback — and its socket-backend describe block is gated to linux/win32. diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 544f64c12..3e2f5e8f7 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -38,6 +38,7 @@ import { parseDiffHunks, coalesceHunksByPath, hunksOverlapRange, + findGitRootByDotGit, getCanonicalRepoRoot, getGitRoot, type FileDiff, @@ -1716,21 +1717,61 @@ export class LocalBackend { * - If only 1 repo, use it * - If 0 or multiple without param, throw with helpful message * - * On a miss, re-reads the registry once in case a new repo was indexed - * while the MCP server was running. + * Re-reads the registry before an omitted implicit target or after an + * explicit miss, so long-running servers see newly indexed repositories. */ async resolveRepo(repoParam?: string, branch?: string): Promise { - let refreshedAfterAmbiguity = false; + return this.selectToolRepository(repoParam, branch); + } + + /** + * Internal resolver variant for CLI/MCP tool routing and discovery. + * - If repoParam is given, match by name or path + * - If only 1 repo, use it + * - If multiple repos exist and repoParam is omitted, callers may opt in to + * the registered repo containing process.cwd() + * - If 0 repos exist, or cwd cannot disambiguate multiple repos, throw + * + * Omitted-repo resolution re-reads the registry before accepting any + * implicit target, including a cached singleton. A caller that just obtained + * a fresh registry snapshot may disable that refresh explicitly. + */ + async selectToolRepository( + repoParam?: string, + branch?: string, + options: { allowCwdDefault?: boolean; refreshRegistry?: boolean } = {}, + ): Promise { + const allowCwdDefault = options.allowCwdDefault === true; + const mayRefresh = options.refreshRegistry !== false; + let refreshed = false; + + // A cached singleton is also an implicit choice: another process may have + // registered a second repo since init, which must not let a repo-less + // mutating call bypass the multi-repo ambiguity guard. + if (!repoParam && mayRefresh) { + await this.refreshRepos(); + refreshed = true; + } + let result: RepoHandle | null; try { - result = this.resolveRepoFromCache(repoParam); + result = this.resolveRepoFromCache(repoParam, allowCwdDefault); } catch (err) { if (!(err instanceof RegistryAmbiguousTargetError)) throw err; + if (!mayRefresh || refreshed) throw err; // Stale in-memory duplicate siblings can linger after unregister; refresh // once before re-throwing so a resolved registry can disambiguate (#1658). await this.refreshRepos(); - refreshedAfterAmbiguity = true; - result = this.resolveRepoFromCache(repoParam); + refreshed = true; + result = this.resolveRepoFromCache(repoParam, allowCwdDefault); + } + + // Explicit misses retain the existing one-refresh retry. Omitted targets + // already refreshed above unless a same-snapshot caller opted out. + if (!result && mayRefresh && !refreshed) { + await this.refreshRepos(); + refreshed = true; + result = this.resolveRepoFromCache(repoParam, allowCwdDefault); } if (result) { @@ -1746,16 +1787,6 @@ export class LocalBackend { return this.applyBranchScope(result, branch); } - // Miss — refresh registry and try once more (skip if already refreshed above) - if (!refreshedAfterAmbiguity) { - await this.refreshRepos(); - } - const retried = this.resolveRepoFromCache(repoParam); - if (retried) { - this.maybeWarnSiblingDrift(retried).catch(() => {}); - return this.applyBranchScope(retried, branch); - } - // Still no match — throw with helpful message if (this.repos.size === 0) { throw new Error('No indexed repositories. Run: gitnexus analyze'); @@ -1905,7 +1936,7 @@ export class LocalBackend { * Throws {@link RegistryAmbiguousTargetError} when `repoParam` matches * multiple handles by name and cwd cannot disambiguate (#1658). */ - private resolveRepoFromCache(repoParam?: string): RepoHandle | null { + private resolveRepoFromCache(repoParam?: string, allowCwdDefault = false): RepoHandle | null { if (this.repos.size === 0) return null; if (repoParam) { @@ -1938,6 +1969,9 @@ export class LocalBackend { ); if (nameMatches.length === 1) return nameMatches[0]; if (nameMatches.length > 1) { + // Explicit duplicate aliases retain the legacy fail-closed contract: + // only an exact cwd Git-root match may disambiguate them. Deepest path + // containment is reserved for an omitted read-only repo (#3073). const cwdPick = this.pickRepoHandleForCwd(nameMatches); if (cwdPick) return cwdPick; throw new RegistryAmbiguousTargetError( @@ -1969,26 +2003,50 @@ export class LocalBackend { return this.repos.values().next().value!; } + if (allowCwdDefault) { + const cwdPick = this.pickRepoHandleForCwd([...this.repos.values()], true); + if (cwdPick) return cwdPick; + } + return null; // Multiple repos, no param — ambiguous } /** - * Prefer the indexed repo whose path matches the git root of process.cwd(). + * Match process.cwd() against indexed repositories. * - * In MCP stdio server mode, `process.cwd()` is the server's launch directory, - * not the agent client's cwd. If the server was started from an unrelated - * directory, `getGitRoot` returns null and duplicate-name resolution throws - * {@link RegistryAmbiguousTargetError} — callers should pass an absolute path. + * Explicit duplicate aliases use exact Git-root matching only. Omitted + * read-only calls opt into deepest containing-path selection. In that mode a + * candidate must not sit above cwd's Git root, so an unindexed nested checkout + * cannot fall through to an indexed ancestor. The `.git` ancestor fallback + * preserves that boundary when the git executable is unavailable. */ - private pickRepoHandleForCwd(candidates: RepoHandle[]): RepoHandle | null { - const cwdRoot = getGitRoot(process.cwd()); - if (!cwdRoot) return null; - const canonicalCwd = canonicalizePath(cwdRoot); + private pickRepoHandleForCwd( + candidates: RepoHandle[], + allowContaining = false, + ): RepoHandle | null { + const cwd = process.cwd(); + const normalize = (value: string): string => { + const canonical = canonicalizePath(value); + return process.platform === 'win32' ? canonical.toLowerCase() : canonical; + }; + const isSameOrDescendant = (parent: string, child: string): boolean => + child === parent || + child.startsWith(parent.endsWith(path.sep) ? parent : `${parent}${path.sep}`); + const canonicalCwd = normalize(cwd); + const cwdRoot = getGitRoot(cwd) ?? findGitRootByDotGit(cwd); + const canonicalRoot = cwdRoot ? normalize(cwdRoot) : null; + if (allowContaining) { + const containing = candidates + .map((handle) => ({ handle, repoPath: normalize(handle.repoPath) })) + .filter(({ repoPath }) => isSameOrDescendant(repoPath, canonicalCwd)) + .filter(({ repoPath }) => !canonicalRoot || isSameOrDescendant(canonicalRoot, repoPath)) + .sort((a, b) => b.repoPath.length - a.repoPath.length); + if (containing.length > 0) return containing[0].handle; + } + + if (!canonicalRoot) return null; const cwdMatches = candidates.filter((handle) => { - const stored = canonicalizePath(handle.repoPath); - return process.platform === 'win32' - ? stored.toLowerCase() === canonicalCwd.toLowerCase() - : stored === canonicalCwd; + return normalize(handle.repoPath) === canonicalRoot; }); return cwdMatches.length === 1 ? cwdMatches[0] : null; } @@ -2415,9 +2473,10 @@ export class LocalBackend { // Resolve repo from optional param (re-reads registry on miss). An optional // `branch` param scopes the resolved handle to that branch's index (#2106). - const repo = await this.resolveRepo( + const repo = await this.selectToolRepository( p.repo as string | undefined, p.branch as string | undefined, + { allowCwdDefault: method !== 'rename' }, ); switch (method) { diff --git a/gitnexus/src/mcp/repository-policy.ts b/gitnexus/src/mcp/repository-policy.ts index 2204334f1..a309ec0cf 100644 --- a/gitnexus/src/mcp/repository-policy.ts +++ b/gitnexus/src/mcp/repository-policy.ts @@ -179,9 +179,44 @@ export class McpRepositoryPolicy { }); } - async requiresExplicitRepo(backend: LocalBackend): Promise { - if (this.defaultRepo) return false; - return (await this.listAllowedRepos(backend)).length > 1; + async toolSchemaRepoRequirements(backend: LocalBackend): Promise<{ + readOnlyRequiresRepo: boolean; + mutatingRequiresRepo: boolean; + }> { + if (this.defaultRepo) { + return { readOnlyRequiresRepo: false, mutatingRequiresRepo: false }; + } + + // Runtime selection is based on the configured allowlist, not on which + // entries happen to remain visible in a later registry refresh. Keep the + // advertised schema aligned with repoForArgs() when that listing shrinks. + if (this.restricted) { + const requiresRepo = this.allowed.length > 1; + return { + readOnlyRequiresRepo: requiresRepo, + mutatingRequiresRepo: requiresRepo, + }; + } + + // One fresh listing supplies both schema decisions. Besides keeping the + // advertised contract internally consistent, this avoids doing two full + // per-repo staleness fan-outs for every tools/list request. + const visibleRepos = await this.listAllowedRepos(backend); + if (visibleRepos.length <= 1) { + return { readOnlyRequiresRepo: false, mutatingRequiresRepo: false }; + } + try { + // listAllowedRepos() refreshed this backend immediately above. Resolve + // against that exact cache snapshot instead of racing another registry + // read; only read-only schemas may advertise the cwd-derived default. + await backend.selectToolRepository(undefined, undefined, { + allowCwdDefault: true, + refreshRegistry: false, + }); + return { readOnlyRequiresRepo: false, mutatingRequiresRepo: true }; + } catch { + return { readOnlyRequiresRepo: true, mutatingRequiresRepo: true }; + } } private async listReposPage( @@ -241,6 +276,22 @@ export class McpRepositoryPolicy { return backend.resolveRepo(selected?.path, branch); } + private async selectToolRepository( + backend: LocalBackend, + repo?: string, + branch?: string, + options?: Parameters[2], + ): Promise>> { + if (!this.configured) return backend.selectToolRepository(repo, branch, options); + if (!this.restricted) { + return backend.selectToolRepository(repo ?? this.defaultRepo?.path, branch, options); + } + const selected = this.repoForArgs(repo === undefined ? undefined : { repo }); + // Restricted policies never allow cwd to select outside the configured + // set; once policy supplies an explicit path, the public resolver is enough. + return backend.resolveRepo(selected?.path, branch); + } + assertResourceUri(uri: string): void { if (!this.restricted) return; let parsed: URL; @@ -307,6 +358,13 @@ export class McpRepositoryPolicy { if (property === 'resolveRepo') { return (repo?: string, branch?: string) => policy.resolveRepo(target, repo, branch); } + if (property === 'selectToolRepository') { + return ( + repo?: string, + branch?: string, + options?: Parameters[2], + ) => policy.selectToolRepository(target, repo, branch, options); + } if (property === 'getContext' && policy.restricted) { return (repoId?: string) => { if (!repoId || !policy.uniqueAllowedContextNames.has(repoId.toLowerCase())) return null; diff --git a/gitnexus/src/mcp/resources.ts b/gitnexus/src/mcp/resources.ts index 3411878bd..3e6c59daf 100644 --- a/gitnexus/src/mcp/resources.ts +++ b/gitnexus/src/mcp/resources.ts @@ -313,7 +313,10 @@ async function getReposResource(backend: LocalBackend): Promise { if (repos.length > 1) { lines.push(''); - lines.push('# Multiple repos indexed. Use repo parameter in tool calls:'); + lines.push( + '# Multiple repos indexed. Read-only tools may omit repo when an MCP default is configured or GitNexus process.cwd() is inside one listed path without crossing an unindexed nested Git checkout.', + ); + lines.push('# Otherwise—and for mutating tools without an MCP default—pass repo explicitly:'); lines.push(`# query({search_query: "auth", repo: "${repos[0].name}"})`); } diff --git a/gitnexus/src/mcp/server.ts b/gitnexus/src/mcp/server.ts index 8370e5c4a..a3df48b4c 100644 --- a/gitnexus/src/mcp/server.ts +++ b/gitnexus/src/mcp/server.ts @@ -185,11 +185,12 @@ export function createMCPServer( } }); - // With multiple visible repositories and no process-wide default, make the - // routing requirement machine-readable. Agents then supply `repo` before the - // call instead of discovering the ambiguity through a failed tool response. + // Make the effective routing contract machine-readable. Read-only tools may + // use a cwd-derived default; mutating rename remains explicit unless policy + // supplies a single/default repository. server.setRequestHandler(ListToolsRequestSchema, async () => { - const requireRepo = await repositoryPolicy.requiresExplicitRepo(backend); + const { readOnlyRequiresRepo, mutatingRequiresRepo } = + await repositoryPolicy.toolSchemaRepoRequirements(backend); return { tools: GITNEXUS_TOOLS.filter( (tool) => @@ -201,7 +202,8 @@ export function createMCPServer( name: tool.name, description: tool.description, inputSchema: - requireRepo && REPO_SCOPED_TOOLS.has(tool.name) + (tool.name === 'rename' ? mutatingRequiresRepo : readOnlyRequiresRepo) && + REPO_SCOPED_TOOLS.has(tool.name) ? { ...tool.inputSchema, required: [...new Set([...tool.inputSchema.required, 'repo'])], diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 4244a95bf..e5f7e8d43 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -82,6 +82,11 @@ export const PDG_QUERY_MAX_LIMIT = 200; // PDG direct backend callers also enforce it before running traversal. export const IMPACT_MAX_DEPTH = 32; +const CWD_AWARE_REPO_OMISSION = + 'Omit when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing an unindexed nested Git checkout; otherwise specify it explicitly.'; +const MUTATING_REPO_OMISSION = + 'Omit only when one repo is indexed or an MCP default is configured; otherwise mutating tools require an explicit repo.'; + export const GITNEXUS_TOOLS: ToolDefinition[] = [ { name: 'list_repos', @@ -94,8 +99,10 @@ PAGINATION: Results are paginated so a large registry is not truncated by MCP/LL WHEN TO USE: First step when multiple repos are indexed, or to discover available repos. AFTER THIS: READ gitnexus://repo/{name}/context for the repo you want to work with. -When multiple repos are indexed, you MUST specify the "repo" parameter -on other tools (query, context, impact, etc.) to target the correct one.`, +When multiple repos are indexed, repo-scoped read-only tools use the configured +MCP default or the registered path containing the GitNexus process cwd, unless +cwd has crossed into an unindexed nested Git checkout. If neither applies, +specify the "repo" parameter explicitly.`, annotations: READ_ONLY_TOOL_ANNOTATIONS, inputSchema: { type: 'object', @@ -184,8 +191,7 @@ SERVICE: optional monorepo path prefix (POSIX-style, case-sensitive segments). W }, repo: { type: 'string', - description: - 'Indexed repository name or path, or group mode "@" / "@/" (member path keys from group.yaml). Omit when only one indexed repo exists.', + description: `Indexed repository name or path, or group mode "@" / "@/" (member path keys from group.yaml). ${CWD_AWARE_REPO_OMISSION}`, }, service: { type: 'string', @@ -266,7 +272,7 @@ TIPS: }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: ['statement'], @@ -331,8 +337,7 @@ SERVICE: optional monorepo path prefix (case-sensitive path segments). When "rep }, repo: { type: 'string', - description: - 'Indexed repository name or path, or group mode "@" / "@/". Omit if only one repo is indexed.', + description: `Indexed repository name or path, or group mode "@" / "@/". ${CWD_AWARE_REPO_OMISSION}`, }, service: { type: 'string', @@ -378,7 +383,7 @@ Returns: changed symbols, affected processes, and a risk summary. }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: [], @@ -417,7 +422,7 @@ A graph too large to analyze at all returns \`{ error, truncated: true }\` with }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: [], @@ -454,7 +459,7 @@ Handles disambiguation via context()'s payload verbatim: an ambiguous symbol_nam }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${MUTATING_REPO_OMISSION}`, }, }, required: ['new_name'], @@ -593,8 +598,7 @@ SERVICE: optional monorepo path prefix (case-sensitive path segments). When "rep }, repo: { type: 'string', - description: - 'Indexed repository name or path, or group mode "@" / "@/". Omit if only one repo is indexed.', + description: `Indexed repository name or path, or group mode "@" / "@/". ${CWD_AWARE_REPO_OMISSION}`, }, service: { type: 'string', @@ -690,7 +694,7 @@ Findings are deliberately NOT part of impact()'s traversal or the web schema — }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: [], @@ -742,7 +746,7 @@ CONTRACT CAVEATS: }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: ['mode', 'target'], @@ -766,7 +770,7 @@ Returns: route nodes with their handlers, middleware wrapper chains (e.g., withA }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: [], @@ -784,7 +788,10 @@ Returns: tool nodes with their handler files and descriptions.`, type: 'object', properties: { tool: { type: 'string', description: 'Filter by tool name. Omit for all tools.' }, - repo: { type: 'string', description: 'Repository name or path.' }, + repo: { + type: 'string', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, + }, }, required: [], }, @@ -807,7 +814,7 @@ Returns routes that have both detected response keys AND consumers. Shows top-le }, repo: { type: 'string', - description: 'Repository name or path. Omit if only one repo is indexed.', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: [], @@ -833,7 +840,10 @@ Response shape is keyed on how many routes match, not on the data: exactly one m description: 'Optional HTTP verb — GET, POST, PUT, PATCH, DELETE, etc. — to narrow a multi-verb route or file lookup to a single method. Returns an error if no matched route uses that verb.', }, - repo: { type: 'string', description: 'Repository name or path.' }, + repo: { + type: 'string', + description: `Repository name or path. ${CWD_AWARE_REPO_OMISSION}`, + }, }, required: [], }, @@ -948,8 +958,7 @@ DESTINATION TRACE (cross-repo): for an "@groupName" trace, OMIT to/to_uid/to_fil }, repo: { type: 'string', - description: - 'Repository name or path, or "@groupName" / "@groupName/memberPath" for a cross-repo trace over a group. Omit if only one repo is indexed.', + description: `Repository name or path, or "@groupName" / "@groupName/memberPath" for a cross-repo trace over a group. ${CWD_AWARE_REPO_OMISSION}`, }, }, required: [], diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index f5693f090..a98b25730 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -420,7 +420,7 @@ describe('LocalBackend.callTool', () => { ['impact', { name: 'validate', symbol: 'login', direction: 'upstream' }], ['context', { name: 'validate', file_path: 'src/auth.ts', file: 'src/login.ts' }], ])('rejects conflicting %s aliases before repository resolution', async (method, params) => { - const resolveSpy = vi.spyOn(backend, 'resolveRepo'); + const resolveSpy = vi.spyOn(backend, 'selectToolRepository'); const result = await backend.callTool(method, params); @@ -434,7 +434,7 @@ describe('LocalBackend.callTool', () => { ['context', { name: 'validate', file: ' ' }], ['context', { name: 'validate', file: null }], ])('rejects invalid %s aliases before repository resolution', async (method, params) => { - const resolveSpy = vi.spyOn(backend, 'resolveRepo'); + const resolveSpy = vi.spyOn(backend, 'selectToolRepository'); const result = await backend.callTool(method, params); @@ -443,7 +443,7 @@ describe('LocalBackend.callTool', () => { }); it('rejects a missing impact target before repository resolution', async () => { - const resolveSpy = vi.spyOn(backend, 'resolveRepo'); + const resolveSpy = vi.spyOn(backend, 'selectToolRepository'); const result = await backend.callTool('impact', { direction: 'upstream' }); @@ -3381,12 +3381,351 @@ describe('LocalBackend.resolveRepo', () => { ); }); - it('throws for ambiguous repos without param', async () => { - setupMultipleRepos(); - await backend.init(); - await expect(backend.callTool('query', { query: 'test' })).rejects.toThrow( - 'Multiple repositories indexed', - ); + it('throws for ambiguous repos when cwd is outside every indexed path', async () => { + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue('/tmp/test-project-sibling'); + + try { + setupMultipleRepos(); + await backend.init(); + await expect(backend.callTool('query', { query: 'test' })).rejects.toThrow( + 'Multiple repositories indexed', + ); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('defaults to the deepest indexed repo containing cwd (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-outer-')); + const nestedDir = path.join(outerDir, 'packages', 'nested'); + const cwdDir = path.join(nestedDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir); + (listRegisteredRepos as any).mockResolvedValue([ + { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }, + { + ...MOCK_REPO_ENTRY, + name: 'nested', + path: nestedDir, + storagePath: path.join(nestedDir, '.gitnexus'), + }, + ]); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + const resolved = await backend.selectToolRepository(undefined, undefined, { + allowCwdDefault: true, + }); + expect(resolved.repoPath).toBe(nestedDir); + const explicit = await backend.resolveRepo('outer'); + expect(explicit.repoPath).toBe(outerDir); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('refreshes before accepting a cached cwd ancestor (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-stale-outer-')); + const otherDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-stale-other-')); + const nestedDir = path.join(outerDir, 'vendor', 'nested'); + const cwdDir = path.join(nestedDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir, otherDir); + + const outerEntry = { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }; + const nestedEntry = { + ...MOCK_REPO_ENTRY, + name: 'nested', + path: nestedDir, + storagePath: path.join(nestedDir, '.gitnexus'), + }; + const otherEntry = { + ...MOCK_REPO_ENTRY, + name: 'other', + path: otherDir, + storagePath: path.join(otherDir, '.gitnexus'), + }; + (listRegisteredRepos as any) + .mockResolvedValueOnce([outerEntry, otherEntry]) + .mockResolvedValue([outerEntry, nestedEntry, otherEntry]); + (getGitRoot as any).mockImplementation((value: string) => { + const resolved = path.resolve(value); + if (resolved === nestedDir || resolved.startsWith(`${nestedDir}${path.sep}`)) { + return nestedDir; + } + if (resolved === outerDir || resolved.startsWith(`${outerDir}${path.sep}`)) { + return outerDir; + } + if (resolved === otherDir || resolved.startsWith(`${otherDir}${path.sep}`)) { + return otherDir; + } + return null; + }); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + const resolved = await backend.selectToolRepository(undefined, undefined, { + allowCwdDefault: true, + }); + expect(resolved.repoPath).toBe(nestedDir); + expect(listRegisteredRepos).toHaveBeenCalledTimes(2); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('refreshes a cached singleton before repo-less read dispatch (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-singleton-outer-')); + const nestedDir = path.join(outerDir, 'packages', 'nested'); + const cwdDir = path.join(nestedDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir); + + const outerEntry = { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }; + const nestedEntry = { + ...MOCK_REPO_ENTRY, + name: 'nested', + path: nestedDir, + storagePath: path.join(nestedDir, '.gitnexus'), + }; + (listRegisteredRepos as any) + .mockResolvedValueOnce([outerEntry]) + .mockResolvedValue([outerEntry, nestedEntry]); + (getGitRoot as any).mockReturnValue(outerDir); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + (executeParameterized as any).mockResolvedValue([]); + + await backend.callTool('cypher', { statement: 'MATCH (n) RETURN n LIMIT 1' }); + + expect((executeParameterized as any).mock.calls.at(-1)?.[0]).toBe( + path.join(nestedDir, '.gitnexus', 'lbug'), + ); + expect(listRegisteredRepos).toHaveBeenCalledTimes(2); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('refreshes a cached singleton before enforcing repo-less rename safety (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-rename-outer-')); + const otherDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-rename-other-')); + const cwdDir = path.join(outerDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir, otherDir); + + const outerEntry = { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }; + const otherEntry = { + ...MOCK_REPO_ENTRY, + name: 'other', + path: otherDir, + storagePath: path.join(otherDir, '.gitnexus'), + }; + (listRegisteredRepos as any) + .mockResolvedValueOnce([outerEntry]) + .mockResolvedValue([outerEntry, otherEntry]); + (getGitRoot as any).mockReturnValue(outerDir); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + await expect( + backend.callTool('rename', { + symbol_name: 'oldName', + new_name: 'newName', + dry_run: false, + }), + ).rejects.toThrow('Multiple repositories indexed'); + expect(listRegisteredRepos).toHaveBeenCalledTimes(2); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('keeps explicit duplicate aliases on exact git-root disambiguation (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-alias-outer-')); + const nestedDir = path.join(outerDir, 'packages', 'nested'); + const cwdDir = path.join(nestedDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir); + (listRegisteredRepos as any).mockResolvedValue([ + { + ...MOCK_REPO_ENTRY, + name: 'shared', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }, + { + ...MOCK_REPO_ENTRY, + name: 'shared', + path: nestedDir, + storagePath: path.join(nestedDir, '.gitnexus'), + }, + ]); + (getGitRoot as any).mockReturnValue(outerDir); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + const resolved = await backend.resolveRepo('shared'); + expect(resolved.repoPath).toBe(outerDir); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('does not cross a nested git boundary when git root shelling fails (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-rootless-outer-')); + const otherDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-rootless-other-')); + const nestedDir = path.join(outerDir, 'vendor', 'nested'); + const cwdDir = path.join(nestedDir, 'src'); + mkdirSync(path.join(nestedDir, '.git'), { recursive: true }); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir, otherDir); + (listRegisteredRepos as any).mockResolvedValue([ + { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }, + { + ...MOCK_REPO_ENTRY, + name: 'other', + path: otherDir, + storagePath: path.join(otherDir, '.gitnexus'), + }, + ]); + (getGitRoot as any).mockReturnValue(null); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + await expect(backend.callTool('query', { query: 'test' })).rejects.toThrow( + 'Multiple repositories indexed', + ); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('keeps cwd routing opt-in for direct backend helpers (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-direct-outer-')); + const otherDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-direct-other-')); + const cwdDir = path.join(outerDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir, otherDir); + (listRegisteredRepos as any).mockResolvedValue([ + { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }, + { + ...MOCK_REPO_ENTRY, + name: 'other', + path: otherDir, + storagePath: path.join(otherDir, '.gitnexus'), + }, + ]); + (getGitRoot as any).mockReturnValue(outerDir); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + await expect(backend.queryProcesses()).rejects.toThrow('Multiple repositories indexed'); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('does not default across an unindexed nested git boundary (#3073)', async () => { + const outerDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-git-outer-')); + const otherDir = mkdtempSync(path.join(os.tmpdir(), 'gnx-cwd-git-other-')); + const nestedDir = path.join(outerDir, 'vendor', 'nested'); + const cwdDir = path.join(nestedDir, 'src'); + mkdirSync(cwdDir, { recursive: true }); + duplicateFixtureDirs.push(outerDir, otherDir); + (listRegisteredRepos as any).mockResolvedValue([ + { + ...MOCK_REPO_ENTRY, + name: 'outer', + path: outerDir, + storagePath: path.join(outerDir, '.gitnexus'), + }, + { + ...MOCK_REPO_ENTRY, + name: 'other', + path: otherDir, + storagePath: path.join(otherDir, '.gitnexus'), + }, + ]); + (getGitRoot as any).mockImplementation((value: string) => { + const resolved = path.resolve(value); + if (resolved === nestedDir || resolved.startsWith(`${nestedDir}${path.sep}`)) { + return nestedDir; + } + if (resolved === outerDir || resolved.startsWith(`${outerDir}${path.sep}`)) { + return outerDir; + } + if (resolved === otherDir || resolved.startsWith(`${otherDir}${path.sep}`)) { + return otherDir; + } + return null; + }); + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(cwdDir); + + try { + await backend.init(); + await expect( + backend.selectToolRepository(undefined, undefined, { allowCwdDefault: true }), + ).rejects.toThrow('Multiple repositories indexed'); + } finally { + cwdSpy.mockRestore(); + } + }); + + it('keeps mutating rename explicit with multiple repos (#3073)', async () => { + const cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue('/tmp/test-project/src'); + + try { + setupMultipleRepos(); + await backend.init(); + await expect( + backend.callTool('rename', { + symbol_name: 'oldName', + new_name: 'newName', + dry_run: true, + }), + ).rejects.toThrow('Multiple repositories indexed'); + } finally { + cwdSpy.mockRestore(); + } }); it('resolves repo by name parameter', async () => { @@ -4682,7 +5021,7 @@ describe('LocalBackend tool-staleness cache keying (#2655 review)', () => { lbugPath: `/r/.gitnexus/${path.join('branches', 'x', 'lbug')}`, lastCommit: 'BRANCHSHA', }; - vi.spyOn(backend, 'resolveRepo') + vi.spyOn(backend, 'selectToolRepository') .mockResolvedValueOnce(flat as any) .mockResolvedValueOnce(branch as any); // The tool itself returns a plain (staleness-carryable) object. @@ -4736,7 +5075,8 @@ describe('LocalBackend tool-staleness signal (#2655 review)', () => { lastCommit: 'HEADSHA', }; - const stubResolve = () => vi.spyOn(backend, 'resolveRepo').mockResolvedValue(handle as any); + const stubResolve = () => + vi.spyOn(backend, 'selectToolRepository').mockResolvedValue(handle as any); const stubStale = async () => { const { checkStalenessAsync } = await import('../../src/core/git-staleness.js'); diff --git a/gitnexus/test/unit/mcp-repository-policy.test.ts b/gitnexus/test/unit/mcp-repository-policy.test.ts index d5809d9bc..11dc4cd43 100644 --- a/gitnexus/test/unit/mcp-repository-policy.test.ts +++ b/gitnexus/test/unit/mcp-repository-policy.test.ts @@ -46,6 +46,11 @@ function createBackend(repos = REPOS) { repoPath: repo ?? repos[0]?.path, lastCommit: 'a'.repeat(40), })), + selectToolRepository: vi.fn().mockImplementation(async (repo?: string) => ({ + name: repos.find((entry) => entry.path === repo)?.name ?? repo ?? repos[0]?.name, + repoPath: repo ?? repos[0]?.path, + lastCommit: 'a'.repeat(40), + })), getContext: vi.fn().mockReturnValue(null), queryClusters: vi.fn().mockResolvedValue({ clusters: [] }), queryProcesses: vi.fn().mockResolvedValue({ processes: [] }), @@ -138,6 +143,24 @@ describe('MCP repository policy', () => { expect(backend.callTool).not.toHaveBeenCalled(); }); + it('keeps restricted schemas explicit when a multi-repo allowlist listing shrinks', async () => { + const backend = createBackend(); + const policy = await createMcpRepositoryPolicy(backend, { + GITNEXUS_MCP_ALLOWED_REPOS: 'Alpha,Beta', + }); + const alpha = REPOS[0]; + if (!alpha) throw new Error('Alpha fixture is required'); + vi.mocked(backend.listRepos).mockResolvedValue([{ ...alpha }]); + + await expect(policy.toolSchemaRepoRequirements(backend)).resolves.toEqual({ + readOnlyRequiresRepo: true, + mutatingRequiresRepo: true, + }); + await expect( + policy.scopeBackend(backend).callTool('query', { search_query: 'auth' }), + ).rejects.toThrow(/explicit repo.*multiple repositories are allowed/i); + }); + it('fails startup when the default is outside the allowlist after canonical resolution', async () => { const backend = createBackend(); await expect( @@ -199,6 +222,7 @@ describe('MCP repository policy', () => { const scoped = policy.scopeBackend(backend); await expect(scoped.resolveRepo('Beta')).rejects.toThrow(/not available/i); + await expect(scoped.selectToolRepository('Beta')).rejects.toThrow(/not available/i); await expect(scoped.readGroupStatusResource('portfolio')).rejects.toThrow( /group.*unavailable/i, ); diff --git a/gitnexus/test/unit/resources.test.ts b/gitnexus/test/unit/resources.test.ts index ff3894be1..bae0e65f6 100644 --- a/gitnexus/test/unit/resources.test.ts +++ b/gitnexus/test/unit/resources.test.ts @@ -396,7 +396,10 @@ describe('readResource', () => { }); const result = await readResource('gitnexus://repos', backend); expect(result).toContain('Multiple repos indexed'); - expect(result).toContain('repo parameter'); + expect(result).toContain('process.cwd()'); + expect(result).toContain('unindexed nested Git checkout'); + expect(result).toContain('mutating tools without an MCP default'); + expect(result).toContain('pass repo explicitly'); // The example must use a registered tool name, not the unregistered // `gitnexus_search` / `gitnexus_*` prefix (#2059). // #2175: advertise the renamed param, not the legacy "query" key. diff --git a/gitnexus/test/unit/server.test.ts b/gitnexus/test/unit/server.test.ts index 68d124ee5..8e10bccae 100644 --- a/gitnexus/test/unit/server.test.ts +++ b/gitnexus/test/unit/server.test.ts @@ -34,6 +34,9 @@ function createMockBackend(overrides: Record = {}): any { resolveRepo: vi .fn() .mockResolvedValue({ name: 'test', repoPath: '/tmp/test', lastCommit: 'abc' }), + selectToolRepository: vi + .fn() + .mockResolvedValue({ name: 'test', repoPath: '/tmp/test', lastCommit: 'abc' }), getContext: vi.fn().mockReturnValue(null), queryClusters: vi.fn().mockResolvedValue({ clusters: [] }), queryProcesses: vi.fn().mockResolvedValue({ processes: [] }), @@ -105,12 +108,13 @@ describe('createMCPServer', () => { await server.close(); } }); - it('requires repo in repo-scoped tool schemas when multiple repos are visible', async () => { + it('requires repo in repo-scoped tool schemas when cwd cannot resolve multiple repos', async () => { const backend = createMockBackend({ listRepos: vi.fn().mockResolvedValue([ { name: 'alpha', path: '/tmp/alpha' }, { name: 'beta', path: '/tmp/beta' }, ]), + selectToolRepository: vi.fn().mockRejectedValue(new Error('Multiple repositories indexed')), }); const server = createMCPServer(backend); const client = new Client({ name: 'multi-repo-client', version: '0.0.0' }); @@ -133,6 +137,43 @@ describe('createMCPServer', () => { } }); + it('keeps repo optional when cwd resolves one of multiple visible repos', async () => { + const backend = createMockBackend({ + listRepos: vi.fn().mockResolvedValue([ + { name: 'alpha', path: '/tmp/alpha' }, + { name: 'beta', path: '/tmp/beta' }, + ]), + selectToolRepository: vi + .fn() + .mockResolvedValue({ name: 'alpha', repoPath: '/tmp/alpha', lastCommit: 'abc' }), + }); + const server = createMCPServer(backend); + const client = new Client({ name: 'cwd-repo-client', version: '0.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + + try { + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + const tools = await client.listTools(); + const context = tools.tools.find((tool) => tool.name === 'context'); + const rename = tools.tools.find((tool) => tool.name === 'rename'); + + expect(context?.inputSchema.required).not.toContain('repo'); + expect(rename?.inputSchema.required).toContain('repo'); + const response = await client.callTool({ name: 'context', arguments: { name: 'Example' } }); + expect(response.isError).not.toBe(true); + expect(backend.callTool).toHaveBeenCalledWith('context', { name: 'Example' }); + expect(backend.listRepos).toHaveBeenCalledTimes(1); + expect(backend.selectToolRepository).toHaveBeenCalledTimes(1); + expect(backend.selectToolRepository).toHaveBeenCalledWith(undefined, undefined, { + allowCwdDefault: true, + refreshRegistry: false, + }); + } finally { + await client.close(); + await server.close(); + } + }); + it('keeps repo optional when a default repo is configured', async () => { const backend = createMockBackend({ listRepos: vi.fn().mockResolvedValue([ diff --git a/gitnexus/test/unit/tools.test.ts b/gitnexus/test/unit/tools.test.ts index 700407f78..bfa409405 100644 --- a/gitnexus/test/unit/tools.test.ts +++ b/gitnexus/test/unit/tools.test.ts @@ -283,6 +283,25 @@ describe('GITNEXUS_TOOLS', () => { } }); + it('repo descriptions explain the cwd default and mutating exception (#3073)', () => { + expect(GITNEXUS_TOOLS.find((tool) => tool.name === 'list_repos')?.description).toMatch( + /process cwd/i, + ); + expect(GITNEXUS_TOOLS.find((tool) => tool.name === 'list_repos')?.description).toMatch( + /unindexed nested Git checkout/i, + ); + for (const tool of GITNEXUS_TOOLS) { + if (tool.name === 'list_repos' || GROUP_TOOLS.has(tool.name)) continue; + const description = tool.inputSchema.properties.repo.description; + if (tool.name === 'rename') { + expect(description).toMatch(/mutating tools require an explicit repo/i); + } else { + expect(description).toMatch(/process cwd/i); + expect(description).toMatch(/unindexed nested Git checkout/i); + } + } + }); + it('per-repo tools have an optional branch scope param (#2106); group/list tools do not', () => { for (const tool of GITNEXUS_TOOLS) { if (tool.name === 'list_repos' || GROUP_TOOLS.has(tool.name)) { From 7e993ab8972386294fb96bf14a8665d0b5325397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 29 Aug 2026 22:48:46 +0100 Subject: [PATCH 23/28] fix(group): fail ambiguous sync names and honor analyze --name (#3094) * fix(group): fail sync when a member name is ambiguous Silent first-match bound the wrong clone when --allow-duplicate-name registered two paths under one alias. Refs #3028. Co-authored-by: Cursor * fix(analyze): apply --name on the already-up-to-date path A rename should not require --force when the index is already current. Register before the same-commit branch restamp. Refs #3028. Co-authored-by: Cursor * fix(group): hint member path when impact --repo is an alias $localRepo stays the yaml key; joining on the registry alias is a non-join. List matching keys so operators can retry. Refs #3028. Co-authored-by: Cursor * fix(group): keep injected sync and alias hints consistent Workspace-deps path maps reuse the resolved handle so duplicate names cannot throw after an injected resolver. Alias hints match case-insensitively. Co-authored-by: Cursor --------- Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- gitnexus/src/cli/analyze.ts | 3 + gitnexus/src/cli/group.ts | 8 +- gitnexus/src/core/group/config-parser.ts | 11 +- gitnexus/src/core/group/cross-impact.ts | 11 + gitnexus/src/core/group/service.ts | 6 +- gitnexus/src/core/group/sync.ts | 43 ++- gitnexus/src/core/run-analyze.ts | 36 +++ gitnexus/src/storage/repo-manager.ts | 28 +- .../test/integration/group/group-cli.test.ts | 45 +++ .../test/unit/group-service-not-found.test.ts | 5 +- .../test/unit/group/config-parser.test.ts | 19 ++ gitnexus/test/unit/group/cross-impact.test.ts | 98 +++++++ .../group/resolve-bridge-neighbors.test.ts | 56 ++++ .../group/service-group-sync-payload.test.ts | 1 + .../group/sync-partial-extraction.test.ts | 12 +- .../unit/group/sync-registry-identity.test.ts | 273 ++++++++++++++++++ .../unit/group/sync-unreadable-repos.test.ts | 12 +- .../group/sync-windowed-resolution.test.ts | 12 +- .../repo-manager-registry-strict-read.test.ts | 16 +- gitnexus/test/unit/repo-manager.test.ts | 8 + .../test/unit/run-analyze-fts-repair.test.ts | 46 +++ gitnexus/test/unit/run-analyze.test.ts | 193 +++++++++++++ 22 files changed, 918 insertions(+), 24 deletions(-) create mode 100644 gitnexus/test/unit/group/sync-registry-identity.test.ts diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 54bf12d11..2719b8f40 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -1436,6 +1436,9 @@ const analyzeCommandImpl = async ( console.error = origError; bar.stop(); console.log(' Already up to date\n'); + if (runOptions.registryName) { + console.log(` Registry name: ${result.repoName}\n`); + } if (baseRefRefreshed.length > 0) { console.log( ` Updated base_ref to "${resolvedDefaultBranch}" in ${baseRefRefreshed.join(', ')}\n`, diff --git a/gitnexus/src/cli/group.ts b/gitnexus/src/cli/group.ts index 7bb7bc180..abc13fa2b 100644 --- a/gitnexus/src/cli/group.ts +++ b/gitnexus/src/cli/group.ts @@ -219,8 +219,9 @@ export function registerGroupCommands(program: Command): void { .action(async (name: string, opts: Record) => { const { getGroupDir, getDefaultGitnexusDir } = await import('../core/group/storage.js'); const { loadGroupConfig } = await import('../core/group/config-parser.js'); - const { syncGroup } = await import('../core/group/sync.js'); + const { syncGroup, formatGroupSyncAmbiguousError } = await import('../core/group/sync.js'); const { GroupSyncLockError } = await import('../core/group/group-lock.js'); + const { RegistryAmbiguousTargetError } = await import('../storage/repo-manager.js'); const groupDir = getGroupDir(getDefaultGitnexusDir(), name); const config = await loadGroupConfig(groupDir); @@ -235,6 +236,11 @@ export function registerGroupCommands(program: Command): void { exactOnly: Boolean(opts.exactOnly), }); } catch (err) { + if (err instanceof RegistryAmbiguousTargetError) { + logger.error(`⚠️ Did not sync group "${name}": ${formatGroupSyncAmbiguousError(err)}`); + process.exitCode = 1; + return; + } // A sync that could not take the group's lock did NOT run and wrote // nothing (R9 fails closed). That is an operator-actionable outcome, not // a crash, so report it as a failed command rather than letting it diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index b831c6706..373e5e5c4 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -60,7 +60,16 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { throw new Error('repos is required in group.yaml (must be a mapping)'); } - const repos = raw.repos as Record; + const reposRaw = raw.repos as Record; + const repos: Record = {}; + for (const [memberPath, registryName] of Object.entries(reposRaw)) { + if (typeof registryName !== 'string' || registryName.trim() === '') { + throw new Error( + `repos["${memberPath}"] must be a non-empty registry name string, not ${typeof registryName}`, + ); + } + repos[memberPath] = registryName.trim(); + } const repoPaths = new Set(Object.keys(repos)); const rawLinks = (raw.links as unknown[]) || []; diff --git a/gitnexus/src/core/group/cross-impact.ts b/gitnexus/src/core/group/cross-impact.ts index 485b1ee8c..fd2ae0779 100644 --- a/gitnexus/src/core/group/cross-impact.ts +++ b/gitnexus/src/core/group/cross-impact.ts @@ -244,6 +244,17 @@ async function resolveGroupRepo( ): Promise { const registryName = config.repos[repoPath]; if (!registryName) { + const matchingMemberPaths = Object.entries(config.repos) + .filter(([, alias]) => alias.toLowerCase() === repoPath.toLowerCase()) + .map(([memberPath]) => memberPath); + if (matchingMemberPaths.length > 0) { + return { + error: + `Unknown repo path "${repoPath}" in this group. ` + + `That value is a registry alias for member path(s): ${matchingMemberPaths.join(', ')}. ` + + `Pass the group.yaml key to --repo, not the alias.`, + }; + } return { error: `Unknown repo path "${repoPath}" in this group.` }; } try { diff --git a/gitnexus/src/core/group/service.ts b/gitnexus/src/core/group/service.ts index af1f1385a..d0aa4882c 100644 --- a/gitnexus/src/core/group/service.ts +++ b/gitnexus/src/core/group/service.ts @@ -479,8 +479,9 @@ export class GroupService { // group tools never need it — so deferring it here keeps that closure off // MCP server startup entirely and off every non-sync group call. The CLI // already does exactly this at `cli/group.ts`'s sync command. - const { syncGroup } = await import('./sync.js'); + const { syncGroup, formatGroupSyncAmbiguousError } = await import('./sync.js'); const { GroupSyncLockError } = await import('./group-lock.js'); + const { RegistryAmbiguousTargetError } = await import('../../storage/repo-manager.js'); let result: Awaited>; try { result = await syncGroup(config, { @@ -492,6 +493,9 @@ export class GroupService { // expects. `SyncOptions.verbose` stays for the CLI, which can see them. }); } catch (err) { + if (err instanceof RegistryAmbiguousTargetError) { + return { error: formatGroupSyncAmbiguousError(err) }; + } // Fails closed (R9): this sync could not be protected against a concurrent // one, so it did not run and wrote nothing. Return it through the same // error channel a missing group uses — NEVER as a success payload of zeroes, diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index d79cd4de5..7981a3d0c 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -8,8 +8,12 @@ import { getMaxResidentRepos, } from '../lbug/pool-adapter.js'; import { + findRegistryEntryByName, + canonicalizePath, + registryPathEquals, readRegistry, readRegistryStrict, + RegistryAmbiguousTargetError, type RegistryEntry, } from '../../storage/repo-manager.js'; import type { @@ -128,9 +132,19 @@ export function stableRepoPoolId(entry: RegistryEntry, allEntries: RegistryEntry return base; } +/** Operator copy for group sync — unique `--name`, not a path in yaml. */ +export function formatGroupSyncAmbiguousError(err: RegistryAmbiguousTargetError): string { + const listing = err.matches.map((m) => ` - ${m.path}`).join('\n'); + return ( + `Multiple registered repos are named "${err.target}":\n${listing}\n` + + `Give each clone a unique registry name with \`gitnexus analyze --name\`, then re-sync. ` + + `Do not put a filesystem path in group.yaml.` + ); +} + function defaultResolveHandle(allEntries: RegistryEntry[]) { return async (registryName: string, groupPath: string): Promise => { - const e = allEntries.find((en) => en.name === registryName); + const e = findRegistryEntryByName(allEntries, registryName); if (!e) return null; const poolId = stableRepoPoolId(e, allEntries); return { @@ -277,7 +291,10 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis // Group-path → pool identity for repos that successfully initialized. Drives // windowed manifest resolution below (re-init + lease per window). Keyed by // group path because manifest links reference repos by group path. - const repoHandles = new Map(); + const repoHandles = new Map(); + // Keep resolved disk paths even when extraction fails and removes the + // corresponding handle; workspace discovery does not need a readable index. + const resolvedRepoPaths = new Map(); // Every eviction lease this sync holds. Window loops release their own leases // (bounding residency); this set is the defensive outer-finally sweep — // release disposers are idempotent, so double-release is a safe no-op. @@ -295,6 +312,11 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis registryEntries = await readRegistryStrict(); const entries = registryEntries; const resolve = opts?.resolveRepoHandle ?? defaultResolveHandle(entries); + if (!opts?.resolveRepoHandle) { + for (const regName of Object.values(config.repos)) { + findRegistryEntryByName(entries, regName); + } + } const httpEx = new HttpRouteExtractor(); const graphqlEx = new GraphqlExtractor(); const grpcEx = new GrpcExtractor(); @@ -308,6 +330,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis missingRepos.push(groupPath); continue; } + resolvedRepoPaths.set(groupPath, handle.repoPath); const poolId = handle.id; const lbugPath = path.join(handle.storagePath, 'lbug'); @@ -327,7 +350,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis // resolution no longer reuses these executors — it re-inits + leases // each repo per window (see windowed resolution below, issue #2189). // Record the pool identity so windowed resolution can re-init. - repoHandles.set(groupPath, { poolId, lbugPath }); + repoHandles.set(groupPath, { poolId, lbugPath, repoPath: handle.repoPath }); const executor: CypherExecutor = (query, params) => executeParameterized(poolId, query, params ?? {}); @@ -409,7 +432,10 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis lastCommit: m.lastCommit || '', }; } catch { - const e = entries.find((en) => en.name === regName); + const resolvedHandlePath = canonicalizePath(handle.repoPath); + const e = entries.find((en) => + registryPathEquals(canonicalizePath(en.path), resolvedHandlePath), + ); repoSnapshots[groupPath] = { indexedAt: e?.indexedAt || '', lastCommit: e?.lastCommit || '', @@ -431,6 +457,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis // read. The loop bounds the append by memory instead. for (const contract of repoContracts) autoContracts.push(contract); } catch (err) { + if (err instanceof RegistryAmbiguousTargetError) throw err; // This spans initLbug plus all contract extraction for the repo. The // error used to be discarded entirely, so the only trace of (say) a // storage-version mismatch was an empty contracts.json and a later @@ -465,7 +492,13 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis const repoPaths = new Map(); if (!registryEntries) registryEntries = await readRegistry(); for (const [groupPath, regName] of Object.entries(config.repos)) { - const e = registryEntries.find((en) => en.name === regName); + const resolvedPath = resolvedRepoPaths.get(groupPath); + if (resolvedPath) { + repoPaths.set(groupPath, resolvedPath); + continue; + } + if (opts?.resolveRepoHandle) continue; + const e = findRegistryEntryByName(registryEntries, regName); if (e) repoPaths.set(groupPath, e.path); } diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 018efc42a..ad9841406 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1308,6 +1308,13 @@ async function runFullAnalysisInner( } progress('fts', 90, 'Search indexes ready'); progress('done', 100, 'Done'); + if (options.registryName) { + await registerRepo(repoPath, existingMeta, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: placement.branch, + }); + } return { repoName: options.registryName ?? @@ -1687,6 +1694,35 @@ async function runFullAnalysisInner( // later read on a host where it loads — which is a legitimate, common // state, and the invariant `analyzer-identity-cli.test.ts` pins. if (!dirty && !healUnregistered) { + if (options.registryName) { + await registerRepo(repoPath, existingMeta, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + branch: placement.branch, + }); + if (!placement.branch) { + try { + await generateAIContextFiles( + repoPath, + storagePath, + options.registryName, + existingMeta.stats ?? {}, + undefined, + { + skipAgentsMd: options.skipAgentsMd, + skipSkills: options.skipSkills, + noStats: options.noStats, + defaultBranch: options.defaultBranch, + // Fast path does not re-run PDG. Using `options.pdg` would + // strip PDG bullets from AGENTS.md on a rename-only analyze. + hasPdg: existingMeta.pdg != null, + }, + ); + } catch { + /* best-effort — never fail the fast path over a context refresh */ + } + } + } // ── #2354: restamp the workspace label on a same-commit branch flip ── // The flat slot follows the checked-out working tree; a branch switch // at the SAME commit with a clean tree changes nothing the pipeline diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index b223d5f69..3f68df80a 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -909,7 +909,10 @@ const registerRepoUnlocked = async ( // falling back to `path.resolve` when the path doesn't exist. const canonicalInput = canonicalizePath(repoPath); - const entries = await readRegistry(); + // Mutating writes must not treat an unreadable/truncated registry as empty + // (#3094): lenient `readRegistry()` returns `[]` on parse failure and would + // replace the machine-wide file with only this entry. ENOENT stays empty. + const entries = await readRegistryStrict(); const existingIdx = entries.findIndex((e) => { // Canonicalise the STORED entry too so pre-canonicalisation // registries (written by older versions, or paths passed in a @@ -1024,7 +1027,7 @@ const registerRepoUnlocked = async ( // R9): re-derive THIS run's delta against the FRESHEST snapshot so a // concurrent change to the OTHER axis (a branch upsert vs a primary refresh) // survives instead of being clobbered by a stale entry-time view. - const fresh = await readRegistry(); + const fresh = await readRegistryStrict(); const freshIdx = fresh.findIndex((e) => { const a = canonicalizePath(e.path); return registryPathEquals(a, canonicalInput); @@ -1469,6 +1472,27 @@ export const resolveRegistryEntry = (entries: RegistryEntry[], target: string): throw new RegistryNotFoundError(target, availableNames); }; +/** + * Name-only registry match (the name tier of {@link resolveRegistryEntry}, + * without path matching). Used by `group.yaml` member *values*, which are + * registry aliases, not filesystem paths. + * + * Zero matches → `undefined` (caller treats as missing). One match → that + * entry. Two or more → {@link RegistryAmbiguousTargetError}. + */ +export const findRegistryEntryByName = ( + entries: RegistryEntry[], + name: string, +): RegistryEntry | undefined => { + const targetLower = name.toLowerCase(); + const nameMatches = entries.filter((e) => e.name.toLowerCase() === targetLower); + if (nameMatches.length === 1) return nameMatches[0]; + if (nameMatches.length > 1) { + throw new RegistryAmbiguousTargetError(name, nameMatches); + } + return undefined; +}; + /** * List all registered repos from the global registry. * diff --git a/gitnexus/test/integration/group/group-cli.test.ts b/gitnexus/test/integration/group/group-cli.test.ts index a9c1840cf..45dda9e7a 100644 --- a/gitnexus/test/integration/group/group-cli.test.ts +++ b/gitnexus/test/integration/group/group-cli.test.ts @@ -55,6 +55,51 @@ describe('group CLI', () => { expect(l.stdout).toContain('acme'); }); + it('sync exits nonzero with formatted copy when a member name is ambiguous', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-group-cli-amb-')); + try { + fs.mkdirSync(path.join(home, 'groups', 'g1'), { recursive: true }); + fs.writeFileSync( + path.join(home, 'groups', 'g1', 'group.yaml'), + `version: 1 +name: g1 +repos: + demo/api: demo-api +`, + ); + const cloneA = path.join(home, 'clone-a'); + const cloneB = path.join(home, 'clone-b'); + fs.mkdirSync(path.join(cloneA, '.gitnexus'), { recursive: true }); + fs.mkdirSync(path.join(cloneB, '.gitnexus'), { recursive: true }); + fs.writeFileSync( + path.join(home, 'registry.json'), + JSON.stringify([ + { + name: 'demo-api', + path: cloneA, + storagePath: path.join(cloneA, '.gitnexus'), + indexedAt: '2026-01-01T00:00:00.000Z', + lastCommit: 'aaa', + }, + { + name: 'demo-api', + path: cloneB, + storagePath: path.join(cloneB, '.gitnexus'), + indexedAt: '2026-01-01T00:00:00.000Z', + lastCommit: 'bbb', + }, + ]), + ); + const r = runGroupIn(home, ['sync', 'g1']); + expect(r.status).not.toBe(0); + // CLI logs JSON (pino): quotes around the group name are escaped in the byte stream. + expect(`${r.stderr}${r.stdout}`).toMatch(/Did not sync group \\"g1\\"/); + expect(`${r.stderr}${r.stdout}`).toContain('demo-api'); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it('test_create_with_invalid_name_fails', () => { const result = runGroup(['create', '../../evil']); expect(result.status).not.toBe(0); diff --git a/gitnexus/test/unit/group-service-not-found.test.ts b/gitnexus/test/unit/group-service-not-found.test.ts index b16f9074f..f3961b9ab 100644 --- a/gitnexus/test/unit/group-service-not-found.test.ts +++ b/gitnexus/test/unit/group-service-not-found.test.ts @@ -21,7 +21,10 @@ vi.mock('../../src/core/group/storage.js', () => ({ listGroups: listGroupsMock, })); -vi.mock('../../src/core/group/sync.js', () => ({ syncGroup: syncGroupMock })); +vi.mock('../../src/core/group/sync.js', () => ({ + syncGroup: syncGroupMock, + formatGroupSyncAmbiguousError: (err: Error) => err.message, +})); vi.mock('../../src/core/git-staleness.js', () => ({ checkStaleness: vi.fn() })); describe('GroupService — missing group error handling', () => { diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index 9644b304b..a17ebe148 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -245,6 +245,25 @@ links: expect(() => parseGroupConfig('version: 1\nname: test')).toThrow(/repos.*required/i); }); + it('throws when a repos value is not a string (YAML number/boolean)', () => { + expect(() => + parseGroupConfig(`version: 1 +name: test +repos: + app: 12 +`), + ).toThrow(/non-empty registry name string/); + }); + + it('trims padded registry aliases so they match the registry name', () => { + const config = parseGroupConfig(`version: 1 +name: test +repos: + app: " my-app " +`); + expect(config.repos.app).toBe('my-app'); + }); + it('allows empty repos object (fresh group before first add)', () => { const yaml = `version: 1 name: new-group diff --git a/gitnexus/test/unit/group/cross-impact.test.ts b/gitnexus/test/unit/group/cross-impact.test.ts index bceaa9195..9d718a0bf 100644 --- a/gitnexus/test/unit/group/cross-impact.test.ts +++ b/gitnexus/test/unit/group/cross-impact.test.ts @@ -444,4 +444,102 @@ describe('cross-impact', () => { cleanup(); } }); + + it('hints the yaml member path when --repo is the registry alias', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-ci-alias-')); + const groupDir = path.join(tmpDir, 'groups', 'g1'); + fs.mkdirSync(groupDir, { recursive: true }); + fs.writeFileSync( + path.join(groupDir, 'group.yaml'), + `version: 1 +name: g1 +repos: + demo/api: demo-api + demo/web: demo-web +`, + ); + vi.stubEnv('GITNEXUS_HOME', tmpDir); + try { + const port: GroupToolPort = { + resolveRepo: vi.fn(), + impact: vi.fn(), + query: vi.fn(), + impactByUid: vi.fn(), + context: vi.fn(), + }; + const r = await runGroupImpact( + { port, gitnexusDir: tmpDir }, + { + name: 'g1', + repo: 'demo-api', + target: 'Sym', + direction: 'upstream', + }, + ); + expect('error' in r).toBe(true); + if ('error' in r) { + expect(r.error).toContain('demo/api'); + expect(r.error).toMatch(/registry alias/i); + expect(r.error).not.toContain('demo/web'); + } + const mixedCase = await runGroupImpact( + { port, gitnexusDir: tmpDir }, + { + name: 'g1', + repo: 'Demo-API', + target: 'Sym', + direction: 'upstream', + }, + ); + expect('error' in mixedCase).toBe(true); + if ('error' in mixedCase) { + expect(mixedCase.error).toContain('demo/api'); + } + } finally { + vi.unstubAllEnvs(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('lists every member path that shares the same registry alias', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-ci-alias-dup-')); + const groupDir = path.join(tmpDir, 'groups', 'g1'); + fs.mkdirSync(groupDir, { recursive: true }); + fs.writeFileSync( + path.join(groupDir, 'group.yaml'), + `version: 1 +name: g1 +repos: + demo/api: shared + demo/other: shared +`, + ); + vi.stubEnv('GITNEXUS_HOME', tmpDir); + try { + const port: GroupToolPort = { + resolveRepo: vi.fn(), + impact: vi.fn(), + query: vi.fn(), + impactByUid: vi.fn(), + context: vi.fn(), + }; + const r = await runGroupImpact( + { port, gitnexusDir: tmpDir }, + { + name: 'g1', + repo: 'shared', + target: 'Sym', + direction: 'upstream', + }, + ); + expect('error' in r).toBe(true); + if ('error' in r) { + expect(r.error).toContain('demo/api'); + expect(r.error).toContain('demo/other'); + } + } finally { + vi.unstubAllEnvs(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/group/resolve-bridge-neighbors.test.ts b/gitnexus/test/unit/group/resolve-bridge-neighbors.test.ts index 73ffa5b7e..13876c897 100644 --- a/gitnexus/test/unit/group/resolve-bridge-neighbors.test.ts +++ b/gitnexus/test/unit/group/resolve-bridge-neighbors.test.ts @@ -127,4 +127,60 @@ describe('resolveBridgeNeighbors', () => { expect(rows).toEqual([]); await closeBridgeDb(handle!); }); + + itLbugReopen( + 'registry alias as localRepo does not join contracts stamped with the member path', + async () => { + const consumer = makeContract({ + repo: 'demo/api', + role: 'consumer', + symbolUid: 'consumer-uid', + symbolRef: { filePath: 'src/api.ts', name: 'fetchUsers' }, + symbolName: 'fetchUsers', + contractId: 'http::GET::/api/users', + confidence: 0.5, + }); + const provider = makeContract({ + repo: 'demo/api', + role: 'provider', + symbolUid: 'provider-uid', + symbolRef: { filePath: 'src/routes.ts', name: 'getUsers' }, + symbolName: 'getUsers', + contractId: 'http::GET::/api/users', + confidence: 0.9, + }); + const link: CrossLink = { + from: { repo: 'web', symbolUid: 'web-uid', symbolRef: consumer.symbolRef }, + to: { repo: 'demo/api', symbolUid: 'provider-uid', symbolRef: provider.symbolRef }, + type: 'http', + contractId: 'http::GET::/api/users', + matchType: 'manifest', + confidence: 0.9, + }; + await writeBridge(tmpDir, { + contracts: [{ ...consumer, repo: 'web' }, provider], + crossLinks: [link], + repoSnapshots: {}, + missingRepos: [], + }); + const handle = await openBridgeDbReadOnly(tmpDir); + const aliasMiss = await resolveBridgeNeighbors(handle!, { + localRepo: 'demo-api', + uids: ['provider-uid'], + direction: 'upstream', + }); + expect(aliasMiss).toEqual([]); + const pathHit = await resolveBridgeNeighbors(handle!, { + localRepo: 'demo/api', + uids: ['provider-uid'], + direction: 'upstream', + }); + expect(pathHit).toHaveLength(1); + expect(pathHit[0]).toMatchObject({ + neighborRepo: 'web', + matchType: 'manifest', + }); + await closeBridgeDb(handle!); + }, + ); }); diff --git a/gitnexus/test/unit/group/service-group-sync-payload.test.ts b/gitnexus/test/unit/group/service-group-sync-payload.test.ts index ddfa32d71..ea2792b7d 100644 --- a/gitnexus/test/unit/group/service-group-sync-payload.test.ts +++ b/gitnexus/test/unit/group/service-group-sync-payload.test.ts @@ -50,6 +50,7 @@ const syncGroupMock = vi.fn<() => Promise>(); vi.mock('../../../src/core/group/sync.js', () => ({ syncGroup: (...args: unknown[]) => syncGroupMock(...(args as [])), + formatGroupSyncAmbiguousError: (err: Error) => err.message, })); const { GroupService } = await import('../../../src/core/group/service.js'); diff --git a/gitnexus/test/unit/group/sync-partial-extraction.test.ts b/gitnexus/test/unit/group/sync-partial-extraction.test.ts index 06b3bf450..0e7d41228 100644 --- a/gitnexus/test/unit/group/sync-partial-extraction.test.ts +++ b/gitnexus/test/unit/group/sync-partial-extraction.test.ts @@ -54,10 +54,14 @@ vi.mock('../../../src/core/lbug/pool-adapter.js', () => ({ getMaxResidentRepos: vi.fn(() => 5), })); -vi.mock('../../../src/storage/repo-manager.js', () => ({ - readRegistry: vi.fn(async () => []), - readRegistryStrict: vi.fn(async () => []), -})); +vi.mock('../../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readRegistry: vi.fn(async () => []), + readRegistryStrict: vi.fn(async () => []), + }; +}); vi.mock('../../../src/core/group/extractors/http-route-extractor.js', () => ({ HttpRouteExtractor: class { diff --git a/gitnexus/test/unit/group/sync-registry-identity.test.ts b/gitnexus/test/unit/group/sync-registry-identity.test.ts new file mode 100644 index 000000000..309a2c0fd --- /dev/null +++ b/gitnexus/test/unit/group/sync-registry-identity.test.ts @@ -0,0 +1,273 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs/promises'; +import { mkdirSync } from 'node:fs'; +import path from 'node:path'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import { RegistryAmbiguousTargetError } from '../../../src/storage/repo-manager.js'; +import { createTempDir } from '../../helpers/test-db.js'; +import type { GroupConfig } from '../../../src/core/group/types.js'; +import { GroupService } from '../../../src/core/group/service.js'; +import type { GroupToolPort } from '../../../src/core/group/service.js'; + +const initLbugMock = vi.fn(async () => {}); + +vi.mock('../../../src/core/lbug/pool-adapter.js', () => ({ + initLbug: (...args: unknown[]) => initLbugMock(...args), + executeParameterized: vi.fn(async () => []), + pinRepo: vi.fn(() => () => {}), + getMaxResidentRepos: vi.fn(() => 5), +})); + +const makeConfig = (repos: Record, extra?: Partial): GroupConfig => ({ + version: 1, + name: 'test', + description: '', + repos, + links: [], + packages: {}, + detect: { + http: false, + graphql: false, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: false, + }, + matching: {}, + ...extra, +}); + +const row = ( + tmpHome: string, + name: string, + clone: string, +): { + name: string; + path: string; + storagePath: string; + indexedAt: string; + lastCommit: string; +} => { + mkdirSync(path.join(tmpHome, 'repos', clone), { recursive: true }); + return { + name, + path: path.join(tmpHome, 'repos', clone), + storagePath: path.join(tmpHome, 'repos', clone, '.gitnexus'), + indexedAt: '2026-01-01T00:00:00.000Z', + lastCommit: 'abc123', + }; +}; + +describe('syncGroup registry name identity', () => { + let tmpHome: Awaited>; + let savedGitnexusHome: string | undefined; + let registryPath: string; + + beforeEach(async () => { + initLbugMock.mockReset(); + initLbugMock.mockResolvedValue(undefined); + tmpHome = await createTempDir('gitnexus-sync-registry-id-'); + savedGitnexusHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + registryPath = path.join(tmpHome.dbPath, 'registry.json'); + }); + + afterEach(async () => { + if (savedGitnexusHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedGitnexusHome; + await tmpHome.cleanup(); + }); + + it('throws RegistryAmbiguousTargetError and does not rewrite group dir files', async () => { + const a = row(tmpHome.dbPath, 'demo-api', 'clone-a'); + const b = row(tmpHome.dbPath, 'demo-api', 'clone-b'); + await fs.writeFile(registryPath, JSON.stringify([a, b])); + + const groupDir = path.join(tmpHome.dbPath, 'groups', 'g'); + await fs.mkdir(groupDir, { recursive: true }); + const contractsPath = path.join(groupDir, 'contracts.json'); + const prior = '{"contracts":[],"crossLinks":[],"marker":"keep"}\n'; + await fs.writeFile(contractsPath, prior); + + await expect(syncGroup(makeConfig({ 'demo/api': 'demo-api' }), { groupDir })).rejects.toSatisfy( + (err: unknown) => { + expect(err).toBeInstanceOf(RegistryAmbiguousTargetError); + const amb = err as RegistryAmbiguousTargetError; + expect(amb.matches).toHaveLength(2); + expect(amb.matches.map((m) => m.path).sort()).toEqual([a.path, b.path].sort()); + return true; + }, + ); + + expect(await fs.readFile(contractsPath, 'utf-8')).toBe(prior); + await expect(fs.access(path.join(groupDir, 'bridge.lbug'))).rejects.toThrow(); + }); + + it('records an unknown yaml value as missing and still extracts other members', async () => { + const known = row(tmpHome.dbPath, 'backend-repo', 'backend'); + await fs.writeFile(registryPath, JSON.stringify([known])); + + const result = await syncGroup( + makeConfig({ 'app/backend': 'backend-repo', 'app/ghost': 'ghost' }), + { skipWrite: true }, + ); + + expect(result.missingRepos).toEqual(['app/ghost']); + expect(result.unreadableRepos).toEqual([]); + expect(result.repoSnapshots['app/backend']).toEqual({ + indexedAt: known.indexedAt, + lastCommit: known.lastCommit, + }); + }); + + it('treats mixed missing and ambiguous names as a terminal ambiguity with no write', async () => { + const a = row(tmpHome.dbPath, 'demo-api', 'clone-a'); + const b = row(tmpHome.dbPath, 'demo-api', 'clone-b'); + await fs.writeFile(registryPath, JSON.stringify([a, b])); + + const groupDir = path.join(tmpHome.dbPath, 'groups', 'g'); + await fs.mkdir(groupDir, { recursive: true }); + const contractsPath = path.join(groupDir, 'contracts.json'); + await fs.writeFile(contractsPath, '{"keep":true}'); + + await expect( + syncGroup(makeConfig({ 'demo/api': 'demo-api', 'app/ghost': 'ghost' }), { groupDir }), + ).rejects.toBeInstanceOf(RegistryAmbiguousTargetError); + + expect(await fs.readFile(contractsPath, 'utf-8')).toBe('{"keep":true}'); + }); + + it('injected resolveRepoHandle still bypasses default name matching', async () => { + const a = row(tmpHome.dbPath, 'demo-api', 'clone-a'); + const b = row(tmpHome.dbPath, 'demo-api', 'clone-b'); + await fs.writeFile(registryPath, JSON.stringify([a, b])); + + const result = await syncGroup(makeConfig({ 'demo/api': 'demo-api' }), { + skipWrite: true, + resolveRepoHandle: async (_name, groupPath) => ({ + id: 'injected', + path: groupPath, + repoPath: a.path, + storagePath: a.storagePath, + }), + }); + + expect(result.missingRepos).toEqual([]); + expect(result.unreadableRepos).toEqual([]); + }); + + it('does not treat a filesystem path yaml value as a registry hit', async () => { + const known = row(tmpHome.dbPath, 'backend-repo', 'backend'); + await fs.writeFile(registryPath, JSON.stringify([known])); + + const result = await syncGroup(makeConfig({ 'app/backend': known.path }), { skipWrite: true }); + + expect(result.missingRepos).toEqual(['app/backend']); + expect(result.repoSnapshots['app/backend']).toBeUndefined(); + }); + + it('injected resolveRepoHandle plus workspace_deps does not throw on duplicate names', async () => { + const a = row(tmpHome.dbPath, 'demo-api', 'clone-a'); + const b = row(tmpHome.dbPath, 'demo-api', 'clone-b'); + await fs.writeFile(registryPath, JSON.stringify([a, b])); + + const result = await syncGroup( + makeConfig( + { 'demo/api': 'demo-api' }, + { + detect: { + http: false, + graphql: false, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: true, + }, + }, + ), + { + skipWrite: true, + resolveRepoHandle: async (_name, groupPath) => ({ + id: 'injected', + path: groupPath, + repoPath: a.path, + storagePath: a.storagePath, + }), + }, + ); + + expect(result.missingRepos).toEqual([]); + }); + + it('injected resolveRepoHandle plus workspace_deps still bypasses name lookup after extraction failure', async () => { + const a = row(tmpHome.dbPath, 'demo-api', 'clone-a'); + const b = row(tmpHome.dbPath, 'demo-api', 'clone-b'); + await fs.writeFile(registryPath, JSON.stringify([a, b])); + initLbugMock.mockRejectedValueOnce(new Error('init failed')); + + const result = await syncGroup( + makeConfig( + { 'demo/api': 'demo-api' }, + { + detect: { + http: false, + graphql: false, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: true, + }, + }, + ), + { + skipWrite: true, + resolveRepoHandle: async (_name, groupPath) => ({ + id: 'injected', + path: groupPath, + repoPath: a.path, + storagePath: a.storagePath, + }), + }, + ); + + expect(result.missingRepos).toEqual([]); + expect(result.unreadableRepos).toEqual(['demo/api']); + }); + + it('MCP groupSync returns { error } for an ambiguous registry name', async () => { + const a = row(tmpHome.dbPath, 'demo-api', 'clone-a'); + const b = row(tmpHome.dbPath, 'demo-api', 'clone-b'); + await fs.writeFile(registryPath, JSON.stringify([a, b])); + + const groupDir = path.join(tmpHome.dbPath, 'groups', 'g1'); + await fs.mkdir(groupDir, { recursive: true }); + await fs.writeFile( + path.join(groupDir, 'group.yaml'), + `version: 1 +name: g1 +repos: + demo/api: demo-api +`, + ); + + const port: GroupToolPort = { + resolveRepo: vi.fn(), + impact: vi.fn(), + query: vi.fn(), + impactByUid: vi.fn(), + context: vi.fn(), + }; + const svc = new GroupService(port); + const payload = (await svc.groupSync({ name: 'g1' })) as { error?: string }; + + expect(payload.error).toBeDefined(); + expect(payload.error).toContain('demo-api'); + expect(payload.error).toContain(a.path); + expect(payload.error).toContain(b.path); + expect(payload.error).toMatch(/unique registry name/i); + expect(payload.error).not.toMatch(/Pass the absolute path/); + }); +}); diff --git a/gitnexus/test/unit/group/sync-unreadable-repos.test.ts b/gitnexus/test/unit/group/sync-unreadable-repos.test.ts index 12a85439c..5f565d443 100644 --- a/gitnexus/test/unit/group/sync-unreadable-repos.test.ts +++ b/gitnexus/test/unit/group/sync-unreadable-repos.test.ts @@ -66,10 +66,14 @@ vi.mock('../../../src/core/lbug/pool-adapter.js', () => ({ getMaxResidentRepos: vi.fn(() => 5), })); -vi.mock('../../../src/storage/repo-manager.js', () => ({ - readRegistry: (...args: unknown[]) => readRegistryLenientMock(...args), - readRegistryStrict: (...args: unknown[]) => readRegistryStrictMock(...args), -})); +vi.mock('../../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readRegistry: (...args: unknown[]) => readRegistryLenientMock(...args), + readRegistryStrict: (...args: unknown[]) => readRegistryStrictMock(...args), + }; +}); /** * Armed by the bridge-write-failure suite at the bottom of this file, `null` diff --git a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts index aaf361b87..d96eeec26 100644 --- a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts +++ b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts @@ -156,10 +156,14 @@ vi.mock('../../../src/core/lbug/sidecar-recovery.js', () => ({ // The registry read happens in syncGroup's else branch; resolveRepoHandle is // supplied, so an empty registry is fine (only the meta.json fallback reads it). -vi.mock('../../../src/storage/repo-manager.js', () => ({ - readRegistry: vi.fn().mockResolvedValue([]), - readRegistryStrict: vi.fn().mockResolvedValue([]), -})); +vi.mock('../../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readRegistry: vi.fn().mockResolvedValue([]), + readRegistryStrict: vi.fn().mockResolvedValue([]), + }; +}); const { syncGroup } = await import('../../../src/core/group/sync.js'); const { closeLbug, getMaxResidentRepos } = await import('../../../src/core/lbug/pool-adapter.js'); diff --git a/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts b/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts index 76d79e295..23241517d 100644 --- a/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts +++ b/gitnexus/test/unit/repo-manager-registry-strict-read.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import fs from 'node:fs/promises'; import path from 'node:path'; import { inspect } from 'node:util'; -import { readRegistry, readRegistryStrict } from '../../src/storage/repo-manager.js'; +import { readRegistry, readRegistryStrict, registerRepo } from '../../src/storage/repo-manager.js'; import { _captureLogger, type LoggerCapture } from '../../src/core/logger.js'; import { createTempDir } from '../helpers/test-db.js'; import { syncGroup } from '../../src/core/group/sync.js'; @@ -116,6 +116,20 @@ describe('readRegistryStrict', () => { await expect(readRegistryStrict()).rejects.toThrow(); }); + it('registerRepo refuses to overwrite a truncated registry with a single entry', async () => { + const prior = '{"truncated": '; + await fs.writeFile(registryPath, prior); + await expect( + registerRepo('/repos/one', { + repoPath: '/repos/one', + lastCommit: 'abc', + indexedAt: '2026-01-01T00:00:00.000Z', + stats: {}, + }), + ).rejects.toThrow('registry is corrupt'); + expect(await fs.readFile(registryPath, 'utf-8')).toBe(prior); + }); + it('throws when a row is missing the fields the resolver needs', async () => { // `[{}]` is a JSON array, so an array-shape check alone waved it through. // Every configured repo then failed to resolve and landed in missingRepos; diff --git a/gitnexus/test/unit/repo-manager.test.ts b/gitnexus/test/unit/repo-manager.test.ts index 2080e2f45..468096a61 100644 --- a/gitnexus/test/unit/repo-manager.test.ts +++ b/gitnexus/test/unit/repo-manager.test.ts @@ -28,6 +28,7 @@ import { adoptFlatBranchLabel, listRegisteredRepos, resolveRegistryEntry, + findRegistryEntryByName, canonicalizePath, registryPathEquals, cloneDirBelongsToEntry, @@ -1535,6 +1536,13 @@ describe('resolveRegistryEntry (#664)', () => { expect(resolveRegistryEntry(entries, 'Website')).toBe(entries[2]); }); + it('findRegistryEntryByName is name-only: a filesystem path is a miss, not a path-tier hit', () => { + expect(findRegistryEntryByName(entries, pathA)).toBeUndefined(); + expect(findRegistryEntryByName(entries, 'website')).toBe(entries[2]); + expect(findRegistryEntryByName(entries, 'WEBSITE')).toBe(entries[2]); + expect(() => findRegistryEntryByName(entries, 'app')).toThrow(RegistryAmbiguousTargetError); + }); + it('path match is case-insensitive on Windows only', () => { if (process.platform !== 'win32') { // On POSIX, a differently-cased path must NOT match. Verify by diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 24d03844f..091c7f915 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -359,6 +359,52 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { } }); + it('--repair-fts applies analyze --name without a full re-index', async () => { + vi.doMock('../../src/core/lbug/lbug-adapter.js', () => mockRepairSuccessLbugAdapter()); + vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + createSearchFTSIndexes: vi.fn(async () => []), + verifySearchFTSIndexes: vi.fn(async () => []), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => ({ + ...(await importActual()), + ensureGitNexusIgnored: vi.fn(async () => undefined), + })); + + const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-name-'); + const tmpHome = await createTempDir('gitnexus-run-analyze-repair-name-home-'); + const savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + const seeded: RepoMeta = { + repoPath: tmpRepo.dbPath, + lastCommit: 'abc123', + indexedAt: new Date().toISOString(), + stats: { files: 1, nodes: 1, edges: 1 }, + }; + await saveMeta(storagePath, seeded); + const { registerRepo, readRegistry } = await import('../../src/storage/repo-manager.js'); + await registerRepo(tmpRepo.dbPath, seeded, { name: 'old' }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true, registryName: 'new' }, + { onProgress: () => {} }, + ); + expect(result.ftsRepairedOnly).toBe(true); + expect((await readRegistry())[0].name).toBe('new'); + } finally { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + await tmpRepo.cleanup(); + } + }); + it('--repair-fts backfills a full capabilities object when the existing meta predates the field entirely (#2767)', async () => { vi.doMock('../../src/core/lbug/lbug-adapter.js', () => mockRepairSuccessLbugAdapter()); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ diff --git a/gitnexus/test/unit/run-analyze.test.ts b/gitnexus/test/unit/run-analyze.test.ts index aef31907c..ea4525b61 100644 --- a/gitnexus/test/unit/run-analyze.test.ts +++ b/gitnexus/test/unit/run-analyze.test.ts @@ -14,6 +14,8 @@ import { loadMeta, registerRepo, saveMeta, + readRegistry, + RegistryNameCollisionError, type RepoMeta, } from '../../src/storage/repo-manager.js'; import { SCHEMA_FINGERPRINT } from '../../src/core/lbug/schema.js'; @@ -90,6 +92,197 @@ describe('run-analyze module', () => { } }); + it('applies analyze --name on the already-up-to-date path without --force', async () => { + const tmpRepo = await createTempDir('gitnexus-run-analyze-fast-name-'); + const tmpHome = await createTempDir('gitnexus-run-analyze-fast-name-home-'); + const savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + try { + execSync('git init', { cwd: tmpRepo.dbPath, stdio: 'pipe' }); + execSync('git -c user.name=t -c user.email=t@t commit --allow-empty -m init', { + cwd: tmpRepo.dbPath, + stdio: 'pipe', + }); + const currentCommit = execSync('git rev-parse HEAD', { + cwd: tmpRepo.dbPath, + encoding: 'utf-8', + }).trim(); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const meta: RepoMeta = { + repoPath: tmpRepo.dbPath, + lastCommit: currentCommit, + indexedAt: new Date().toISOString(), + schemaFingerprint: SCHEMA_FINGERPRINT, + analysisFeatures: CURRENT_ANALYSIS_FEATURES, + runnerIdentity: currentRunnerIdentity(), + }; + await saveMeta(storagePath, meta); + await registerRepo(tmpRepo.dbPath, meta, { name: 'old' }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { registryName: 'new' }, + { onProgress: () => {} }, + ); + + expect(result.alreadyUpToDate).toBe(true); + expect(result.repoName).toBe('new'); + const entries = await readRegistry(); + expect(entries).toHaveLength(1); + expect(entries[0].name).toBe('new'); + const agents = await fs.readFile(path.join(tmpRepo.dbPath, 'AGENTS.md'), 'utf-8'); + expect(agents).toContain('**new**'); + } finally { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + await tmpRepo.cleanup(); + } + }); + + it('repeating the same --name on the fast path is a no-op, not an error', async () => { + const tmpRepo = await createTempDir('gitnexus-run-analyze-fast-name-repeat-'); + const tmpHome = await createTempDir('gitnexus-run-analyze-fast-name-repeat-home-'); + const savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + try { + execSync('git init', { cwd: tmpRepo.dbPath, stdio: 'pipe' }); + execSync('git -c user.name=t -c user.email=t@t commit --allow-empty -m init', { + cwd: tmpRepo.dbPath, + stdio: 'pipe', + }); + const currentCommit = execSync('git rev-parse HEAD', { + cwd: tmpRepo.dbPath, + encoding: 'utf-8', + }).trim(); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const meta: RepoMeta = { + repoPath: tmpRepo.dbPath, + lastCommit: currentCommit, + indexedAt: new Date().toISOString(), + schemaFingerprint: SCHEMA_FINGERPRINT, + analysisFeatures: CURRENT_ANALYSIS_FEATURES, + runnerIdentity: currentRunnerIdentity(), + }; + await saveMeta(storagePath, meta); + await registerRepo(tmpRepo.dbPath, meta, { name: 'kept' }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { registryName: 'kept' }, + { onProgress: () => {} }, + ); + + expect(result.alreadyUpToDate).toBe(true); + expect((await readRegistry())[0].name).toBe('kept'); + } finally { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + await tmpRepo.cleanup(); + } + }); + + it('fast-path --name still collides when another path already owns the alias', async () => { + const tmpA = await createTempDir('gitnexus-run-analyze-fast-name-col-a-'); + const tmpB = await createTempDir('gitnexus-run-analyze-fast-name-col-b-'); + const tmpHome = await createTempDir('gitnexus-run-analyze-fast-name-col-home-'); + const savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + try { + for (const tmp of [tmpA, tmpB]) { + execSync('git init', { cwd: tmp.dbPath, stdio: 'pipe' }); + execSync('git -c user.name=t -c user.email=t@t commit --allow-empty -m init', { + cwd: tmp.dbPath, + stdio: 'pipe', + }); + } + const commitB = execSync('git rev-parse HEAD', { + cwd: tmpB.dbPath, + encoding: 'utf-8', + }).trim(); + const metaA: RepoMeta = { + repoPath: tmpA.dbPath, + lastCommit: 'aaaa', + indexedAt: new Date().toISOString(), + schemaFingerprint: SCHEMA_FINGERPRINT, + analysisFeatures: CURRENT_ANALYSIS_FEATURES, + runnerIdentity: currentRunnerIdentity(), + }; + await registerRepo(tmpA.dbPath, metaA, { name: 'new' }); + + const { storagePath } = getStoragePaths(tmpB.dbPath); + const metaB: RepoMeta = { + repoPath: tmpB.dbPath, + lastCommit: commitB, + indexedAt: new Date().toISOString(), + schemaFingerprint: SCHEMA_FINGERPRINT, + analysisFeatures: CURRENT_ANALYSIS_FEATURES, + runnerIdentity: currentRunnerIdentity(), + }; + await saveMeta(storagePath, metaB); + await registerRepo(tmpB.dbPath, metaB, { name: 'old' }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await expect( + runFullAnalysis(tmpB.dbPath, { registryName: 'new' }, { onProgress: () => {} }), + ).rejects.toBeInstanceOf(RegistryNameCollisionError); + } finally { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + await tmpA.cleanup(); + await tmpB.cleanup(); + } + }); + + it('plain fast path does not call registerRepo when --name is absent', async () => { + const tmpRepo = await createTempDir('gitnexus-run-analyze-fast-no-name-'); + const tmpHome = await createTempDir('gitnexus-run-analyze-fast-no-name-home-'); + const savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + try { + execSync('git init', { cwd: tmpRepo.dbPath, stdio: 'pipe' }); + execSync('git -c user.name=t -c user.email=t@t commit --allow-empty -m init', { + cwd: tmpRepo.dbPath, + stdio: 'pipe', + }); + const currentCommit = execSync('git rev-parse HEAD', { + cwd: tmpRepo.dbPath, + encoding: 'utf-8', + }).trim(); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const meta: RepoMeta = { + repoPath: tmpRepo.dbPath, + lastCommit: currentCommit, + indexedAt: new Date().toISOString(), + schemaFingerprint: SCHEMA_FINGERPRINT, + analysisFeatures: CURRENT_ANALYSIS_FEATURES, + runnerIdentity: currentRunnerIdentity(), + }; + await saveMeta(storagePath, meta); + await registerRepo(tmpRepo.dbPath, meta, { name: 'original' }); + + const registerSpy = vi.spyOn( + await import('../../src/storage/repo-manager.js'), + 'registerRepo', + ); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis(tmpRepo.dbPath, {}, { onProgress: () => {} }); + expect(result.alreadyUpToDate).toBe(true); + expect(registerSpy).not.toHaveBeenCalled(); + expect((await readRegistry())[0].name).toBe('original'); + registerSpy.mockRestore(); + } finally { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + await tmpRepo.cleanup(); + } + }); + it('resumes a matching embedding checkpoint instead of taking the clean fast path', async () => { const tmpRepo = await createTempDir('gitnexus-run-analyze-embedding-checkpoint-'); const tmpHome = await createTempDir('gitnexus-run-analyze-embedding-checkpoint-home-'); From 4e51213c82249d5c235f8535d8292bd56de8c01a Mon Sep 17 00:00:00 2001 From: svector Date: Sun, 30 Aug 2026 09:03:05 +0100 Subject: [PATCH 24/28] fix(deps): bump transitive packages to patch disclosed CVEs (#3095) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(deps): bump transitive packages to patch disclosed CVEs Automated dependency bumps addressing already-disclosed advisories in gitnexus and gitnexus-web lockfiles. - undici 6.24.0 → 6.28.0 (@vercel/node override) — GHSA-vxpw-j846-p89q / CVE-2026-12151 - undici 7.25.0 → 7.29.0 (jsdom override) — GHSA-4cwx-7wf7-3272, GHSA-hm92-r4w5-c3mj, GHSA-vmh5-mc38-953g, GHSA-vxpw-j846-p89q - js-yaml 4.1.1 → 4.3.1 — GHSA-52cp-r559-cp3m / CVE-2026-59869, GHSA-5p4m-2wfm-xmqj - nanoid 3.3.16 → 3.3.18 — GHSA-2v37-7h3g-55p8 / CVE-2026-67213 - tar 7.5.20 → 7.5.22 — GHSA-r292-9mhp-454m / CVE-2026-73566 - protobufjs 7.6.4 → 7.6.6 (gitnexus override) — GHSA-j3f2-48v5-ccww / CVE-2026-59877 No application code changes. npm audit and osv-scanner report 0 remaining vulnerabilities on these two lockfiles after the bump. * fix(deps): prefer root bump and lockfile refresh --------- Co-authored-by: Svector-anu Co-authored-by: Gergő Magyar --- gitnexus-web/package-lock.json | 186 ++++----------------------------- gitnexus-web/package.json | 4 +- gitnexus/package-lock.json | 6 +- 3 files changed, 25 insertions(+), 171 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 82f082d15..53e1b5c69 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -57,7 +57,7 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.10.1", + "@vercel/node": "^5.10.2", "@vitejs/plugin-react": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", "jsdom": "^29.1.1", @@ -307,13 +307,6 @@ "specificity": "bin/cli.js" } }, - "node_modules/@bytecodealliance/preview2-shim": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@bytecodealliance/preview2-shim/-/preview2-shim-0.17.6.tgz", - "integrity": "sha512-n3cM88gTen5980UOBAD6xDcNNL3ocTK8keab21bpx1ONdA+ARj7uD1qoFxOWCyKlkpSi195FH+GeAut7Oc6zZw==", - "dev": true, - "license": "(Apache-2.0 WITH LLVM-exception)" - }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", @@ -1419,17 +1412,6 @@ "node": ">=20" } }, - "node_modules/@renovatebot/pep440": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@renovatebot/pep440/-/pep440-4.2.1.tgz", - "integrity": "sha512-2FK1hF93Fuf1laSdfiEmJvSJPVIDHEUTz68D3Fi9s0IZrrpaEcj6pTFBTbYvsgC5du4ogrtf5re7yMMvrKNgkw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.9.0 || ^22.11.0 || ^24", - "pnpm": "^10.0.0" - } - }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", @@ -1517,9 +1499,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1536,9 +1515,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1555,9 +1531,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1574,9 +1547,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1593,9 +1563,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1612,9 +1579,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1865,9 +1829,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1884,9 +1845,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1903,9 +1861,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1922,9 +1877,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2641,13 +2593,12 @@ } }, "node_modules/@vercel/build-utils": { - "version": "14.1.1", - "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-14.1.1.tgz", - "integrity": "sha512-kW9CeW0aokEBvX1rSgNyOKg90VyIQOmT0wBl7KXneM3Qs1+x4Puakqp97BdIgttWEtmN96UvdhQVG2bCA5JsPA==", + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-14.2.0.tgz", + "integrity": "sha512-GwmtB31tBXQEzFw11grr8BKFCBdUORmYeooB0ZtonaCXZMZaPCHLBFTMFKsvaV6ZciQORPInRwXShbFvmnjqtg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@vercel/python-analysis": "0.13.2", "cjs-module-lexer": "1.2.3", "es-module-lexer": "1.5.0" } @@ -2694,9 +2645,9 @@ } }, "node_modules/@vercel/node": { - "version": "5.10.1", - "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.10.1.tgz", - "integrity": "sha512-muj+t8sZ2XHQDkWcHxkql2rbvr/HhZOqYdZBG7pw8F5RLasL3o0gjHLXJKwHAEHp2I3fd3AgbMud2oz+hzeV0g==", + "version": "5.10.2", + "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.10.2.tgz", + "integrity": "sha512-YBXcoQVOh5O2ySXvzE+POhPEQEPMJJo4ctlMMdp5why/NIoa8m6gotv14j8Uo6D5qyZsnc+0+++JgUiV4mYB6w==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2704,7 +2655,7 @@ "@edge-runtime/primitives": "4.1.0", "@edge-runtime/vm": "3.2.0", "@types/node": "20.11.0", - "@vercel/build-utils": "14.1.1", + "@vercel/build-utils": "14.2.0", "@vercel/error-utils": "2.2.1", "@vercel/nft": "1.10.0", "@vercel/static-config": "3.4.1", @@ -2741,32 +2692,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@vercel/python-analysis": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/@vercel/python-analysis/-/python-analysis-0.13.2.tgz", - "integrity": "sha512-IEr5K2gvX143NBoQc1W4BWrdDWjZwxnIT6UrL5Y1dnyH7Cqc4AV00FIAddB1YpnIZBJwT4ZhE8QbgqBeO6C9Zw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bytecodealliance/preview2-shim": "0.17.6", - "@renovatebot/pep440": "4.2.1", - "fs-extra": "11.1.1", - "js-yaml": "4.1.1", - "minimatch": "10.1.1", - "smol-toml": "1.5.2", - "zod": "3.22.4" - } - }, - "node_modules/@vercel/python-analysis/node_modules/zod": { - "version": "3.22.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", - "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, "node_modules/@vercel/static-config": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/@vercel/static-config/-/static-config-3.4.1.tgz", @@ -3054,13 +2979,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -4514,21 +4432,6 @@ "node": ">=0.4.x" } }, - "node_modules/fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -5200,19 +5103,6 @@ "license": "MIT", "peer": true }, - "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/jsdom": { "version": "29.1.1", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", @@ -5268,9 +5158,9 @@ } }, "node_modules/jsdom/node_modules/undici": { - "version": "7.25.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", - "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { @@ -5322,19 +5212,6 @@ "dev": true, "license": "MIT" }, - "node_modules/jsonfile": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", - "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, "node_modules/katex": { "version": "0.16.47", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", @@ -6887,9 +6764,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", @@ -7808,19 +7685,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/smol-toml": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", - "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">= 18" - }, - "funding": { - "url": "https://github.com/sponsors/cyyynthia" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -7955,9 +7819,9 @@ } }, "node_modules/tar": { - "version": "7.5.20", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.20.tgz", - "integrity": "sha512-9FcyK4PA6+WbzlTM9WhQm6vB5W7cP7dUiPsv1g7YDwEQnQ1CGpK3MGlKk/ITVWMk05kHZuBhmVhiv8LZoy/PFQ==", + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { @@ -8193,9 +8057,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "6.24.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.24.0.tgz", - "integrity": "sha512-lVLNosgqo5EkGqh5XUDhGfsMSoO8K0BAN0TyJLvwNRSl4xWGZlCVYsAIpa/OpA3TvmnM01GWcoKmc3ZWo5wKKA==", + "version": "6.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", "dev": true, "license": "MIT", "engines": { @@ -8296,16 +8160,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/use-sync-external-store": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 440bf4a54..09ac0b640 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -67,7 +67,7 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.10.1", + "@vercel/node": "^5.10.2", "@vitejs/plugin-react": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", "jsdom": "^29.1.1", @@ -83,7 +83,7 @@ }, "@vercel/node": { "path-to-regexp": "6.3.0", - "undici": "6.24.0" + "undici": "6.28.0" }, "@vercel/python-analysis": { "minimatch": "10.2.3", diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index b8b4c8b32..bd3ebd088 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -4521,9 +4521,9 @@ "license": "MIT" }, "node_modules/protobufjs": { - "version": "7.6.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", - "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "optional": true, From 9718e1247aec56745a32fbc19ae83f392f656f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sun, 30 Aug 2026 09:31:47 +0100 Subject: [PATCH 25/28] fix(parse): stabilize parse-cache chunks and cheapen ParsedFile loads (#3093) * fix(parse): stabilize parse-cache chunks and cheapen ParsedFile loads Hash-bucket membership so worker count and add/delete no longer reshuffle cache keys; GC and path sidecars keep small-shard scope-resolution from full-store JSON and empty GCs. Co-authored-by: Cursor * fix(review): apply review findings Drop the unused pool argument from cache-budget resolution, reuse path compare helpers, and copy durable sidecars via full shard paths. Co-authored-by: Cursor * fix(store): copy durable path sidecars via full shard paths Keep restore destinations relative to the run store even when sidecar names are derived from absolute json paths. Co-authored-by: Cursor * fix(store): fail closed on truncated ParsedFile path sidecars Skip JSON only when a sidecar is complete (NUL-free, trailing newline). Truncated listings without a NUL were able to omit wanted paths. Co-authored-by: Cursor * style: apply prettier to ParsedFile store and tests Match the PR autofix formatter so CI quality does not flag wrap-only diffs. Co-authored-by: Cursor * fix(store): tighten ParsedFile path sidecars from review Skip sidecar writes when a path contains CR/LF, and assert the skip path does not open non-intersecting JSON shards. Co-authored-by: Cursor * fix(store): yield on sidecar skips and assert restore copies listing bytes Skipped shards now count toward the 128-shard event-loop yield, and restore tests check sidecar contents rather than existence only. Co-authored-by: Cursor * fix(store): treat path sidecars as best-effort after a JSON shard write A sidecar ENOSPC/EACCES must not fail persist; load already falls back to the JSON shard when the listing is missing. Co-authored-by: Cursor * fix(store): drop stale path sidecars when a shard is no longer listing-safe Rewriting a shard with a newline-bearing path must unlink the old listing so load does not skip the JSON payload. Co-authored-by: Cursor * fix(parse): keep worker-integration tests aligned with hash buckets Quarantine cache-skip asserts the poison pack hash, clone-skip keeps poison and survivors in one bucket, and restore unlinks a stale dest sidecar when the durable source has none. Co-authored-by: Cursor * fix(parse): address review follow-ups for cache packs and sidecars Record SCHEMA_BUMP 80, pin pack locality and sidecar load/restore tests, and keep sidecar I/O best-effort with shared ENOENT handling. Co-authored-by: Cursor * fix(store): drop stale path sidecars after a failed listing write A leftover .paths file after ENOSPC (or similar) made load skip the new JSON shard. Hash expected packs with the same env budget production uses. Co-authored-by: Cursor * fix(store): drop path sidecars before overwriting parsed-file JSON Load trusts a leftover .paths listing, so rewriting a shard must unlink that listing first. Otherwise an interrupted sidecar refresh can hide newly written files. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command * fix(store): fail closed on truncated or CR path sidecars Count-prefix listings so a newline-terminated partial sidecar cannot skip the JSON shard, and reject CR instead of stripping it. Co-authored-by: Cursor * fix(ci): expect single-file watch refresh telemetry The production analyze --watch e2e was still pinned to the old pack-cascade "8 re-parsed" line, so shard 1/3 timed out after a correct 1-file refresh. Co-authored-by: Cursor * fix(ci): expect one reparsed file on a non-bean incremental touch Pack-cascade leftover: the drift-skip test still required 7 reparsed files after logger.ts-only edits. Cheap ParsedFile loads now reparse just that file. Co-authored-by: Cursor --------- Co-authored-by: Gergo Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- README.md | 2 +- .../ingestion/pipeline-phases/parse-impl.ts | 99 ++----- gitnexus/src/storage/parse-cache.ts | 81 ++++- gitnexus/src/storage/parsedfile-store.ts | 279 +++++++++++++++--- gitnexus/test/integration/cli-e2e.test.ts | 9 +- .../integration/parse-impl-clone-skip.test.ts | 29 +- .../integration/parse-impl-env-reads.test.ts | 39 ++- .../parse-impl-quarantine-cache-skip.test.ts | 72 +++-- .../unit/incremental-orchestration.test.ts | 2 +- .../test/unit/incremental-parse-cache.test.ts | 64 +++- gitnexus/test/unit/parsedfile-store.test.ts | 268 ++++++++++++++++- 11 files changed, 777 insertions(+), 167 deletions(-) diff --git a/README.md b/README.md index 173cd1d42..405d85201 100644 --- a/README.md +++ b/README.md @@ -546,7 +546,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | | `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code. The worker is terminated at its next JS-safe point instead of mid-native-call (which aborts the whole process with `Napi::Error`, #2432); on expiry it is left running, unref'd, and terminated when it surfaces. | Shutdown latency matters more than draining a wedged worker (lower), or a legitimately-slow native grammar needs longer to surface (raise). | | `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction. On breach the file keeps the captures accumulated so far and logs a warning — the worker returns to JS instead of stalling in native-heavy loops (#2432). `0` expires immediately. | Pathological generated C++ that still exceeds the budget after the indexed lookups; raise for completeness, lower to fail-fast. | -| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | +| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Per-bucket byte budget for parse-cache packing. Files are grouped by `(language, hash(path) mod 128)`; packs inside a bucket are cut at this limit. Smaller = finer-grained invalidation and more dispatch. Default is always 2 MiB and no longer scales with worker count. | Tuning incremental-analyze cache invalidation on monorepos without changing `--workers`. | | `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | | `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | | `GITNEXUS_MCP_READ_ONLY` | unset | Set to `1` to expose only proven single-repository read tools and resources; `0` disables the policy and any other value fails startup. | The MCP server runs in an environment where graph mutation, raw Cypher, and cross-repository group routing must be unavailable. | diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 157e29027..b6ca75ea0 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -2,7 +2,7 @@ * Parse implementation — chunked parse + resolve loop. * * This is the core parsing engine of the ingestion pipeline. It reads - * source files in byte-budget chunks (~20MB each), parses via the worker + * source files in stable hash-bucket packs (~2MB each by default), parses via the worker * pool (the sole parse path — there is no sequential fallback), and emits * route CALLS edges. Import, * call, and inheritance resolution are owned by the scope-resolution @@ -27,6 +27,7 @@ import { loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, + packParseCacheChunks, } from '../../../storage/parse-cache.js'; import { clearParsedFileStore, @@ -192,39 +193,19 @@ export function heapPressureRemedy(heapLimitBytes: number): string { ); } -/** Max bytes of source content to load per parse chunk. +/** Max bytes of source content to load per parse cache pack. * - * Memory bound for the worker pool dispatch + a granularity knob for - * the parse cache. A single file change invalidates only its enclosing - * chunk, so smaller budgets → finer-grained invalidation. - * - * Override via GITNEXUS_CHUNK_BYTE_BUDGET (bytes) — the default of 2MB - * gives a useful invalidation floor (~1/N chunks on a multi-MB repo) - * while keeping worker dispatch overhead under 5% on cold runs. - */ -/** - * Built-in chunk byte budget when neither `PipelineOptions.chunkByteBudget` - * nor `GITNEXUS_CHUNK_BYTE_BUDGET` is set. Tuned to give a useful - * cache-invalidation floor (~1/N chunks on a multi-MB repo) while keeping - * worker dispatch overhead under 5% on cold runs. Resolution happens at - * call time inside `runChunkedParseAndResolve` (U14 from PR #1693 review) - * — previously this was a module-load IIFE, which froze the env value at - * import time and meant per-call option threading silently no-op'd. + * Granularity knob for the parse cache: a single file change invalidates only + * its enclosing pack. Override via GITNEXUS_CHUNK_BYTE_BUDGET. Resolution + * happens at call time (U14 from PR #1693) — not at module load. */ const DEFAULT_CHUNK_BYTE_BUDGET = 2 * 1024 * 1024; /** - * Per-worker share of a chunk's byte budget when auto-scaling (#worker-idle). - * - * A chunk is a single `WorkerPool.dispatch` unit; the pool fans a chunk's files - * into sub-batch jobs and assigns them to idle workers (`wakeIdleSlots`). When - * the chunk budget (2 MB) was far below the 8 MB sub-batch cap, every chunk - * produced exactly ONE job → ONE busy worker while the other N-1 sat idle. To - * keep all workers fed, the auto chunk budget now scales as - * `poolSize × CHUNK_BYTES_PER_WORKER`, so each dispatch carries enough work to - * fan across the whole pool. Sequential / explicit-budget runs are unaffected. + * Byte unit for auto pool sizing (one worker per this much source). Same + * magnitude as the default cache pack, but not a membership input (#3088). */ -const CHUNK_BYTES_PER_WORKER = 2 * 1024 * 1024; +const CHUNK_BYTES_PER_WORKER = DEFAULT_CHUNK_BYTE_BUDGET; /** * Target jobs-per-worker per dispatch. More jobs than workers gives the pool's @@ -236,14 +217,12 @@ const TARGET_JOBS_PER_WORKER = 3; /** Floor for a derived sub-batch so jobs don't shrink to per-file IPC churn. */ const MIN_SUB_BATCH_BYTES = 256 * 1024; -function resolveChunkByteBudget(options?: PipelineOptions, effectivePoolSize = 1): number { +function resolveChunkByteBudget(options?: PipelineOptions): number { const opt = options?.chunkByteBudget; if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0) return opt; const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET); if (Number.isFinite(env) && env > 0) return env; - // Auto: size each chunk so a dispatch can fan across the whole pool. A - // single-worker (tiny-repo) run keeps the original 2 MB invalidation floor. - return Math.max(DEFAULT_CHUNK_BYTE_BUDGET, effectivePoolSize * CHUNK_BYTES_PER_WORKER); + return DEFAULT_CHUNK_BYTE_BUDGET; } // ── Main parse + resolve function ────────────────────────────────────────── @@ -524,15 +503,6 @@ export async function runChunkedParseAndResolve( 0, ); - // Sort parseableScanned alphabetically for stable chunk membership - // across runs (Finding 4). Without this, filesystem-scan order can - // shift between runs (notably on macOS APFS where directory entry - // order can change after modifications) — different files in the - // same chunk → different chunk hash → cache miss even when no file - // content changed. The cache also becomes platform-specific: a - // Linux-built cache misses on macOS for the same repo. - parseableScanned.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); - const totalParseable = parseableScanned.length; const totalBytes = parseableScanned.reduce((sum, f) => sum + f.size, 0); @@ -579,25 +549,25 @@ export async function runChunkedParseAndResolve( // runs. Resolving in the function body restores per-call configurability // and matches the pattern used by resolveAutoPoolSize and the U1 // parseChunkConcurrency resolver. - // Effective worker count, computed up-front so the chunk budget can scale to - // keep the whole pool busy (#worker-idle). The pool is ALWAYS used (sequential - // parsing was removed; the disabled channels threw above). Size it to the - // work: an explicit `--workers ` pins the size; otherwise the cores-based - // auto size is capped by the repo's worth of work (~one worker per - // CHUNK_BYTES_PER_WORKER of source) so a tiny repo spawns ~1 worker instead of - // a full pool, replacing the job the deleted small-repo threshold used to do. - // KTD-3 of the remove-sequential plan; the cap formula is intentionally coarse - // (tuning deferred). + // Effective worker count: explicit `--workers ` pins it; otherwise + // cores-based auto size is capped by source bytes / CHUNK_BYTES_PER_WORKER + // so a tiny repo does not spawn a full idle pool. Cache pack membership + // is independent of this number (#3088). const explicitPoolSize = options?.workerPoolSize; const workProportionalCap = Math.max(1, Math.ceil(totalBytes / CHUNK_BYTES_PER_WORKER)); const effectivePoolSize = explicitPoolSize && explicitPoolSize > 0 ? explicitPoolSize : Math.min(resolveAutoPoolSize(), workProportionalCap); - const chunkByteBudget = resolveChunkByteBudget(options, effectivePoolSize); - // Sub-batch size so each chunk fans into ~`TARGET_JOBS_PER_WORKER` jobs per - // worker, giving the pool's idle-slot assignment room to load-balance. An - // explicit `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` operator override wins. + // Cache packs: stable (language, hash(path) mod 128) buckets, then the + // per-call byte budget inside each bucket (#3088). Pool size is used only + // for worker count and sub-batch fan-out, not membership. + const chunkByteBudget = resolveChunkByteBudget(options); + // Sub-batch size so a 2 MiB pack fans into ~TARGET_JOBS_PER_WORKER jobs + // per worker, floored at MIN_SUB_BATCH_BYTES (256 KiB) so an 8-worker + // pool still gets ~8 jobs from one pack instead of one idle-heavy job + // (#worker-idle). Do not derive this from pool×2 MiB while dispatching a + // 2 MiB pack. An explicit GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES wins. const subBatchEnv = Number(process.env.GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES); const dispatchSubBatchMaxBytes = Number.isFinite(subBatchEnv) && subBatchEnv > 0 @@ -622,19 +592,14 @@ export async function runChunkedParseAndResolve( ); } - const chunks: string[][] = []; - let currentChunk: string[] = []; - let currentBytes = 0; - for (const file of parseableScanned) { - if (currentChunk.length > 0 && currentBytes + file.size > chunkByteBudget) { - chunks.push(currentChunk); - currentChunk = []; - currentBytes = 0; - } - currentChunk.push(file.path); - currentBytes += file.size; - } - if (currentChunk.length > 0) chunks.push(currentChunk); + const chunks: string[][] = packParseCacheChunks( + parseableScanned.map((file) => ({ + path: file.path, + size: file.size, + language: getLanguageFromFilename(file.path) ?? 'unknown', + })), + chunkByteBudget, + ); const numChunks = chunks.length; diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index bcbb4c189..c99f4d87e 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -6,10 +6,12 @@ * does is skip the tree-sitter worker dispatch when a chunk's contents * haven't changed since the last run. * - * Granularity: chunk-level. The parse phase chunks files into ~20MB byte - * budgets. The cache key is `sha256(joined(filePath:contentHash for each - * file in the chunk, sorted))`. A change to a single file invalidates only - * that file's chunk — typically 1 of ~50 chunks on a 1000-file repo. + * Granularity: chunk-level. Files are assigned to a stable + * `(language, hash(path) mod 128)` bucket, then packed to a 2 MiB (or + * operator) byte budget *inside* that bucket. Membership does not depend + * on worker count. The cache key is `sha256(joined(filePath:contentHash + * for each file in the chunk, sorted))`. A content edit invalidates only + * that file's pack; add/delete/rename only the affected bucket. * * Why not per-file: * - Workers process sub-batches and emit aggregated `ParseWorkerResult`s. @@ -27,6 +29,7 @@ import { createRequire } from 'module'; import fs from 'fs/promises'; import path from 'path'; import { fileURLToPath } from 'url'; +import { compareCodeUnits } from '../lib/utils.js'; import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.js'; /** @@ -632,6 +635,16 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // therefore takes 79, the next free value above origin/main and every open PR // found by the contents-API scan at their exact head SHAs. // +// 79 -> 80 for #3088: parse-cache membership is `(language, sha256(path) mod +// 128)` then the byte budget *inside* that bucket. Worker count is no longer a +// membership input, so a warm v79 cache keyed sequential scan-order packs (and +// on multi-worker hosts, pool×2 MiB mega-chunks) must miss. Sidecar-era +// ParsedFile stores (#3086/#3087) share PARSE_CACHE_VERSION, so both stores +// invalidate in lockstep. origin/main at allocation is 79; open PRs that still +// touch gitnexus/src/storage/parse-cache.ts claim 78 (#3060), 71 (#2840), and +// 2 (#1616) — none claim 80. RE-CHECK AGAINST origin/main AND OPEN PRs +// IMMEDIATELY BEFORE MERGING. +// // WHY THIS IS STILL A HAND-PICKED NUMBER, when `SCHEMA_FINGERPRINT` next door // is a derived sha256 that cannot collide. The derivation exists and already // runs: `resolveAnalyzerRunnerIdentity` computes `build.digest` over the @@ -650,7 +663,7 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // `route-extractors/` and `workers/` module content — would close the missing- // bump axis without invalidating on unrelated churn, and is the real follow-up. // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING. -const SCHEMA_BUMP = 79; +const SCHEMA_BUMP = 80; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from @@ -676,6 +689,58 @@ const GITNEXUS_PKG_VERSION = (() => { })(); export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`; +/** SHA-256 hex of a string or buffer (paths for bucket ids, contents for cache keys). */ +const sha256Hex = (input: Buffer | string): string => + createHash('sha256') + .update(typeof input === 'string' ? Buffer.from(input) : input) + .digest('hex'); + +/** Stable parse-cache bucket count (#3088). Changing this requires SCHEMA_BUMP. */ +export const PARSE_CACHE_BUCKET_COUNT = 128; + +/** Bucket id for cache membership: `sha256(path) mod N` without IEEE-754 truncation. */ +export const parseCacheBucketId = (filePath: string): number => + Number(BigInt(`0x${sha256Hex(filePath)}`) % BigInt(PARSE_CACHE_BUCKET_COUNT)); + +export type ParseCachePackFile = { path: string; size: number; language: string }; + +/** + * Pack files into parse-cache chunks: group by (language, bucket id), sort + * paths inside the group, then cut at `byteBudget`. Bucket visit order is + * the lexicographic order of `${language}\\0${bucketId}` keys (deterministic, + * independent of scan order and worker count). + */ +export const packParseCacheChunks = ( + files: readonly ParseCachePackFile[], + byteBudget: number, +): string[][] => { + const buckets = new Map(); + for (const file of files) { + const key = `${file.language}\0${parseCacheBucketId(file.path)}`; + const list = buckets.get(key); + if (list) list.push(file); + else buckets.set(key, [file]); + } + const chunks: string[][] = []; + for (const key of [...buckets.keys()].sort()) { + const group = buckets.get(key)!; + group.sort((a, b) => compareCodeUnits(a.path, b.path)); + let current: string[] = []; + let bytes = 0; + for (const file of group) { + if (current.length > 0 && bytes + file.size > byteBudget) { + chunks.push(current); + current = []; + bytes = 0; + } + current.push(file.path); + bytes += file.size; + } + if (current.length > 0) chunks.push(current); + } + return chunks; +}; + const LEGACY_CACHE_FILENAME = 'parse-cache.json'; const CACHE_DIRNAME = 'parse-cache'; const CACHE_INDEX_FILENAME = 'index.json'; @@ -720,12 +785,6 @@ export interface ParseCache { onDiskKeys?: Set; } -/** SHA-256 hex of a single string or buffer. */ -const sha256Hex = (input: Buffer | string): string => - createHash('sha256') - .update(typeof input === 'string' ? Buffer.from(input) : input) - .digest('hex'); - /** Stable hash of a single file's contents — used by callers to compose a chunk hash. */ export const fileContentHash = (content: Buffer | string): string => sha256Hex(content); diff --git a/gitnexus/src/storage/parsedfile-store.ts b/gitnexus/src/storage/parsedfile-store.ts index c9a6e0482..1f34ff083 100644 --- a/gitnexus/src/storage/parsedfile-store.ts +++ b/gitnexus/src/storage/parsedfile-store.ts @@ -48,7 +48,7 @@ * file changes its chunk hash, which misses BOTH stores and re-dispatches. */ -import { promises as fs, mkdirSync, writeFileSync } from 'node:fs'; +import { promises as fs, mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; import path from 'node:path'; import v8 from 'node:v8'; import vm from 'node:vm'; @@ -180,6 +180,140 @@ const serializeParsedFileShard = (parsedFiles: readonly ParsedFile[]): string | const shardPath = (storagePath: string, shardId: string): string => path.join(getParsedFileStoreDir(storagePath), `${shardId}.json`); +/** Sidecar listing `filePath`s in a shard; not matched by `endsWith('.json')`. */ +const shardPathsSidecarPath = (jsonPath: string): string => `${jsonPath}.paths`; + +const LOAD_YIELD_EVERY_SHARDS = 128; + +/** + * Test seam for #3086. Production always calls {@link forceGc}; unit tests + * replace `run` to count cadence without requiring `--expose-gc`. + */ +export const parsedFileLoadGc = { + run: forceGc, + /** Raw UTF-8 JSON shard bytes between GCs (#3086). Tests may lower this. */ + byteBudget: 128 * 1024 * 1024, +}; + +const encodeShardPathsSidecar = (parsedFiles: readonly ParsedFile[]): string => { + const paths = parsedFiles.map((pf) => pf.filePath); + return `${paths.length}\n${paths.length === 0 ? '' : `${paths.join('\n')}\n`}`; +}; + +/** + * Parse a counted NDJSON path listing. Returns `null` when the sidecar must + * not be trusted to skip the JSON shard: missing trailing newline, CR/NUL, + * a truncated listing that still ends on a complete line, or a count that + * does not match the remaining lines. + */ +const parseShardPathsSidecar = (sidecarRaw: string): string[] | null => { + if (sidecarRaw.includes('\0') || sidecarRaw.includes('\r') || !sidecarRaw.endsWith('\n')) { + return null; + } + const nl = sidecarRaw.indexOf('\n'); + if (nl < 0) return null; + const countToken = sidecarRaw.slice(0, nl); + if (!/^[0-9]+$/.test(countToken)) return null; + const count = Number(countToken); + const body = sidecarRaw.slice(nl + 1); + const listed = body === '' ? [] : body.slice(0, -1).split('\n'); + if (listed.length !== count) return null; + return listed; +}; + +/** NDJSON sidecars cannot encode paths that themselves contain CR/LF/NUL. */ +const shardPathsSidecarSafe = (parsedFiles: readonly ParsedFile[]): boolean => + parsedFiles.every((pf) => !/[\r\n\0]/.test(pf.filePath)); + +const isEnoent = (err: unknown): boolean => (err as NodeJS.ErrnoException).code === 'ENOENT'; + +const warnSidecarIo = (err: unknown, jsonPath: string, msg: string): void => { + logger.warn({ err, jsonPath }, msg); +}; + +const ignoreMissingSidecarUnlink = (err: unknown, jsonPath: string): void => { + if (isEnoent(err)) return; + warnSidecarIo( + err, + jsonPath, + 'parsedfile-store: failed to drop path sidecar; JSON remains authoritative', + ); +}; + +/** Drop a leftover listing before publishing JSON so load cannot skip new paths. */ +const dropPathSidecar = async (jsonPath: string): Promise => { + try { + await fs.unlink(shardPathsSidecarPath(jsonPath)); + } catch (err) { + ignoreMissingSidecarUnlink(err, jsonPath); + } +}; + +const dropPathSidecarSync = (jsonPath: string): void => { + try { + unlinkSync(shardPathsSidecarPath(jsonPath)); + } catch (err) { + ignoreMissingSidecarUnlink(err, jsonPath); + } +}; + +const writeShardPathsSidecar = async ( + jsonPath: string, + parsedFiles: readonly ParsedFile[], +): Promise => { + if (!shardPathsSidecarSafe(parsedFiles)) { + try { + await fs.unlink(shardPathsSidecarPath(jsonPath)); + } catch (err) { + ignoreMissingSidecarUnlink(err, jsonPath); + } + return; + } + try { + await fs.writeFile( + shardPathsSidecarPath(jsonPath), + encodeShardPathsSidecar(parsedFiles), + 'utf-8', + ); + } catch (err) { + warnSidecarIo( + err, + jsonPath, + 'parsedfile-store: path sidecar write failed; JSON shard remains authoritative', + ); + try { + await fs.unlink(shardPathsSidecarPath(jsonPath)); + } catch (unlinkErr) { + ignoreMissingSidecarUnlink(unlinkErr, jsonPath); + } + } +}; + +const writeShardPathsSidecarSync = (jsonPath: string, parsedFiles: readonly ParsedFile[]): void => { + if (!shardPathsSidecarSafe(parsedFiles)) { + try { + unlinkSync(shardPathsSidecarPath(jsonPath)); + } catch (err) { + ignoreMissingSidecarUnlink(err, jsonPath); + } + return; + } + try { + writeFileSync(shardPathsSidecarPath(jsonPath), encodeShardPathsSidecar(parsedFiles), 'utf-8'); + } catch (err) { + warnSidecarIo( + err, + jsonPath, + 'parsedfile-store: path sidecar write failed; JSON shard remains authoritative', + ); + try { + unlinkSync(shardPathsSidecarPath(jsonPath)); + } catch (unlinkErr) { + ignoreMissingSidecarUnlink(unlinkErr, jsonPath); + } + } +}; + /** * Write one parse chunk's `ParsedFile[]` to the store as a single shard (async). * No-op for an empty chunk. `shardId` must be unique within a run. Used by the @@ -194,7 +328,10 @@ export const persistParsedFileChunk = async ( const payload = serializeParsedFileShard(parsedFiles); if (payload === null) return; await fs.mkdir(getParsedFileStoreDir(storagePath), { recursive: true }); - await fs.writeFile(shardPath(storagePath, shardId), payload, 'utf-8'); + const dest = shardPath(storagePath, shardId); + await dropPathSidecar(dest); + await fs.writeFile(dest, payload, 'utf-8'); + await writeShardPathsSidecar(dest, parsedFiles); }; // Per-process set of store dirs we've already `mkdir`ed, so the sync worker @@ -223,7 +360,10 @@ export const persistParsedFileShardSync = ( mkdirSync(dir, { recursive: true }); createdStoreDirs.add(dir); } - writeFileSync(shardPath(storagePath, shardId), payload, 'utf-8'); + const dest = shardPath(storagePath, shardId); + dropPathSidecarSync(dest); + writeFileSync(dest, payload, 'utf-8'); + writeShardPathsSidecarSync(dest, parsedFiles); }; /** @@ -255,54 +395,90 @@ export const loadParsedFilesForPaths = async ( let filesWithDroppedSites = 0; let droppedChains = 0; let rejectedFiles = 0; + let bytesSinceGc = 0; + let shardsSinceYield = 0; + const maybeYieldAndGc = async (forceByteGc: boolean): Promise => { + if (forceByteGc) { + parsedFileLoadGc.run(); + bytesSinceGc = 0; + shardsSinceYield = 0; + await new Promise((resolve) => setImmediate(resolve)); + return; + } + shardsSinceYield++; + if (shardsSinceYield >= LOAD_YIELD_EVERY_SHARDS) { + shardsSinceYield = 0; + await new Promise((resolve) => setImmediate(resolve)); + } + }; for (let i = 0; i < shards.length; i++) { + const jsonName = shards[i]; + const jsonFull = path.join(dir, jsonName); + try { + const sidecarRaw = await fs.readFile(shardPathsSidecarPath(jsonFull), 'utf-8'); + // Fail closed: complete writers emit `\n` plus one path per line + // and a trailing newline, never CR. Stripping CR (or accepting a + // newline-terminated prefix) would let a truncated listing skip JSON. + const listed = parseShardPathsSidecar(sidecarRaw); + if (listed === null) { + throw new Error('corrupt sidecar'); + } + if (listed.length > 0 && !listed.some((p) => wantPaths.has(p))) { + await maybeYieldAndGc(false); + continue; + } + } catch { + // Missing or unreadable sidecar → read the shard (pre-sidecar stores). + } // Per-shard def pool: a SymbolDefinition's three serialized copies live within // a single shard (one ParsedFile), so the dedup is shard-local. A cross-shard // pool would retain defs of files NOT in `wantPaths` (loaded-but-discarded // shards), reintroducing the leak; per-shard drops them with the shard. const defPool = new Map(); const reviver = makeInterningReviver(pool, defPool); - let parsed: ParsedFile[]; + let raw: string; + try { + raw = await fs.readFile(jsonFull, 'utf-8'); + } catch { + continue; // skip a missing shard; missing files fall back to fresh extract + } + bytesSinceGc += Buffer.byteLength(raw, 'utf8'); + const crossedBudget = bytesSinceGc >= parsedFileLoadGc.byteBudget; + let parsed: ParsedFile[] | undefined; try { - const raw = await fs.readFile(path.join(dir, shards[i]), 'utf-8'); parsed = JSON.parse(raw, reviver) as ParsedFile[]; } catch { - continue; // skip a corrupt shard; missing files fall back to fresh extract + parsed = undefined; } - if (!Array.isArray(parsed)) continue; - for (const pf of parsed) { - if (!pf || typeof pf.filePath !== 'string' || !wantPaths.has(pf.filePath)) continue; - const flow = sanitizeCallableFlowSites(pf.callableFlowSites); - if (flow === undefined) { - // non-array garbage → distrust the file, re-extract - rejectedFiles++; - continue; - } - const chains = sanitizeReceiverChains(pf.referenceSites); - if (chains === undefined) { - rejectedFiles++; - continue; - } - if (flow.dropped === 0 && chains.dropped === 0) { - out.set(pf.filePath, pf); - } else { - droppedSites += flow.dropped; - droppedChains += chains.dropped; - filesWithDroppedSites++; - out.set(pf.filePath, { - ...pf, - ...(flow.dropped === 0 ? {} : { callableFlowSites: flow.sites }), - ...(chains.dropped === 0 ? {} : { referenceSites: chains.sites }), - }); + if (Array.isArray(parsed)) { + for (const pf of parsed) { + if (!pf || typeof pf.filePath !== 'string' || !wantPaths.has(pf.filePath)) continue; + const flow = sanitizeCallableFlowSites(pf.callableFlowSites); + if (flow === undefined) { + // non-array garbage → distrust the file, re-extract + rejectedFiles++; + continue; + } + const chains = sanitizeReceiverChains(pf.referenceSites); + if (chains === undefined) { + rejectedFiles++; + continue; + } + if (flow.dropped === 0 && chains.dropped === 0) { + out.set(pf.filePath, pf); + } else { + droppedSites += flow.dropped; + droppedChains += chains.dropped; + filesWithDroppedSites++; + out.set(pf.filePath, { + ...pf, + ...(flow.dropped === 0 ? {} : { callableFlowSites: flow.sites }), + ...(chains.dropped === 0 ? {} : { referenceSites: chains.sites }), + }); + } } } - // Every few shards, reclaim the transient pre-intern parse churn before it - // piles up against the heap limit (~5 GB avoidable on the kernel), and - // yield so the GC + any pending I/O can run. - if ((i & 7) === 7) { - forceGc(); - await new Promise((resolve) => setImmediate(resolve)); - } + await maybeYieldAndGc(crossedBudget); } if (droppedSites > 0 || droppedChains > 0) { // Facts for the dropped sites are omitted this run (the file itself is @@ -595,7 +771,10 @@ export const persistDurableParsedFileShardSync = ( mkdirSync(dir, { recursive: true }); createdDurableDirs.add(dir); } - writeFileSync(path.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.json`), payload, 'utf-8'); + const dest = path.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.json`); + dropPathSidecarSync(dest); + writeFileSync(dest, payload, 'utf-8'); + writeShardPathsSidecarSync(dest, parsedFiles); }; /** @@ -624,7 +803,27 @@ export const restoreDurableParsedFileShard = async ( const dst = getParsedFileStoreDir(runStoragePath); await fs.mkdir(dst, { recursive: true }); for (const name of shards) { - await fs.copyFile(path.join(src, name), path.join(dst, name)); + const srcJson = path.join(src, name); + const dstJson = path.join(dst, name); + await dropPathSidecar(dstJson); + await fs.copyFile(srcJson, dstJson); + try { + await fs.copyFile(shardPathsSidecarPath(srcJson), shardPathsSidecarPath(dstJson)); + } catch (copyErr) { + if (!isEnoent(copyErr)) { + warnSidecarIo( + copyErr, + srcJson, + 'parsedfile-store: durable path sidecar copy failed; JSON remains authoritative', + ); + continue; + } + try { + await fs.unlink(shardPathsSidecarPath(dstJson)); + } catch (err) { + ignoreMissingSidecarUnlink(err, dstJson); + } + } } return shards.length; }; diff --git a/gitnexus/test/integration/cli-e2e.test.ts b/gitnexus/test/integration/cli-e2e.test.ts index ffff7dcfd..9998a1425 100644 --- a/gitnexus/test/integration/cli-e2e.test.ts +++ b/gitnexus/test/integration/cli-e2e.test.ts @@ -1126,7 +1126,10 @@ describe('CLI end-to-end', () => { }); return; } - if (stage === 'proof' && /Refresh complete: 1 changed, 8 re-parsed,/.test(output)) { + if ( + stage === 'proof' && + /Refresh complete: 1 changed, 1 re-parsed, 0 affected dependent\(s\)/.test(output) + ) { const meta = JSON.parse( fs.readFileSync(path.join(repo, '.gitnexus', 'gitnexus.json'), 'utf8'), ); @@ -1214,7 +1217,9 @@ describe('CLI end-to-end', () => { return; } expect(stage).toBe('stopping'); - expect(transcript).toContain('Refresh complete: 1 changed, 8 re-parsed,'); + expect(transcript).toContain( + 'Refresh complete: 1 changed, 1 re-parsed, 0 affected dependent(s)', + ); resolve(); }); }); diff --git a/gitnexus/test/integration/parse-impl-clone-skip.test.ts b/gitnexus/test/integration/parse-impl-clone-skip.test.ts index 1fd155cf0..f34c8c3fd 100644 --- a/gitnexus/test/integration/parse-impl-clone-skip.test.ts +++ b/gitnexus/test/integration/parse-impl-clone-skip.test.ts @@ -32,6 +32,7 @@ import { pathToFileURL } from 'node:url'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; import { _captureLogger } from '../../src/core/logger.js'; +import { parseCacheBucketId } from '../../src/storage/parse-cache.js'; // file:// URL of the BUILT production result-delivery helper, imported by the // ESM test worker so it exercises the REAL postResultCloneSafe wiring (the @@ -140,10 +141,17 @@ parentPort.on('message', (msg) => { }); `; -const FIXTURE_FILES = { - 'src/good_a.ts': 'export function good_a() { return 1; }\n', - 'src/poison.ts': 'export function poison() { return 2; }\n', - 'src/good_c.ts': 'export function good_c() { return 3; }\n', +const POISON_PATH = 'src/poison.ts'; +/** Pinned same-bucket fixtures (sha256(path) mod 128 of poison.ts). */ +const GOOD_A_PATH = 'src/good_a_16.ts'; +const GOOD_C_PATH = 'src/good_c_51.ts'; +const GOOD_A_NAME = path.basename(GOOD_A_PATH, '.ts'); +const GOOD_C_NAME = path.basename(GOOD_C_PATH, '.ts'); + +const FIXTURE_FILES: Record = { + [GOOD_A_PATH]: 'export function good_a() { return 1; }\n', + [POISON_PATH]: 'export function poison() { return 2; }\n', + [GOOD_C_PATH]: 'export function good_c() { return 3; }\n', }; const nodeNames = (graph: ReturnType): Set => { @@ -165,6 +173,11 @@ const nodeNames = (graph: ReturnType): Set const STRICT = process.env.GITNEXUS_STRICT_CLONE === '1'; describe.skipIf(STRICT)('#2112: worker result clone-safety integration (POOL_SIZE=1)', () => { + it('pins survivors into the same parse-cache bucket as poison.ts', () => { + expect(parseCacheBucketId(GOOD_A_PATH)).toBe(parseCacheBucketId(POISON_PATH)); + expect(parseCacheBucketId(GOOD_C_PATH)).toBe(parseCacheBucketId(POISON_PATH)); + }); + let tempDir: string; let repoDir: string; @@ -228,8 +241,8 @@ describe.skipIf(STRICT)('#2112: worker result clone-safety integration (POOL_SIZ const graph = await runWith(writeWorker(CLONE_SAFE_WORKER)); const names = nodeNames(graph); // Survivors AND the sanitized poison file are all present — the run did not abort. - expect(names.has('good_a')).toBe(true); - expect(names.has('good_c')).toBe(true); + expect(names.has(GOOD_A_NAME)).toBe(true); + expect(names.has(GOOD_C_NAME)).toBe(true); // The poison node is delivered with its legitimate data intact (only the // leaked native `toString` was stripped), so it still lands in the graph. expect(names.has('poison')).toBe(true); @@ -256,8 +269,8 @@ describe.skipIf(STRICT)('#2112: worker result clone-safety integration (POOL_SIZ // rejects; with it, all files (incl. the sanitized poison node) are present. const graph = await runWith(writeWorker(GETTER_WORKER)); const names = nodeNames(graph); - expect(names.has('good_a')).toBe(true); - expect(names.has('good_c')).toBe(true); + expect(names.has(GOOD_A_NAME)).toBe(true); + expect(names.has(GOOD_C_NAME)).toBe(true); expect(names.has('poison')).toBe(true); }); diff --git a/gitnexus/test/integration/parse-impl-env-reads.test.ts b/gitnexus/test/integration/parse-impl-env-reads.test.ts index 1ccdffb7b..2d93e12dd 100644 --- a/gitnexus/test/integration/parse-impl-env-reads.test.ts +++ b/gitnexus/test/integration/parse-impl-env-reads.test.ts @@ -28,6 +28,7 @@ import path from 'node:path'; import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { PARSE_CACHE_VERSION, parseCacheBucketId } from '../../src/storage/parse-cache.js'; const ORIGINAL_BUDGET = process.env.GITNEXUS_CHUNK_BYTE_BUDGET; @@ -123,12 +124,12 @@ describe('parse-impl chunkByteBudget resolution (U14 / F7)', () => { expect(chunks).toBe(3); }); - it('default-fallback: large built-in budget keeps the fixture in a single chunk', async () => { - // Both option and env unset → falls through to DEFAULT_CHUNK_BYTE_BUDGET - // (2 MB). The fixture totals well under that, so exactly one chunk. + it('default-fallback: 2 MB budget packs by bucket, not one sequential mega-chunk', async () => { delete process.env.GITNEXUS_CHUNK_BYTE_BUDGET; - const chunks = await countChunksFromProgress(repoPath, ['a.ts', 'b.ts', 'c.ts']); - expect(chunks).toBe(1); + const files = ['a.ts', 'b.ts', 'c.ts']; + const expectedBuckets = new Set(files.map((f) => `typescript\0${parseCacheBucketId(f)}`)); + const chunks = await countChunksFromProgress(repoPath, files); + expect(chunks).toBe(expectedBuckets.size); }); it('per-call: two back-to-back runs with different option values observe their own values, not the previous call', async () => { @@ -146,6 +147,32 @@ describe('parse-impl chunkByteBudget resolution (U14 / F7)', () => { chunkByteBudget: 10 * 1024 * 1024, }); expect(small).toBe(3); - expect(large).toBe(1); + const expectedBuckets = new Set(files.map((f) => `typescript\0${parseCacheBucketId(f)}`)); + expect(large).toBe(expectedBuckets.size); + }); + + it('workerPoolSize 1 vs 2 produce the same cache keys when budget is unset (#3088)', async () => { + delete process.env.GITNEXUS_CHUNK_BYTE_BUDGET; + const files = ['a.ts', 'b.ts', 'c.ts']; + const keysForPool = async (workerPoolSize: number): Promise => { + const parseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set(), + }; + const graph = createKnowledgeGraph(); + await runChunkedParseAndResolve( + graph, + scanned(repoPath, files), + files, + files.length, + repoPath, + Date.now(), + () => {}, + { workerPoolSize, parseCache }, + ); + return [...parseCache.usedKeys].sort(); + }; + expect(await keysForPool(1)).toEqual(await keysForPool(2)); }); }); diff --git a/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts b/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts index 0bb8dbec6..949f0d75e 100644 --- a/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts +++ b/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts @@ -73,7 +73,11 @@ import { pathToFileURL } from 'node:url'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; -import { computeChunkHash, fileContentHash } from '../../src/storage/parse-cache.js'; +import { + computeChunkHash, + fileContentHash, + packParseCacheChunks, +} from '../../src/storage/parse-cache.js'; import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; /** @@ -180,6 +184,41 @@ const FIXTURE_FILES = { 'src/good_c.ts': 'export function good_c() { return 3; }\n', }; +const POISON_PATH = 'src/poison.ts'; +const DEFAULT_TEST_CHUNK_BUDGET = 2 * 1024 * 1024; + +const resolveTestChunkByteBudget = (): number => { + const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET); + if (Number.isFinite(env) && env > 0) return env; + return DEFAULT_TEST_CHUNK_BUDGET; +}; + +const hashPacks = ( + scanned: { path: string; size: number }[], +): { poison: string; others: string[] } => { + const packs = packParseCacheChunks( + scanned.map((file) => ({ + path: file.path, + size: file.size, + language: 'typescript', + })), + resolveTestChunkByteBudget(), + ); + const hashOf = (pack: string[]) => + computeChunkHash( + pack.map((p) => ({ + filePath: p, + contentHash: fileContentHash(FIXTURE_FILES[p as keyof typeof FIXTURE_FILES]), + })), + ); + const poisonPack = packs.find((paths) => paths.includes(POISON_PATH)); + if (!poisonPack) throw new Error('poison.ts was not packed'); + return { + poison: hashOf(poisonPack), + others: packs.filter((paths) => !paths.includes(POISON_PATH)).map(hashOf), + }; +}; + describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex finding)', () => { let tempDir: string; let repoDir: string; @@ -216,16 +255,7 @@ describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex f size: statSync(path.join(repoDir, rel)).size, })); - // The chunk hash is computed from EVERY file's content hash. The - // load-bearing U2 assertion below checks `parseCache.entries.has` - // against this exact value, so we compute it the same way - // parse-impl does. - const expectedChunkHash = computeChunkHash( - filePaths.map((p) => ({ - filePath: p, - contentHash: fileContentHash(FIXTURE_FILES[p as keyof typeof FIXTURE_FILES]), - })), - ); + const expectedChunkHash = hashPacks(scanned).poison; const parseCache = { version: 'test', @@ -293,23 +323,20 @@ describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex f // against a fresh-quarantine pool. expect(parseCache.entries.has(expectedChunkHash)).toBe(false); expect(parseCache.usedKeys.has(expectedChunkHash)).toBe(true); - expect(parseCache.entries.size).toBe(0); + for (const hash of hashPacks(scanned).others) { + expect(parseCache.entries.has(hash)).toBe(true); + } }); - it('cross-run: unchanged fixture re-dispatches on a second pass because the cache was empty', async () => { - // First pass: same setup as the previous test. Cache stays empty - // because poison.ts triggered quarantine. + it('cross-run: unchanged fixture re-dispatches the poison pack because that pack was not cached', async () => { + // First pass: same setup as the previous test. The poison pack is not + // cached; other packs may be. const filePaths = Object.keys(FIXTURE_FILES); const scanned = filePaths.map((rel) => ({ path: rel, size: statSync(path.join(repoDir, rel)).size, })); - const expectedChunkHash = computeChunkHash( - filePaths.map((p) => ({ - filePath: p, - contentHash: fileContentHash(FIXTURE_FILES[p as keyof typeof FIXTURE_FILES]), - })), - ); + const expectedChunkHash = hashPacks(scanned).poison; const parseCache = { version: 'test', @@ -369,6 +396,9 @@ describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex f // load-bearing cross-run protection. expect(parseCache.entries.has(expectedChunkHash)).toBe(false); expect(parseCache.usedKeys.has(expectedChunkHash)).toBe(true); + for (const hash of hashPacks(scanned).others) { + expect(parseCache.entries.has(hash)).toBe(true); + } // Worker path ran again; surviving files in the graph; poison // still absent per the U20 contract (workers are the sole // resilience layer, no sequential reparse). diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index eb792b701..b0c44917b 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -848,7 +848,7 @@ describe('runFullAnalysis — incremental orchestration', () => { deletedFiles: 0, writeMode: 'incremental', }); - expect(incremental.incrementalStats?.reparsedFiles).toBe(7); + expect(incremental.incrementalStats?.reparsedFiles).toBe(1); expect( querySpy.mock.calls.some( ([query]) => diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index 8097c8ca5..14c4c8359 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -4,8 +4,11 @@ import { tmpdir } from 'os'; import path from 'path'; import { PARSE_CACHE_VERSION, + PARSE_CACHE_BUCKET_COUNT, computeChunkHash, fileContentHash, + packParseCacheChunks, + parseCacheBucketId, loadParseCache, loadParseCacheChunk, persistParseCacheChunk, @@ -240,15 +243,11 @@ describe('PARSE_CACHE_VERSION', () => { // collided, because each re-checked once and neither re-checked after the // other moved — which is why the rule is re-applied AT MERGE, not when the // number is picked. - it('pins SCHEMA_BUMP to 79 so concurrent bumps cannot silently collide (#2766, #3015)', () => { - expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(79); - // The PREVIOUS version must fail the reuse gate, not merely differ from the - // current one — a hardcoded number outside the conflict hunk rebases cleanly - // while being wrong, which is exactly how the 37/38 exact clashes landed. - // Every nearby historical or in-flight value is rejected, including 69, - // which carried the route-table payload before this merge. + it('pins SCHEMA_BUMP to 80 so concurrent bumps cannot silently collide (#2766, #3015, #3088)', () => { + expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(80); + expect(PARSE_CACHE_BUCKET_COUNT).toBe(128); for (const taken of [ - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, ]) { expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).not.toBe(taken); } @@ -260,6 +259,55 @@ describe('PARSE_CACHE_VERSION', () => { }); }); +describe('packParseCacheChunks (#3088)', () => { + const files = [ + { path: 'src/a.ts', size: 100, language: 'typescript' }, + { path: 'src/b.ts', size: 100, language: 'typescript' }, + { path: 'pkg/c.py', size: 100, language: 'python' }, + ]; + const budget = 2 * 1024 * 1024; + const packKey = (chunk: string[]): string => + `${files.find((f) => f.path === chunk[0])?.language ?? 'typescript'}\0${parseCacheBucketId(chunk[0])}`; + + it('is independent of scan order', () => { + expect(packParseCacheChunks(files, budget)).toEqual( + packParseCacheChunks([...files].reverse(), budget), + ); + }); + + it('add/delete only rewrites packs in the affected (language, bucket)', () => { + const a = packParseCacheChunks(files, budget); + const added = { path: 'AAA.ts', size: 150_000, language: 'typescript' }; + const withNew = packParseCacheChunks([...files, added], budget); + const addedKey = packKey([added.path]); + const untouched = (packs: string[][]) => + packs.filter((c) => packKey(c) !== addedKey).map((c) => c.join('|')); + expect(untouched(withNew).sort()).toEqual(untouched(a).sort()); + expect(withNew.some((c) => c.includes(added.path))).toBe(true); + + const withoutB = packParseCacheChunks( + files.filter((f) => f.path !== 'src/b.ts'), + budget, + ); + const removedKey = packKey(['src/b.ts']); + const leftover = (packs: string[][]) => + packs.filter((c) => packKey(c) !== removedKey).map((c) => c.join('|')); + expect(leftover(withoutB).sort()).toEqual(leftover(a).sort()); + expect(withoutB.every((c) => !c.includes('src/b.ts'))).toBe(true); + }); + + it('parseCacheBucketId uses the full sha256 digest, not an IEEE-754 prefix', () => { + const path = 'src/foo.ts'; + const hex = fileContentHash(path); + const full = Number(BigInt(`0x${hex}`) % BigInt(PARSE_CACHE_BUCKET_COUNT)); + const truncated = Number.parseInt(hex.slice(0, 8), 16) % PARSE_CACHE_BUCKET_COUNT; + expect(parseCacheBucketId(path)).toBe(full); + expect(parseCacheBucketId(path)).toBeGreaterThanOrEqual(0); + expect(parseCacheBucketId(path)).toBeLessThan(PARSE_CACHE_BUCKET_COUNT); + expect(full).not.toBe(truncated); + }); +}); + describe('pruneCache', () => { it('drops entries whose hashes are not in the used-set', () => { const cache: ParseCache = { diff --git a/gitnexus/test/unit/parsedfile-store.test.ts b/gitnexus/test/unit/parsedfile-store.test.ts index 6ce5cd252..d799f2054 100644 --- a/gitnexus/test/unit/parsedfile-store.test.ts +++ b/gitnexus/test/unit/parsedfile-store.test.ts @@ -1,5 +1,6 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtemp, rm, readdir, readFile } from 'fs/promises'; +import { describe, it, expect, vi } from 'vitest'; +import { promises as nodeFsPromises } from 'node:fs'; +import { mkdtemp, rm, readdir, readFile, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import path from 'path'; import type { ParsedFile } from 'gitnexus-shared'; @@ -7,8 +8,12 @@ import { clearParsedFileStore, persistParsedFileChunk, persistParsedFileShardSync, + persistDurableParsedFileShardSync, + restoreDurableParsedFileShard, loadParsedFilesForPaths, getParsedFileStoreDir, + getDurableParsedFileDir, + parsedFileLoadGc, } from '../../src/storage/parsedfile-store.js'; /** @@ -250,6 +255,15 @@ describe('parsedfile-store', () => { 'utf-8', ); expect(syncBytes).toBe(asyncBytes); + const asyncPaths = await readFile( + path.join(getParsedFileStoreDir(asyncDir), 'shard.json.paths'), + 'utf-8', + ); + const syncPaths = await readFile( + path.join(getParsedFileStoreDir(syncDir), 'shard.json.paths'), + 'utf-8', + ); + expect(syncPaths).toBe(asyncPaths); } finally { await rm(asyncDir, { recursive: true, force: true }); await rm(syncDir, { recursive: true, force: true }); @@ -614,4 +628,254 @@ describe('parsedfile-store receiverChain sanitation', () => { await rm(dir, { recursive: true, force: true }); } }); + + it('writes a .json.paths sidecar and skips JSON for non-intersecting shards (#3087)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-')); + try { + await persistParsedFileChunk(dir, 'chunk-0', [makeParsedFile('a.c')]); + await persistParsedFileChunk(dir, 'chunk-1', [makeParsedFile('b.c')]); + const storeDir = getParsedFileStoreDir(dir); + const names = await readdir(storeDir); + expect(names.sort()).toEqual([ + 'chunk-0.json', + 'chunk-0.json.paths', + 'chunk-1.json', + 'chunk-1.json.paths', + ]); + const readSpy = vi.spyOn(nodeFsPromises, 'readFile'); + try { + const loaded = await loadParsedFilesForPaths(dir, new Set(['b.c'])); + expect([...loaded.keys()]).toEqual(['b.c']); + const jsonReads = readSpy.mock.calls.filter(([p]) => { + const n = String(p); + return n.endsWith('.json') && !n.endsWith('.json.paths'); + }); + expect(jsonReads).toHaveLength(1); + expect(String(jsonReads[0][0])).toMatch(/chunk-1\.json$/); + } finally { + readSpy.mockRestore(); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('reads a shard when its sidecar is missing or garbage (#3087)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-fb-')); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')]); + await persistParsedFileChunk(dir, 'bad', [makeParsedFile('b.c')]); + const storeDir = getParsedFileStoreDir(dir); + await rm(path.join(storeDir, 'ok.json.paths'), { force: true }); + await writeFile(path.join(storeDir, 'bad.json.paths'), 'not\x00valid', 'utf-8'); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c', 'b.c'])); + expect(loaded.has('a.c')).toBe(true); + expect(loaded.has('b.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('reads a shard when its sidecar is truncated without a trailing newline (#3087)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-trunc-')); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('wanted.c')]); + const storeDir = getParsedFileStoreDir(dir); + await writeFile(path.join(storeDir, 'ok.json.paths'), 'unrelated.c', 'utf-8'); + const loaded = await loadParsedFilesForPaths(dir, new Set(['wanted.c'])); + expect(loaded.has('wanted.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('reads a shard when its sidecar is a newline-terminated partial listing', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-partial-')); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('wanted.c')]); + const storeDir = getParsedFileStoreDir(dir); + await writeFile(path.join(storeDir, 'ok.json.paths'), 'unrelated.c\n', 'utf-8'); + const loaded = await loadParsedFilesForPaths(dir, new Set(['wanted.c'])); + expect(loaded.has('wanted.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('reads a shard when its sidecar contains CR', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-cr-')); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('wanted.c')]); + const storeDir = getParsedFileStoreDir(dir); + await writeFile(path.join(storeDir, 'ok.json.paths'), 'unrelated.c\r\n', 'utf-8'); + const loaded = await loadParsedFilesForPaths(dir, new Set(['wanted.c'])); + expect(loaded.has('wanted.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('omits a sidecar when a filePath contains a newline and still loads JSON', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-nl-')); + const weird = 'weird\nname.c'; + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile(weird)]); + const storeDir = getParsedFileStoreDir(dir); + expect(await readdir(storeDir)).toEqual(['ok.json']); + const loaded = await loadParsedFilesForPaths(dir, new Set([weird])); + expect(loaded.has(weird)).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('removes a stale sidecar when a rewritten shard is no longer listing-safe', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-stale-')); + const weird = 'weird\nname.c'; + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('safe.c')]); + await persistParsedFileChunk(dir, 'ok', [makeParsedFile(weird)]); + const storeDir = getParsedFileStoreDir(dir); + expect(await readdir(storeDir)).toEqual(['ok.json']); + const loaded = await loadParsedFilesForPaths(dir, new Set([weird])); + expect(loaded.has(weird)).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('does not forceGc on a small store (byte budget, not every 8 shards) (#3086)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-gc-')); + const gc = vi.fn(); + const prev = parsedFileLoadGc.run; + parsedFileLoadGc.run = gc; + try { + for (let i = 0; i < 16; i++) { + await persistParsedFileChunk(dir, `s${i}`, [makeParsedFile(`f${i}.c`)]); + } + await loadParsedFilesForPaths(dir, new Set(Array.from({ length: 16 }, (_, i) => `f${i}.c`))); + expect(gc).not.toHaveBeenCalled(); + } finally { + parsedFileLoadGc.run = prev; + await rm(dir, { recursive: true, force: true }); + } + }); + + it('forceGc when accumulated raw JSON bytes reach parsedFileLoadGc.byteBudget (#3086)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-gc-pos-')); + const gc = vi.fn(); + const prevRun = parsedFileLoadGc.run; + const prevBudget = parsedFileLoadGc.byteBudget; + parsedFileLoadGc.run = gc; + parsedFileLoadGc.byteBudget = 8; + try { + await persistParsedFileChunk(dir, 's0', [makeParsedFile('f0.c')]); + await loadParsedFilesForPaths(dir, new Set(['f0.c'])); + expect(gc).toHaveBeenCalled(); + } finally { + parsedFileLoadGc.run = prevRun; + parsedFileLoadGc.byteBudget = prevBudget; + await rm(dir, { recursive: true, force: true }); + } + }); + + it('restoreDurableParsedFileShard copies sidecars and returns JSON shard count (#3087)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-restore-')); + try { + const durable = getDurableParsedFileDir(dir); + persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); + const restored = await restoreDurableParsedFileShard(durable, dir, 'abc'); + expect(restored).toBe(1); + const storeDir = getParsedFileStoreDir(dir); + expect(await readdir(storeDir)).toEqual( + expect.arrayContaining(['abc-w1-0.json', 'abc-w1-0.json.paths']), + ); + expect(await readFile(path.join(storeDir, 'abc-w1-0.json.paths'), 'utf-8')).toBe('1\na.c\n'); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); + expect(loaded.has('a.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('restoreDurableParsedFileShard unlinks a stale dest sidecar when the source has none', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-restore-stale-')); + try { + const durable = getDurableParsedFileDir(dir); + persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); + const durableShard = path.join(durable, 'abc', 'abc-w1-0.json'); + await rm(`${durableShard}.paths`, { force: true }); + const storeDir = getParsedFileStoreDir(dir); + await nodeFsPromises.mkdir(storeDir, { recursive: true }); + await writeFile(path.join(storeDir, 'abc-w1-0.json.paths'), 'stale.c\n', 'utf-8'); + const restored = await restoreDurableParsedFileShard(durable, dir, 'abc'); + expect(restored).toBe(1); + await expect(readFile(path.join(storeDir, 'abc-w1-0.json.paths'), 'utf-8')).rejects.toThrow(); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); + expect(loaded.has('a.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('drops a leftover sidecar before overwriting JSON so load cannot skip new paths', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-rewrite-')); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('stale.c')]); + const origUnlink = nodeFsPromises.unlink.bind(nodeFsPromises); + const origWrite = nodeFsPromises.writeFile.bind(nodeFsPromises); + const order: string[] = []; + const unlinkSpy = vi + .spyOn(nodeFsPromises, 'unlink') + .mockImplementation(async (p, ...rest) => { + order.push(`unlink:${path.basename(String(p))}`); + return origUnlink(p, ...rest); + }); + const writeSpy = vi + .spyOn(nodeFsPromises, 'writeFile') + .mockImplementation(async (p, data, enc) => { + order.push(`write:${path.basename(String(p))}`); + return origWrite(p, data, enc); + }); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')]); + } finally { + unlinkSpy.mockRestore(); + writeSpy.mockRestore(); + } + const jsonIdx = order.indexOf('write:ok.json'); + const pathsIdx = order.indexOf('unlink:ok.json.paths'); + expect(pathsIdx).toBeGreaterThanOrEqual(0); + expect(pathsIdx).toBeLessThan(jsonIdx); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); + expect(loaded.has('a.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('persist still succeeds when the sidecar write fails', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-enospc-')); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('stale.c')]); + const orig = nodeFsPromises.writeFile.bind(nodeFsPromises); + const spy = vi.spyOn(nodeFsPromises, 'writeFile').mockImplementation(async (p, data, enc) => { + if (String(p).endsWith('.paths')) { + throw Object.assign(new Error('ENOSPC'), { code: 'ENOSPC' }); + } + return orig(p, data, enc); + }); + try { + await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')]); + } finally { + spy.mockRestore(); + } + const storeDir = getParsedFileStoreDir(dir); + await expect(readFile(path.join(storeDir, 'ok.json.paths'), 'utf-8')).rejects.toThrow(); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); + expect(loaded.has('a.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); From dc5c816a020e76788a50d357e6f72c3d1cb8f79a Mon Sep 17 00:00:00 2001 From: FenjuFu <92919259+FenjuFu@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:12:46 +0800 Subject: [PATCH 26/28] fix(serve): protect MCP route with optional bearer auth (#3100) * fix(serve): protect MCP route with optional bearer auth Signed-off-by: FenjuFu <92919259+FenjuFu@users.noreply.github.com> * fix(serve): clarify MCP auth proxy boundaries Document the Render token incompatibility, expose serve auth in CLI help, and replace source-order assertions with live middleware coverage. Note: full test suite has pre-existing worktree failures because generated parse-worker.js is absent; targeted auth and proxy suites pass. Co-authored-by: Cursor * chore(docs): preserve existing table formatting Keep the auth clarifications focused without reformatting unrelated Markdown tables. Co-authored-by: Cursor * fix(proxy): inject backend MCP credentials Replace the consumed edge credential with the configured protocol token only for MCP routes so proxied serve authentication remains composable. Co-authored-by: Cursor --------- Signed-off-by: FenjuFu <92919259+FenjuFu@users.noreply.github.com> Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- README.md | 1 + SECURITY.md | 3 +- docker-server.mjs | 22 +++- docker-server.test.mjs | 82 ++++++++++++- gitnexus/src/cli/i18n/en.ts | 2 +- gitnexus/src/cli/i18n/zh-CN.ts | 2 +- gitnexus/src/cli/index.ts | 2 +- gitnexus/src/server/api.ts | 5 +- gitnexus/src/server/mcp-http.ts | 22 +++- gitnexus/test/unit/mcp-http-transport.test.ts | 108 +++++++++++++++++- render.yaml | 4 +- 11 files changed, 239 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 405d85201..75bdd8475 100644 --- a/README.md +++ b/README.md @@ -530,6 +530,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | | `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | +| `GITNEXUS_MCP_AUTH_TOKEN` | unset | Bearer token for the dedicated `gitnexus mcp --http` server, for a **directly reachable** `gitnexus serve` `/api/mcp` route, and for the `docker-server` / web proxy in front of one. A non-loopback dedicated MCP bind requires it; `serve` enables protocol-layer MCP auth when it is set. Behind a proxy, set the **same** value on both services: the proxy spends the edge `GITNEXUS_SERVE_AUTH_TOKEN`, then replaces `Authorization` with this token on `/api/mcp` only. | Dedicated MCP, a `serve` the client can reach directly, or a proxied deploy (Render Blueprint) where the backend runs protocol-layer MCP auth — configure it on the proxy too. | | `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | | `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | | `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | diff --git a/SECURITY.md b/SECURITY.md index d1fbcd051..89368a1ea 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -59,7 +59,8 @@ The `render.yaml` Blueprint (see the README's **Deploy to Render**) puts `gitnex - **The generated `GITNEXUS_SERVE_AUTH_TOKEN` is the only access control.** The proxy rejects any `/api/*` request without it with a `401` before forwarding. Rotate it by editing the environment variable on the `gitnexus-web` service and redeploying. - **The CSRF guard is inert on this path.** The proxy strips `Origin` before forwarding, so the server's write-origin guard does nothing for proxied traffic — it passes `Origin`-less requests through by design. The token is not a second layer behind the guard. - **Anyone holding the token can read every indexed repo's source.** These routes carry no origin guard, and the first three carry no rate limiter either: `GET /api/repos`, `GET /api/graph`, `POST /api/query`, `GET /api/file`, `GET /api/grep`. Whoever has the token can also index and delete repositories. -- **`POST /api/mcp` rides the same path.** `serve` mounts the MCP handler via `mountMCPEndpoints`, and `createStreamableHttpHandler` is called with no `authToken` — a **pre-existing** gap in `serve` itself, not something this deploy introduces. On Render it is closed only by the edge token and the private network. A `serve` bound directly to a public interface has no such cover. +- **`POST /api/mcp` rides the same path.** When `GITNEXUS_MCP_AUTH_TOKEN` is set on the backend, `serve` protects `/api/mcp` with the same constant-time Bearer check as the dedicated HTTP MCP server, before parsing the request body. The Render Blueprint does not set a backend MCP token by default. To enable it behind the proxy, set the **same** `GITNEXUS_MCP_AUTH_TOKEN` on both the `gitnexus-web` proxy and the `gitnexus-server` backend: the proxy consumes the edge `GITNEXUS_SERVE_AUTH_TOKEN`, then replaces `Authorization` with the MCP token on `/api/mcp` (and its subpaths) only — the edge credential is never forwarded, and other `/api/*` routes stay stripped. Configuring it on the backend alone makes every proxied MCP request `401`. +- **A directly reachable `serve` still needs an explicit control.** If neither `GITNEXUS_MCP_AUTH_TOKEN` nor an authenticated edge/private-network boundary is present, `/api/mcp` is unauthenticated. Do not bind that topology to a LAN or public interface: MCP readers can access indexed source and graph context. - **Rate limits bound cost, not access.** They cap what a token holder can spend; they do not decide who gets in. Do not hand the URL out as a public demo. A token holder has read access to everything the deploy has indexed. diff --git a/docker-server.mjs b/docker-server.mjs index e3e9f68ee..3b036f7db 100644 --- a/docker-server.mjs +++ b/docker-server.mjs @@ -112,6 +112,14 @@ const upstreamOrigin = upstreamBase ? new URL(upstreamBase).origin : null; // (gitnexus/src/mcp/http-transport.ts). const authToken = process.env.GITNEXUS_SERVE_AUTH_TOKEN?.trim() || null; +// The protocol-layer credential the upstream `serve` expects on /api/mcp when it +// runs with MCP Bearer auth enabled. Set it to the SAME value on both services: +// the edge token is spent here and replaced with this one for MCP requests only +// (see proxyToUpstream). Unset — the default — means no injection, so a backend +// without MCP auth is unaffected. Blank-is-absent follows resolveAuthToken +// (gitnexus/src/mcp/http-transport.ts). Never logged. +const mcpAuthToken = process.env.GITNEXUS_MCP_AUTH_TOKEN?.trim() || null; + // Mirrors the non-loopback refusal in http-transport.ts (startMcpHttpServer), // relocated because the trust boundary is here: an unguarded `serve` behind a // private service is legitimate, an unguarded public proxy is not. @@ -341,11 +349,17 @@ async function proxyToUpstream(req, res) { // talks to this same-origin web service. delete headers.origin; delete headers.referer; - // The edge token is spent here. `serve` reads no Authorization header - // (gitnexus/src/server/mcp-http.ts mounts /api/mcp unguarded), so forwarding - // it would only copy a live credential into another service's logs. Pinned by - // test. + // The edge token is spent here and must never be forwarded: copying + // Authorization would put a live credential into another service's logs. So + // drop it unconditionally first, then — for the MCP route alone, and only + // when a backend token is configured — replace it with that separate + // protocol credential. Unset GITNEXUS_MCP_AUTH_TOKEN (the default) leaves + // every request stripped, as before. The scope is the normalized pathname, + // so a query string can't widen it and /api/mcpfoo doesn't qualify. delete headers.authorization; + const upstreamPath = upstream.pathname; + const isMcpRoute = upstreamPath === '/api/mcp' || upstreamPath.startsWith('/api/mcp/'); + if (isMcpRoute && mcpAuthToken) headers.authorization = `Bearer ${mcpAuthToken}`; headers.host = upstream.host; // Replace, never forward, the inbound chain (see clientAddressFor). const clientAddress = clientAddressFor(req); diff --git a/docker-server.test.mjs b/docker-server.test.mjs index 80e742f7e..6d2a9f6c2 100644 --- a/docker-server.test.mjs +++ b/docker-server.test.mjs @@ -271,6 +271,12 @@ it('does not inject config into static assets', async () => { const TEST_AUTH_TOKEN = 'proxy-test-token-0123456789abcdefghij'; const TEST_BEARER = `Bearer ${TEST_AUTH_TOKEN}`; +// The protocol token the upstream expects on /api/mcp. Deliberately unlike the +// edge token, so "injected the backend credential" and "forwarded the edge one" +// can never both satisfy an assertion. +const TEST_MCP_TOKEN = 'backend-mcp-token-0123456789abcdefghij'; +const TEST_MCP_BEARER = `Bearer ${TEST_MCP_TOKEN}`; + // rawRequest never sends credentials; apiRequest does. In a file whose subject // is who gets let through, no test should pass because a helper quietly // authenticated for it. @@ -376,6 +382,11 @@ async function withProxy( const proc = spawnServerWithEnv(dir, port, { GITNEXUS_UPSTREAM_URL: schemeless ? target : `http://${target}`, GITNEXUS_SERVE_AUTH_TOKEN: TEST_AUTH_TOKEN, + // An ambient GITNEXUS_MCP_AUTH_TOKEN in the developer's shell would make the + // proxy inject one on /api/mcp, so drop it: spawn omits undefined entries, + // which unsets the inherited value. A test that wants injection sets it via + // `env` below. + GITNEXUS_MCP_AUTH_TOKEN: undefined, ...env, }); proc.stderr.setEncoding('utf8'); @@ -969,8 +980,9 @@ it('forwards an /api/* request that carries the correct token', async () => { }); it('strips the Authorization header instead of forwarding the edge token', async () => { - // The token is spent at this hop. `serve` reads no Authorization header, so - // forwarding would only copy a live credential into another service's logs. + // The edge credential is spent and stripped at this hop. Forwarding it + // would copy a live credential into another service's logs. With no + // GITNEXUS_MCP_AUTH_TOKEN configured — the default — nothing replaces it. await withProxy({}, async (port, ctx) => { const res = await apiRequest(port, '/api/mcp', { method: 'POST', body: '{}' }); assert.equal(res.status, 200, 'the request itself must still be proxied'); @@ -978,6 +990,72 @@ it('strips the Authorization header instead of forwarding the edge token', async }); }); +// -- Upstream MCP token injection (GITNEXUS_MCP_AUTH_TOKEN) ----------------- +// +// A backend running protocol-layer MCP auth expects its own Bearer on +// /api/mcp, and the edge credential can't serve as one. Both services are +// configured with the same GITNEXUS_MCP_AUTH_TOKEN; this hop spends the edge +// token and substitutes the backend one, for that route only. + +// Stands in for a `serve` with MCP Bearer auth enabled: only the exact backend +// credential gets through, so a passing two-hop request proves what was sent. +const mcpBackend = (req, res) => { + if (req.headers.authorization !== TEST_MCP_BEARER) { + res.writeHead(401, { 'Content-Type': 'application/json; charset=utf-8' }); + res.end('{"error":"unauthorized"}'); + return; + } + res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' }); + res.end('{"ok":true}'); +}; + +it('treats a blank GITNEXUS_MCP_AUTH_TOKEN as unset and still strips', async () => { + const env = { GITNEXUS_MCP_AUTH_TOKEN: ' ' }; + await withProxy({ env }, async (port, ctx) => { + const res = await apiRequest(port, '/api/mcp', { method: 'POST', body: '{}' }); + assert.equal(res.status, 200); + assert.equal(ctx.received.headers.authorization, undefined); + }); +}); + +it('replaces the edge credential with the upstream MCP token on /api/mcp', async () => { + const env = { GITNEXUS_MCP_AUTH_TOKEN: TEST_MCP_TOKEN }; + await withProxy({ upstream: mcpBackend, env }, async (port, ctx) => { + const res = await apiRequest(port, '/api/mcp', { method: 'POST', body: '{}' }); + assert.equal(res.status, 200, 'a backend that demands the MCP token must accept this hop'); + assert.equal(ctx.received.headers.authorization, TEST_MCP_BEARER); + assert.notEqual( + ctx.received.headers.authorization, + TEST_BEARER, + 'the edge credential must never be forwarded', + ); + }); +}); + +it('injects the upstream MCP token on /api/mcp subpaths and ignores the query string', async () => { + const env = { GITNEXUS_MCP_AUTH_TOKEN: TEST_MCP_TOKEN }; + await withProxy({ upstream: mcpBackend, env }, async (port, ctx) => { + for (const path of ['/api/mcp/messages', '/api/mcp?session=abc']) { + const res = await apiRequest(port, path, { method: 'POST', body: '{}' }); + assert.equal(res.status, 200, `${path} must reach the MCP backend authenticated`); + assert.equal(ctx.received.headers.authorization, TEST_MCP_BEARER, path); + } + }); +}); + +it('leaves non-MCP routes stripped when an upstream MCP token is configured', async () => { + // /api/mcpfoo shares a prefix with the MCP route but is not it, and a plain + // API route never carries a protocol credential. + const env = { GITNEXUS_MCP_AUTH_TOKEN: TEST_MCP_TOKEN }; + await withProxy({ env }, async (port, ctx) => { + for (const path of ['/api/mcpfoo', '/api/health']) { + const res = await apiRequest(port, path); + assert.equal(res.status, 200); + assert.equal(ctx.received.headers.authorization, undefined, path); + } + }); +}); + it('never gates static assets behind the token', async () => { // The UI has to load before it can prompt for a token. await withProxy({}, async (port, ctx) => { diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 698e27d4e..f5654d1bf 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -238,7 +238,7 @@ export const en = { 'help.option.mcp.host': 'HTTP bind address (only with --http). Default: 127.0.0.1 (loopback). Use 0.0.0.0 to expose to all interfaces.', 'help.option.mcp.authToken': - 'Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.', + "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.", 'help.option.force.confirmation': 'Skip confirmation prompt', 'help.option.uninstall.force': 'Apply the changes (default is a dry-run preview)', 'help.option.clean.all': 'Clean all indexed repos', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 24a6a2ad6..4aff202b6 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -222,7 +222,7 @@ export const zhCN = { 'help.option.mcp.host': 'HTTP 绑定地址（仅与 --http 搭配使用）。默认：127.0.0.1（回环）。使用 0.0.0.0 向所有接口开放。', 'help.option.mcp.authToken': - '要求 Authorization 头携带此 Bearer Token（仅与 --http 搭配使用）；也可通过 GITNEXUS_MCP_AUTH_TOKEN 环境变量设置。非回环绑定（--host 0.0.0.0/::）时必填，否则拒绝启动。', + '要求 Authorization 头携带此 Bearer Token（仅与 --http 搭配使用）；也可通过 GITNEXUS_MCP_AUTH_TOKEN 环境变量设置，该变量同时为 gitnexus serve 的 /api/mcp 路由启用 MCP Bearer 认证。非回环绑定（--host 0.0.0.0/::）时必填，否则拒绝启动。', 'help.option.force.confirmation': '跳过确认提示', 'help.option.uninstall.force': '应用更改（默认仅为预演预览）', 'help.option.clean.all': '清理所有已索引仓库', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 8296787af..8f6a0f2df 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -245,7 +245,7 @@ program ) .option( '--auth-token ', - 'Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.', + "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.", ) .action(createLbugLazyAction(() => import('./mcp.js'), 'mcpCommand')); diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index bd7ab685c..fc2e7d943 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -39,7 +39,7 @@ import { searchFTSFromLbug } from '../core/search/bm25-index.js'; import { hybridSearch } from '../core/search/hybrid-search.js'; import { ftsDegradedWarning } from '../core/search/fts-indexes.js'; import { LocalBackend } from '../mcp/local/local-backend.js'; -import { mountMCPEndpoints } from './mcp-http.js'; +import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js'; import { fileURLToPath } from 'url'; import { isTerminalJobStatus, JobManager, type AnalyzeJobPartialOutcome } from './analyze-job.js'; import { mountSSEProgress } from './sse-progress.js'; @@ -755,6 +755,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }, }), ); + // Optional protocol-layer auth for the MCP route. Keep this before the + // global body parser so rejected requests do not consume the JSON budget. + installServeMcpAuth(app); app.use(express.json({ limit: '10mb' })); // Origin guard for write routes: loopback, the server's own bound host, and diff --git a/gitnexus/src/server/mcp-http.ts b/gitnexus/src/server/mcp-http.ts index cf17bf763..74fd6b706 100644 --- a/gitnexus/src/server/mcp-http.ts +++ b/gitnexus/src/server/mcp-http.ts @@ -9,11 +9,31 @@ */ import type { Express, Request, Response } from 'express'; -import { createStreamableHttpHandler } from '../mcp/http-transport.js'; +import { + createAuthMiddleware, + createStreamableHttpHandler, + resolveAuthToken, +} from '../mcp/http-transport.js'; import type { LocalBackend } from '../mcp/local/local-backend.js'; import { createMcpRepositoryPolicy } from '../mcp/repository-policy.js'; import { logger } from '../core/logger.js'; +/** + * Protect serve's /api/mcp route when the shared MCP bearer token is configured. + * + * This middleware must be installed before Express's global JSON parser so an + * unauthenticated request body is rejected before it is parsed. The standalone + * `gitnexus mcp --http` server resolves the same environment variable. + */ +export function installServeMcpAuth(app: Express, env: NodeJS.ProcessEnv = process.env): boolean { + const authToken = resolveAuthToken(undefined, env); + if (!authToken) return false; + + app.use('/api/mcp', createAuthMiddleware(authToken)); + logger.info('Bearer authentication enabled for serve /api/mcp'); + return true; +} + export async function mountMCPEndpoints( app: Express, backend: LocalBackend, diff --git a/gitnexus/test/unit/mcp-http-transport.test.ts b/gitnexus/test/unit/mcp-http-transport.test.ts index 7cdf39f9f..d5853a63a 100644 --- a/gitnexus/test/unit/mcp-http-transport.test.ts +++ b/gitnexus/test/unit/mcp-http-transport.test.ts @@ -34,7 +34,7 @@ import { installSignalShutdown, SHUTDOWN_EXIT_CODES, } from '../../src/mcp/server.js'; -import { mountMCPEndpoints } from '../../src/server/mcp-http.js'; +import { installServeMcpAuth, mountMCPEndpoints } from '../../src/server/mcp-http.js'; // ─── Live-HTTP helpers (real req/res for SDK-touching paths) ─────────── @@ -638,6 +638,112 @@ describe('createSseHandlers', () => { // ─── mountMCPEndpoints refactor safety ─────────────────────────────── describe('mountMCPEndpoints', () => { + it.each([{}, { GITNEXUS_MCP_AUTH_TOKEN: '' }, { GITNEXUS_MCP_AUTH_TOKEN: ' ' }])( + 'does not install serve auth without a nonblank token (%j)', + (env) => { + const app = { use: vi.fn() }; + + expect(installServeMcpAuth(app as never, env)).toBe(false); + expect(app.use).not.toHaveBeenCalled(); + }, + ); + + it('installs the shared Bearer middleware for serve /api/mcp', () => { + const app = { use: vi.fn() }; + + expect(installServeMcpAuth(app as never, { GITNEXUS_MCP_AUTH_TOKEN: 'serve-secret' })).toBe( + true, + ); + expect(app.use).toHaveBeenCalledTimes(1); + expect(app.use.mock.calls[0]?.[0]).toBe('/api/mcp'); + + const middleware = app.use.mock.calls[0]?.[1] as ( + req: Request, + res: Response, + next: NextFunction, + ) => void; + const missingRes = createMockRes(); + const missingNext = vi.fn(); + middleware(createMockReq(), missingRes, missingNext); + expect(missingRes._status).toBe(401); + expect(missingNext).not.toHaveBeenCalled(); + + const wrongRes = createMockRes(); + const wrongNext = vi.fn(); + middleware(createMockReq({ authorization: 'Bearer wrong-secret' }), wrongRes, wrongNext); + expect(wrongRes._status).toBe(401); + expect(wrongNext).not.toHaveBeenCalled(); + + const validRes = createMockRes(); + const validNext = vi.fn(); + middleware(createMockReq({ authorization: 'Bearer serve-secret' }), validRes, validNext); + expect(validNext).toHaveBeenCalledOnce(); + expect(validRes._status).toBe(200); + }); + + it('wires serve MCP auth before the global JSON body parser', async () => { + const app = express(); + let parsedBodies = 0; + + expect(installServeMcpAuth(app, { GITNEXUS_MCP_AUTH_TOKEN: 'serve-secret' })).toBe(true); + app.use( + express.json({ + limit: '10mb', + verify: () => { + parsedBodies += 1; + }, + }), + ); + app.all('/api/mcp', (_req: Request, res: Response) => { + res.status(204).end(); + }); + app.post('/api/other', (req: Request, res: Response) => { + res.status(200).json(req.body); + }); + + const { port, close } = await listen(app); + const json = { 'Content-Type': 'application/json' }; + const payload = JSON.stringify({ jsonrpc: '2.0', method: 'tools/list', id: 1 }); + + try { + const missing = await request(port, 'POST', '/api/mcp', json, payload); + expect(missing.status).toBe(401); + expect(JSON.parse(missing.body)).toMatchObject({ + jsonrpc: '2.0', + error: { code: -32001, message: 'Unauthorized' }, + }); + expect(parsedBodies).toBe(0); + + const wrong = await request( + port, + 'POST', + '/api/mcp', + { ...json, Authorization: 'Bearer wrong-secret' }, + payload, + ); + expect(wrong.status).toBe(401); + expect(parsedBodies).toBe(0); + + const valid = await request( + port, + 'POST', + '/api/mcp', + { ...json, Authorization: 'Bearer serve-secret' }, + payload, + ); + expect(valid.status).toBe(204); + expect(parsedBodies).toBe(1); + + // The auth gate is scoped to /api/mcp: other routes stay unauthenticated and parsed. + const other = await request(port, 'POST', '/api/other', json, JSON.stringify({ ok: true })); + expect(other.status).toBe(200); + expect(JSON.parse(other.body)).toEqual({ ok: true }); + expect(parsedBodies).toBe(2); + } finally { + await close(); + } + }); + it('returns a cleanup function', async () => { const backend = createMockBackend(); const mockApp = { diff --git a/render.yaml b/render.yaml index 977765108..124574872 100644 --- a/render.yaml +++ b/render.yaml @@ -17,7 +17,9 @@ projects: environments: - name: production services: - # Private: no public URL. `serve` has no authentication of its own. + # Private: no public URL. `serve`'s own protocol auth (MCP Bearer) is + # optional and unset by this Blueprint; the public edge token on the + # web service below remains the access control. - type: pserv name: gitnexus-server runtime: docker From 94f67d79d5ac1e83dd0c7baa7de05c3b9b562439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sun, 30 Aug 2026 18:07:57 +0100 Subject: [PATCH 27/28] fix(analyze): make incremental analyze skip the derived layers it can reuse (#3016) (#3102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(analyze): make incremental analyze skip the derived layers it can reuse (#3016) A warm incremental run only ever wrote a handful of files, but it still paid for the whole graph on the way out: Leiden ran over every node, flow extraction re-derived every process, and all FTS indexes were dropped and rebuilt from scratch. On a small edit that tail dominated the run, which is why "incremental" did not feel incremental. Reuse what the previous run already derived when the write plan allows it. The pipeline holds back community detection and flow extraction whenever the persisted metadata says this run is a candidate for a surgical write; the DB keeps its Community/Process rows instead of a wipe-and-rewrite; and the FTS sweep is narrowed to the indexes the run actually has to touch. The bet is placed before the pipeline and settled after it. Any plan that turns out to need a freshly derived layer — full rebuild, escalated write, or an incremental diff with deleted files — runs the held-back phases through `runDeferredDerivedPhases`, against the same graph and phase outputs, so its output is identical to never having skipped them. Correctness details worth naming, since each one silently loses data if got wrong: - The MEMBER_OF / STEP_IN_PROCESS edges of the changed files are snapshotted before the DETACH DELETE and reattached after the subgraph load. Both endpoints are matched by explicit label: `labels(n)[0]` over an unlabelled match returns an empty string on this engine, which produced a snapshot that restored nothing. - The FTS narrowing unions three sets — what the writeback deletes (a DB probe, because a symbol the edit removed is in no fresh graph but is still a row), what it inserts (the fresh graph), and what is missing right now (else a prior escalation's dropped indexes would never come back). An unreadable index catalog withdraws the narrowing entirely. - Deletions disqualify reuse outright: persisted derived rows can reference nodes this run removes, and nothing short of re-deriving can tell which. Covered by the existing incremental suites, including the incremental-equals-force byte-equivalence test and the #2589 drop-before-delete ordering test, plus unit tests for the new helpers. Co-authored-by: Cursor * fix(analyze): address #3102 review on derived reuse and FTS narrowing Re-run Leiden/flows unless the file-hash diff is empty, restore ENTRY_POINT_OF on the preserve path, always drop class_fts before Spring synthetic Class DML, and reject seeded duplicate phase names. Prettier and exact FTS drop-ordering assertions unblock CI and pin the #2589/#3016 contract. Co-authored-by: Cursor * refactor(analyze): reuse FileHashDiff for derived-layer preserve Drop the count DTO, share phase-name uniqueness, and remove the File FTS sentinel that Class already makes unreachable. Refs #3102 Co-authored-by: Cursor * style(analyze): prettier-wrap shouldPreservePersistedDerivedGraph quality / format failed on the Pick signature wrapping. Refs #3102 Co-authored-by: Cursor --------- Co-authored-by: Gergo Magyar Co-authored-by: Cursor --- .../src/core/incremental/derived-writeback.ts | 83 ++++++++ .../src/core/incremental/subgraph-extract.ts | 13 +- .../core/ingestion/pipeline-phases/runner.ts | 55 +++-- gitnexus/src/core/ingestion/pipeline.ts | 69 ++++++- gitnexus/src/core/lbug/lbug-adapter.ts | 188 +++++++++++++++++- gitnexus/src/core/run-analyze.ts | 145 +++++++++++++- gitnexus/src/core/search/fts-indexes.ts | 39 +++- gitnexus/src/types/pipeline.ts | 12 ++ gitnexus/test/unit/fts-indexes.test.ts | 37 ++++ .../incremental-derived-writeback.test.ts | 98 +++++++++ .../incremental-fts-drop-ordering.test.ts | 26 ++- .../unit/incremental-subgraph-extract.test.ts | 13 ++ .../ingestion/pipeline-phase-registry.test.ts | 24 +++ gitnexus/test/unit/pipeline-runner.test.ts | 72 +++++++ 14 files changed, 828 insertions(+), 46 deletions(-) create mode 100644 gitnexus/src/core/incremental/derived-writeback.ts create mode 100644 gitnexus/test/unit/incremental-derived-writeback.test.ts diff --git a/gitnexus/src/core/incremental/derived-writeback.ts b/gitnexus/src/core/incremental/derived-writeback.ts new file mode 100644 index 000000000..9cec191eb --- /dev/null +++ b/gitnexus/src/core/incremental/derived-writeback.ts @@ -0,0 +1,83 @@ +/** + * Incremental derived-layer writeback helpers (#3016). + * + * The derived layers — Leiden communities, execution flows, and the FTS + * indexes — are graph-wide, so every analyze run rebuilt all three in full no + * matter how small the diff. A surgical incremental write can instead: + * - drop and rebuild only the FTS indexes whose tables hold rows in the + * write set (LadybugDB still cannot DML a table with a live FTS index — + * #2589 — so a table being written must still lose its index first); + * - leave the untouched tables' rows alone, so their indexes stay live; + * - reuse persisted Community/Process rows only when the file-hash diff is + * empty (no added, changed, or deleted files). Any content change can + * add, rename, or retarget symbols that Leiden and flow extraction + * consume — a no-deletion edit is not a validity proof. + */ +import { FTS_INDEXES } from '../search/fts-schema.js'; +import type { KnowledgeGraph } from '../graph/types.js'; +import type { FileHashDiff } from '../../storage/file-hash.js'; + +const FTS_TABLE_NAMES: ReadonlySet = new Set(FTS_INDEXES.map((i) => i.table)); + +/** The FTS-backed members of `tables`. */ +export const ftsTablesAmong = (tables: Iterable): Set => { + const out = new Set(); + for (const table of tables) { + if (FTS_TABLE_NAMES.has(table)) out.add(table); + } + return out; +}; + +/** + * Whether a surgical incremental write may reuse the persisted derived layer. + * + * Deletions disqualify it: the persisted Community/Process rows and their + * MEMBER_OF / STEP_IN_PROCESS edges can reference nodes that no longer exist + * after this run, and nothing short of re-deriving can tell which. + * + * Added or content-changed files also disqualify it: they can introduce, + * rename, or retarget symbols and CALLS edges that Leiden and flow extraction + * consume. File-deletion-only was too weak a proof that the derived graph is + * still valid. + */ +export const shouldPreservePersistedDerivedGraph = ( + diff: Pick, +): boolean => diff.deleted.length === 0 && diff.added.length === 0 && diff.changed.length === 0; + +/** + * FTS-backed node tables that the fresh graph will WRITE rows into for + * `fileSet` — the inserting half of the DML. + * + * Callers must union this with a DB probe for the deleting half + * (`nodeTablesWithRowsForFiles`): a table whose last row in these files was + * just removed by the edit has nothing here, but still holds a stale row that + * the writeback must delete, and deleting it means taking its index down too. + */ +export const incrementalFtsTablesFromGraph = ( + graph: KnowledgeGraph, + fileSet: ReadonlySet, +): Set => { + const touched = new Set(); + graph.forEachNode((n) => { + const filePath = n.properties?.filePath as string | undefined; + if (!filePath || !fileSet.has(filePath)) return; + if (FTS_TABLE_NAMES.has(n.label)) touched.add(n.label); + }); + return touched; +}; + +/** + * The node tables an incremental DETACH DELETE should target, given the FTS + * tables this run is rebuilding. + * + * Every non-FTS table (Folder, CodeElement, …) deletes as before. An FTS-backed + * table only deletes when its index is being rebuilt anyway, because deleting + * from it otherwise would mean DML against a live FTS index (#2589). + */ +export const nodeTablesForIncrementalDelete = ( + allNodeTables: readonly string[], + rebuildingFtsTables: ReadonlySet, +): string[] => + allNodeTables.filter( + (tableName) => !FTS_TABLE_NAMES.has(tableName) || rebuildingFtsTables.has(tableName), + ); diff --git a/gitnexus/src/core/incremental/subgraph-extract.ts b/gitnexus/src/core/incremental/subgraph-extract.ts index e0f0e41eb..276dba4e2 100644 --- a/gitnexus/src/core/incremental/subgraph-extract.ts +++ b/gitnexus/src/core/incremental/subgraph-extract.ts @@ -6,9 +6,9 @@ * replaced, produce a smaller KnowledgeGraph that contains: * * - Every node whose `properties.filePath` is in `toWriteSet`. - * - Every graph-wide node (Community, Process, and Spring metadata - * placeholders) — these are regenerated each run and must be fully - * rewritten. + * - Graph-wide Community/Process nodes unless `includeDerivedGraphWide` + * is false (#3016 incremental preserve). Spring metadata placeholders + * are always included. * - Every relationship where AT LEAST ONE endpoint is in the writable * set above. Relationships entirely between unchanged-file nodes * are skipped — their rows are still in the DB and re-inserting @@ -122,13 +122,18 @@ const indexNodeFilePaths = (fullGraph: KnowledgeGraph): Map => { export const extractChangedSubgraph = ( fullGraph: KnowledgeGraph, toWriteSet: ReadonlySet, + options?: { includeDerivedGraphWide?: boolean }, ): KnowledgeGraph => { const sub = createKnowledgeGraph(); const writableNodeIds = new Set(); + const includeDerivedGraphWide = options?.includeDerivedGraphWide !== false; + fullGraph.forEachNode((n: GraphNode) => { const filePath = n.properties?.filePath as string | undefined; - const include = (filePath && toWriteSet.has(filePath)) || isGraphWideNode(n); + const derivedWide = + includeDerivedGraphWide || (n.label !== 'Community' && n.label !== 'Process'); + const include = (filePath && toWriteSet.has(filePath)) || (isGraphWideNode(n) && derivedWide); if (include) { sub.addNode(n); writableNodeIds.add(n.id); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts index 0bfc45bd4..da8e4dd8f 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts @@ -16,23 +16,36 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; import { isDev } from '../utils/env.js'; import { logger } from '../../logger.js'; + +function assertUniquePhaseNames(phases: readonly PipelinePhase[]): void { + const seen = new Set(); + for (const phase of phases) { + if (seen.has(phase.name)) { + throw new Error(`Duplicate phase name: '${phase.name}'`); + } + seen.add(phase.name); + } +} + /** * Validate that the phases form a valid dependency graph (no cycles, all deps present). * Returns phases in topological execution order. + * + * `satisfied` names phases whose results are already available (a deferred + * follow-up run over the same context, #3016). Their edges are dropped rather + * than validated, because they are resolved by definition. */ -function topologicalSort(phases: readonly PipelinePhase[]): PipelinePhase[] { - const phaseMap = new Map(); - for (const phase of phases) { - if (phaseMap.has(phase.name)) { - throw new Error(`Duplicate phase name: '${phase.name}'`); - } - phaseMap.set(phase.name, phase); - } +function topologicalSort( + phases: readonly PipelinePhase[], + satisfied: ReadonlySet = new Set(), +): PipelinePhase[] { + assertUniquePhaseNames(phases); + const phaseMap = new Map(phases.map((p) => [p.name, p])); // Validate all deps exist for (const phase of phases) { for (const dep of phase.deps) { - if (!phaseMap.has(dep)) { + if (!phaseMap.has(dep) && !satisfied.has(dep)) { throw new Error(`Phase '${phase.name}' depends on '${dep}', which is not registered`); } } @@ -43,8 +56,9 @@ function topologicalSort(phases: readonly PipelinePhase[]): PipelinePhase[] { const reverseDeps = new Map(); for (const phase of phases) { - inDegree.set(phase.name, phase.deps.length); - for (const dep of phase.deps) { + const pendingDeps = phase.deps.filter((dep) => !satisfied.has(dep)); + inDegree.set(phase.name, pendingDeps.length); + for (const dep of pendingDeps) { let rev = reverseDeps.get(dep); if (!rev) { rev = []; @@ -143,15 +157,30 @@ function findCyclePath( * * @param phases All phases to execute (order doesn't matter — sorted internally) * @param ctx Shared pipeline context + * @param seed Results of phases that already ran against this same context, + * available to `phases` as dependencies (#3016 deferred derived + * phases). Included in the returned map. * @returns Map of phase name → PhaseResult (all completed phases) */ export async function runPipeline( phases: readonly PipelinePhase[], ctx: PipelineContext, + seed?: ReadonlyMap>, ): Promise>> { + // A seeded phase has already run against this context; re-running it would + // apply its graph writes a second time. "Already ran" is the whole meaning of + // the seed, so honour it here rather than making every caller pre-filter. + const satisfied = new Set(seed?.keys() ?? []); let sorted: PipelinePhase[]; try { - sorted = topologicalSort(phases); + // Duplicate names must be rejected on the caller-supplied list *before* + // seed-filtering. Filtering first would drop a seeded duplicate and let + // `topologicalSort` see a unique name (#3102). + assertUniquePhaseNames(phases); + sorted = topologicalSort( + phases.filter((p) => !satisfied.has(p.name)), + satisfied, + ); } catch (err) { // Emit a terminal 'error' progress event for graph-validation failures // (cycle detected, duplicate phase, missing dep) so CLI/MCP consumers see @@ -171,7 +200,7 @@ export async function runPipeline( } throw err; } - const results = new Map>(); + const results = new Map>(seed); for (const phase of sorted) { const start = Date.now(); diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 050822e87..440bf689c 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -46,6 +46,7 @@ import { PhaseRegistry, type ScopeResolutionOutput, type PipelinePhase, + type PipelineContext, type CommunitiesOutput, type ProcessesOutput, } from './pipeline-phases/index.js'; @@ -58,6 +59,12 @@ export interface PipelineOptions { * to retain those nodes under `skipGraphPhases`. */ skipGraphPhases?: boolean; + /** + * Skip only Leiden community detection and process/flow extraction (#3016). + * MRO/DI still run. Used on warm incremental analyze so persisted + * Community/Process rows can be kept instead of wipe+rewrite. + */ + skipDerivedGraphPhases?: boolean; /** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */ springAopMaxCandidateInspectionsPerAdvice?: number; /** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */ @@ -310,8 +317,12 @@ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { .register(mroPhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(springAopInheritancePhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(diPhase, { enabledWhen: (o) => !o.skipGraphPhases }) - .register(communitiesPhase, { enabledWhen: (o) => !o.skipGraphPhases }) - .register(processesPhase, { enabledWhen: (o) => !o.skipGraphPhases }) + .register(communitiesPhase, { + enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true, + }) + .register(processesPhase, { + enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true, + }) // Normalize a missing options object once here so phase predicates above // take a required PipelineOptions and need no `?.` guard (#2080 review S1). .build(options ?? {}) @@ -351,18 +362,19 @@ export const runPipelineFromRepo = async ( } const phases = buildPhaseList(options); + const ctx: PipelineContext = { + repoPath, + graph: graphEmitSink ?? graph, + onProgress, + options, + pipelineStart, + graphEmit: graphEmitSink, + }; let graphEmitManifest: GraphEmitManifest | undefined; let results; try { - results = await runPipeline(phases, { - repoPath, - graph: graphEmitSink ?? graph, - onProgress, - options, - pipelineStart, - graphEmit: graphEmitSink, - }); + results = await runPipeline(phases, ctx); graphEmitManifest = graphEmitSink?.finalize(); } finally { // Release per-pair fds when the pipeline threw before finalize ran. @@ -412,7 +424,7 @@ export const runPipelineFromRepo = async ( }, }); - return { + const result: PipelineResult = { // The RAW graph, deliberately — NOT `graphEmitSink`. Phases above received // the sink so their reads are complete, but `loadGraphToLbug` feeds this to // `streamAllCSVsToDisk`, and the sink's complete iterator would then emit @@ -434,4 +446,39 @@ export const runPipelineFromRepo = async ( pdgEmitManifest, propertyInference, }; + + // #3016: hand back a way to run the derived phases `skipDerivedGraphPhases` + // held back. Which phases those are is answered by re-asking the registry + // with only that flag cleared — the one form of the question that stays + // correct when a different predicate (`skipGraphPhases`) also disables them, + // since then they are absent for a reason a deferred run cannot fix and the + // filter yields nothing. The sink guard mirrors the `graph` note above: a + // streaming run is a full rebuild, which never sets the skip flag, so an + // active sink here means the two got combined by mistake — and deferred + // phases writing into a finalized sink would emit past its manifest. + const deferredDerivedPhases = + options?.skipDerivedGraphPhases === true && graphEmitSink === undefined + ? buildPhaseList({ ...options, skipDerivedGraphPhases: false }).filter( + (p) => (p.name === 'communities' || p.name === 'processes') && !results.has(p.name), + ) + : []; + + if (deferredDerivedPhases.length > 0) { + result.runDeferredDerivedPhases = async () => { + const derived = await runPipeline(deferredDerivedPhases, ctx, results); + // Presence-checked for the same reason as the block above: a phase the + // registry filtered out is absent, and `getPhaseOutput` throws on absent. + if (derived.has('communities')) { + result.communityResult = getPhaseOutput( + derived, + 'communities', + ).communityResult; + } + if (derived.has('processes')) { + result.processResult = getPhaseOutput(derived, 'processes').processResult; + } + }; + } + + return result; }; diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 3058c3d81..fe563aa4c 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -2575,7 +2575,11 @@ export const DELETE_FILES_CHUNK_SIZE = 200; */ export const deleteNodesForFiles = async ( filePaths: readonly string[], - options: { onChunk?: (filesDone: number, filesTotal: number) => void } = {}, + options: { + onChunk?: (filesDone: number, filesTotal: number) => void; + /** When set, only these node tables are DETACH DELETEd (#3016). */ + nodeTables?: readonly string[]; + } = {}, ): Promise => { if (!conn) { throw new Error('LadybugDB not initialized. Call initLbug first.'); @@ -2619,7 +2623,8 @@ export const deleteNodesForFiles = async ( ); } } - for (const tableName of NODE_TABLES) { + const tables = options.nodeTables ?? NODE_TABLES; + for (const tableName of tables) { // Community/Process are graph-wide (no filePath); the orchestrator // drops them wholesale via deleteAllCommunitiesAndProcesses. if (tableName === 'Community' || tableName === 'Process') continue; @@ -2636,6 +2641,185 @@ export const deleteNodesForFiles = async ( } }; +/** + * Which of `candidateTables` currently hold at least one row for `filePaths`. + * + * The incremental writeback uses this to decide which FTS-backed tables it is + * about to DML (#3016). It has to be a question about the DB, not about the + * freshly built graph: an edit that DELETES the last Rust trait in a file + * leaves no Trait node in the new graph, but the old row is still in the index + * and still has to be deleted — and its FTS index still has to come down first. + */ +export const nodeTablesWithRowsForFiles = async ( + filePaths: readonly string[], + candidateTables: readonly string[], +): Promise> => { + const c = conn; + if (!c) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + const found = new Set(); + return withConnLock(async () => { + for (const batch of chunk(filePaths, DELETE_FILES_CHUNK_SIZE)) { + const listLiteral = `[${batch.map((p) => formatCypherValue(p)).join(', ')}]`; + for (const tableName of candidateTables) { + // Graph-wide tables have no filePath column to filter on. + if (tableName === 'Community' || tableName === 'Process') continue; + if (found.has(tableName)) continue; + // determinism: probe — asks only whether the table has any row for + // these files, so which row comes back cannot change the answer. + const queryResult = await c.query( + `MATCH (n:${escapeTableName(tableName)}) WHERE n.filePath IN ${listLiteral} ` + + `RETURN n.id LIMIT 1`, + ); + try { + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + if ((await result.getAll()).length > 0) found.add(tableName); + } finally { + await closeQueryResults(queryResult); + } + } + } + return found; + }); +}; + +/** + * The graph-wide derived edges and the node table each one points at. Both are + * produced by the derived phases (Leiden, flow extraction) rather than by + * parsing, which is why an incremental run that skips those phases has to carry + * them across the writeback itself. + */ +const DERIVED_REL_KINDS = [ + { type: 'MEMBER_OF', targetLabel: 'Community' }, + { type: 'STEP_IN_PROCESS', targetLabel: 'Process' }, + { type: 'ENTRY_POINT_OF', targetLabel: 'Process' }, +] as const; + +/** + * One MEMBER_OF / STEP_IN_PROCESS edge, carrying everything needed to recreate + * it byte-for-byte: both endpoint labels (so the re-MATCH is label-scoped + * rather than a scan of every node) and every column of the relationship + * table, `step` included — process traces order by it (`ORDER BY r.step`), so + * an edge restored without it silently scrambles the flow it belongs to. + */ +export interface DerivedRelSnapshot { + sourceId: string; + sourceLabel: string; + targetId: string; + targetLabel: string; + type: string; + confidence: number; + reason: string; + step: number; +} + +/** + * Capture the MEMBER_OF / STEP_IN_PROCESS / ENTRY_POINT_OF edges owned by `filePaths`, before a + * surgical incremental write DETACH DELETEs their file-side endpoints (#3016). + * + * Only meaningful on the write plan that keeps the persisted Community/Process + * nodes: those nodes survive the delete, but the edges tying this run's changed + * files to them do not, and the pipeline did not re-derive them. + * + * Both endpoints are matched by an EXPLICIT label — `sourceTables` on one side, + * the edge type's fixed target table on the other — so the labels come from the + * query rather than the rows. `labels(n)[0]` over an unlabelled match returns + * an empty string on this engine, which silently produced a snapshot that + * restored nothing. + * + * Read failures propagate. This runs against a warm index whose derived tables + * the caller has already established exist, so a failure here is a real fault — + * and swallowing it would drop the edges silently, which looks identical to a + * repo that genuinely has no communities. + */ +export const snapshotDerivedRelsForFiles = async ( + filePaths: readonly string[], + sourceTables: readonly string[], +): Promise => { + const c = conn; + if (!c) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + const out: DerivedRelSnapshot[] = []; + return withConnLock(async () => { + for (const batch of chunk(filePaths, DELETE_FILES_CHUNK_SIZE)) { + const listLiteral = `[${batch.map((p) => formatCypherValue(p)).join(', ')}]`; + for (const sourceLabel of sourceTables) { + if (sourceLabel === 'Community' || sourceLabel === 'Process') continue; + for (const { type, targetLabel } of DERIVED_REL_KINDS) { + const queryResult = await c.query( + `MATCH (n:${escapeTableName(sourceLabel)})-[r:${REL_TABLE_NAME}]->` + + `(m:${escapeTableName(targetLabel)}) ` + + `WHERE n.filePath IN ${listLiteral} AND r.type = ${formatCypherValue(type)} ` + + `RETURN n.id AS sourceId, m.id AS targetId, ` + + `r.confidence AS confidence, r.reason AS reason, r.step AS step`, + ); + try { + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + for (const row of await result.getAll()) { + const rec = row as Record; + if (typeof rec.sourceId !== 'string' || typeof rec.targetId !== 'string') continue; + out.push({ + sourceId: rec.sourceId, + sourceLabel, + targetId: rec.targetId, + targetLabel, + type, + confidence: typeof rec.confidence === 'number' ? rec.confidence : 1.0, + reason: typeof rec.reason === 'string' ? rec.reason : '', + step: + typeof rec.step === 'number' + ? rec.step + : typeof rec.step === 'bigint' + ? Number(rec.step) + : 0, + }); + } + } finally { + await closeQueryResults(queryResult); + } + } + } + } + return out; + }); +}; + +/** + * Re-create the edges captured by `snapshotDerivedRelsForFiles`, after the + * incremental subgraph load has put their file-side endpoints back. + * + * Endpoints are matched by label + id, mirroring `fallbackRelationshipInserts`: + * an unlabelled `MATCH (a), (b)` is a cartesian product over the whole graph + * and does not finish on a real index. An endpoint the load did not restore + * simply matches nothing, so the edge is dropped rather than mis-attached. + */ +export const restoreDerivedRels = async (rels: readonly DerivedRelSnapshot[]): Promise => { + const c = conn; + if (!c) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + if (rels.length === 0) return; + const escapeLabel = (label: string): string => + BACKTICK_TABLES.has(label) ? `\`${label}\`` : label; + // No outer `withConnLock`: `queryAndDrain` takes the lock per statement, and + // wrapping the loop as well trips the re-entry guard in conn-lock.ts. Same + // shape as `fallbackRelationshipInserts`, the other per-edge CREATE loop. + for (const rel of rels) { + if (!NODE_TABLES.includes(rel.sourceLabel as NodeTableName)) continue; + if (!NODE_TABLES.includes(rel.targetLabel as NodeTableName)) continue; + await queryAndDrain( + c, + `MATCH (a:${escapeLabel(rel.sourceLabel)} {id: ${formatCypherValue(rel.sourceId)}}), ` + + `(b:${escapeLabel(rel.targetLabel)} {id: ${formatCypherValue(rel.targetId)}}) ` + + `CREATE (a)-[:${REL_TABLE_NAME} {type: ${formatCypherValue(rel.type)}, ` + + `confidence: ${rel.confidence}, reason: ${formatCypherValue(rel.reason)}, ` + + `step: ${rel.step}}]->(b)`, + ); + } +}; + export const getEmbeddingTableName = (): string => EMBEDDING_TABLE_NAME; /** diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index ad9841406..71fece179 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -36,6 +36,9 @@ import { closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, + nodeTablesWithRowsForFiles, + snapshotDerivedRelsForFiles, + restoreDerivedRels, ensureEmbeddingRowDmlSafe, ensureFtsRowDmlSafe, readIndexCatalogSnapshot, @@ -67,6 +70,7 @@ import { createSearchFTSIndexes, summarizeFtsIndexBuildFailures, dropSearchFTSIndexes, + missingSearchFTSIndexTables, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js'; @@ -136,6 +140,13 @@ import { } from './incremental/subgraph-extract.js'; import { shadowCandidatesFor } from './incremental/shadow-candidates.js'; import { shouldEscalateIncrementalWrite } from './incremental/escalation-gate.js'; +import { + ftsTablesAmong, + incrementalFtsTablesFromGraph, + nodeTablesForIncrementalDelete, + shouldPreservePersistedDerivedGraph, +} from './incremental/derived-writeback.js'; +import { NODE_TABLES } from './lbug/schema.js'; import { loadParseCache, saveParseCache, @@ -1885,6 +1896,25 @@ async function runFullAnalysisInner( // `resolveStreamPdgEmit` — read fresh at the same point — behaves.) const streamGraphEmitActive = resolveStreamGraphEmit(options); + // #3016: hold back Leiden and flow extraction when the persisted metadata + // says this run is a candidate for a surgical incremental write, whose + // derived layer is reused rather than recomputed. Deliberately the same + // conditions as the `isIncremental` decision below MINUS the two that only + // the pipeline can answer (the analysis-feature re-check and a non-empty + // file list), so this is a superset: every run that turns out incremental + // had the phases skipped, and the runs that do not are caught by + // `runDeferredDerivedPhases` once the write plan is known. Excluded on the + // streaming path because that is a full rebuild by construction, and the + // deferred phases must not write into a finalized emit sink. + const skipDerivedGraphPhases = + !streamGraphEmitActive && + !options.force && + !!existingMeta && + !!existingMeta.fileHashes && + Object.keys(existingMeta.fileHashes).length > 0 && + repoHasGit && + !schemaFingerprintMismatch(existingMeta.schemaFingerprint); + // ── Phase 1: Full Pipeline (0–60%) ──────────────────────────────── const pipelineResult = await runPipelineFromRepo( repoPath, @@ -1927,6 +1957,7 @@ async function runFullAnalysisInner( ? resolveNativeSafeStorageDir(storagePath, 'graph-csv') : undefined, fetchWrappers: options.fetchWrappers, + skipDerivedGraphPhases, }, ); @@ -1986,6 +2017,28 @@ async function runFullAnalysisInner( ? diffFileHashes(newFileHashes, existingMeta!.fileHashes) : undefined; + // #3016: `skipDerivedGraphPhases` was decided BEFORE the pipeline, from the + // persisted metadata alone, so it can only ever be a bet that this run stays + // surgical. Settle the bet here, where `isIncremental` and the deletion set + // are both known, and pay it off by running the held-back phases whenever the + // write plan needs a freshly derived layer: + // - not incremental → full rebuild writes the whole graph, and a graph + // with no Community/Process nodes would publish an + // index with no communities and no flows; + // - added/changed/deleted files → the persisted derived layer can miss new + // symbols, keep stale memberships, or reference + // removed ids. Only an empty file-hash diff is a + // proof that Leiden/flows still match. + const preserveDerivedLayer = + skipDerivedGraphPhases && + isIncremental && + !!hashDiff && + shouldPreservePersistedDerivedGraph(hashDiff); + if (skipDerivedGraphPhases && !preserveDerivedLayer) { + progress('communities', 58, 'Detecting code communities and flows...'); + await pipelineResult.runDeferredDerivedPhases?.(); + } + // #2 atomic index publish: on a full rebuild, build the fresh DB at a temp // path and swap it over the live index in one rename at the very end, so a // concurrent MCP reader opening mid-build only ever sees the previous @@ -2196,6 +2249,7 @@ async function runFullAnalysisInner( // collapse check compares the whole in-memory graph against the whole DB, // which is only a like-for-like comparison on a full rebuild. let wroteChangedSubgraphOnly = false; + let incrementalFtsRebuildTables: Set | undefined; if (isIncremental && hashDiff) { // ── Incremental DB writeback ─────────────────────────────────── // 0. Expand the writable set with transitive importers of @@ -2490,6 +2544,14 @@ async function runFullAnalysisInner( ); if (extensionForcedRebuild || sizeForcedRebuild) { escalatedFullWrite = true; + // #3016: escalation converts this run into a wipe + full bulk COPY of + // the in-memory graph, so the derived layer the skip was betting on + // preserving has to exist in that graph after all. Same reasoning as + // the not-incremental branch above, just discovered later. + if (preserveDerivedLayer) { + progress('communities', 63, 'Detecting code communities and flows...'); + await pipelineResult.runDeferredDerivedPhases?.(); + } // Every live cause is named, not just the first: a DB can carry BOTH a // vector index and FTS indexes, and reporting one cause while the other // is equally fatal is how #2841 stayed mis-diagnosed for so long. §5.D: @@ -2701,7 +2763,53 @@ async function runFullAnalysisInner( // in between — so re-reading would only weaken the one-read invariant // the snapshot type exists to enforce. if (buildPath === lbugPath) liveIndexMutationStarted = true; - await dropSearchFTSIndexes(indexCatalogRows); + // FTS narrowing is independent of Leiden/flow reuse: even when this + // run re-derives communities, Ladybug still cannot DML a live FTS + // index (#2589), so only the tables this write set touches should + // lose their index. The probe is a question about the DB rather than + // the fresh graph — a symbol the edit DELETED is in no fresh graph + // but is still a row that has to go. + const tablesWithRows = await nodeTablesWithRowsForFiles(filesToDelete, NODE_TABLES); + // Narrowing 1 — the FTS sweep, from "every configured index" to "the + // indexes this run must touch". Three sources, and dropping any one of + // them strands something: + // - what the writeback DELETES (the probe above), because a symbol + // the edit removed is in no fresh graph but is still a row; + // - what it INSERTS (the fresh graph), because inserting under a live + // FTS index is the same #2589 hazard as deleting under one; + // - what is MISSING right now, because narrowing to the written + // tables would otherwise leave keyword search degraded forever on + // tables whose index a previous escalation dropped — the next full + // rebuild would be the only thing that ever restored them. + // An unreadable catalog proves nothing about that third set, so it + // withdraws the narrowing entirely rather than guess. + const missingFts = await missingSearchFTSIndexTables(indexCatalogRows); + const touchedFts = missingFts + ? new Set([ + ...ftsTablesAmong(tablesWithRows), + ...incrementalFtsTablesFromGraph(pipelineResult.graph, new Set(filesToDelete)), + ...missingFts, + ]) + : undefined; + // Graph-wide Spring synthetic Class nodes are DETACH DELETEd on this + // branch even when Class is not in the write set + // (`deleteSpringAutoConfigurationSyntheticClasses`). Always include + // Class so class_fts is not live across that DML (#2589), including + // when the fresh graph no longer materializes the synthetics but the + // DB still holds them. + if (touchedFts) { + touchedFts.add('Class'); + } + incrementalFtsRebuildTables = touchedFts; + // MEMBER_OF / STEP_IN_PROCESS / ENTRY_POINT_OF edges hang off the nodes + // the DETACH DELETE below removes, so preserving the Community/Process + // nodes preserves only half the layer unless these are reattached after + // the subgraph write puts the member nodes back. Only the probed tables + // can own such an edge, so they are the only ones worth scanning. + const derivedSnapshot = preserveDerivedLayer + ? await snapshotDerivedRelsForFiles(filesToDelete, [...tablesWithRows]) + : []; + await dropSearchFTSIndexes(indexCatalogRows, incrementalFtsRebuildTables); // 1b. Remove the write set's existing rows — batched (#2409): one // DETACH DELETE per table per 200-file chunk. The former per-file // loop issued a count + delete per table per FILE — ~13k @@ -2716,6 +2824,9 @@ async function runFullAnalysisInner( await deleteNodesForFiles(filesToDelete, { onChunk: (done, total) => progress('lbug', 62, `Removing rows for changed files (${done}/${total})...`), + nodeTables: incrementalFtsRebuildTables + ? nodeTablesForIncrementalDelete(NODE_TABLES, incrementalFtsRebuildTables) + : undefined, }); // Surgical path: Phase 3.5 restores exactly these files' embedding // rows (FIX 3). Sound because deleteNodesForFiles propagates errors @@ -2723,10 +2834,12 @@ async function runFullAnalysisInner( // deterministically — and this process holds the exclusive DB lock, // so no concurrent writer can disturb the derivation. deletedFilePathsForRestore = new Set(filesToDelete); - // 2. Drop graph-wide nodes (Community, Process). They'll be re-inserted - // from the fresh pipeline output below. Required for the - // "Leiden runs on the FULL graph" correctness invariant. - await deleteAllCommunitiesAndProcesses(); + if (!preserveDerivedLayer) { + // 2. Drop graph-wide nodes (Community, Process). They'll be re-inserted + // from the fresh pipeline output below. Required for the + // "Leiden runs on the FULL graph" correctness invariant. + await deleteAllCommunitiesAndProcesses(); + } // 2a. Drop INJECTS edges (DI collection injection, #2200) — their // validity is a whole-program property (a third-file change to the // interface or an implementer creates/invalidates edges between two @@ -2774,7 +2887,9 @@ async function runFullAnalysisInner( // only that. Unchanged-file rows in the DB stay untouched. Pass // the SAME effectiveWriteSet so the subgraph and the deletes // cover identical files (asymmetry would silently corrupt). - const subgraph = extractChangedSubgraph(pipelineResult.graph, effectiveWriteSet); + const subgraph = extractChangedSubgraph(pipelineResult.graph, effectiveWriteSet, { + includeDerivedGraphWide: !preserveDerivedLayer, + }); wroteChangedSubgraphOnly = true; await saveIncrementalDirtyState('load-graph', { importerExpansion, @@ -2787,6 +2902,9 @@ async function runFullAnalysisInner( const pct = Math.min(84, 65 + Math.round((lbugMsgCount / (lbugMsgCount + 10)) * 19)); progress('lbug', pct, msg); }); + if (preserveDerivedLayer && derivedSnapshot.length > 0) { + await restoreDerivedRels(derivedSnapshot); + } } // Boundary drain (#2409): checkpoint at the end of the incremental @@ -2846,6 +2964,7 @@ async function runFullAnalysisInner( // pre-existing row (#2544/#2546) must not discard this run's otherwise- // successful graph/embeddings work — only keyword search degrades. const ftsResult = await buildSearchIndexesOrDegrade(executeQuery, { + tables: incrementalFtsRebuildTables, onIndexStart: options.verbose ? (table, indexName) => log(`FTS: creating ${table}.${indexName}`) : undefined, @@ -3600,8 +3719,11 @@ async function runFullAnalysisInner( files: pipelineResult.totalFileCount, nodes: stats.nodes, edges: stats.edges, - communities: pipelineResult.communityResult?.stats.totalCommunities, - processes: pipelineResult.processResult?.stats.totalProcesses, + communities: + pipelineResult.communityResult?.stats.totalCommunities ?? + existingMeta?.stats?.communities, + processes: + pipelineResult.processResult?.stats.totalProcesses ?? existingMeta?.stats?.processes, embeddings: persistedEmbeddingCount, }, capabilities: { @@ -3830,9 +3952,12 @@ async function runFullAnalysisInner( files: pipelineResult.totalFileCount, nodes: stats.nodes, edges: stats.edges, - communities: pipelineResult.communityResult?.stats.totalCommunities, + communities: + pipelineResult.communityResult?.stats.totalCommunities ?? + existingMeta?.stats?.communities, clusters: aggregatedClusterCount, - processes: pipelineResult.processResult?.stats.totalProcesses, + processes: + pipelineResult.processResult?.stats.totalProcesses ?? existingMeta?.stats?.processes, }, undefined, { diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index b53a96778..951734d4e 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -158,6 +158,12 @@ export const SUPPORTED_FTS_STEMMERS: ReadonlySet = new Set([ export interface CreateSearchFTSIndexesOptions { onIndexStart?: (table: string, indexName: string) => void; onIndexReady?: (table: string, indexName: string) => void; + /** + * When set, only these node-table names are dropped/rebuilt (#3016). + * Omit to rebuild every configured FTS index (full analyze / deleted-file + * incremental / `--repair-fts`). + */ + tables?: ReadonlySet; } let resolvedStemmer: string | undefined; @@ -219,7 +225,10 @@ export function getSearchFTSStemmer(): string { * contract, and the same one-shared-`SHOW_INDEXES`-read purpose, as the gates in * `lbug-adapter.ts`. Omit it to have the sweep read the catalog itself. */ -export async function dropSearchFTSIndexes(indexRows?: IndexCatalogSnapshot): Promise { +export async function dropSearchFTSIndexes( + indexRows?: IndexCatalogSnapshot, + tables?: ReadonlySet, +): Promise { // One catalog read for the whole sweep, decided PER CONFIGURED INDEX on // IDENTITY (#2841 cleanup review). `undefined` = the catalog could not be // read, which proves nothing — attempt every drop rather than skip a real one, @@ -240,6 +249,7 @@ export async function dropSearchFTSIndexes(indexRows?: IndexCatalogSnapshot): Pr // whether the sweep ran or not. const rows = await resolveGateRows(indexRows); for (const { table, indexName } of FTS_INDEXES) { + if (tables && !tables.has(table)) continue; // Skip only what the catalog POSITIVELY proves absent. Without this, a // machine whose FTS extension cannot load, analyzing a DB that never carried // an FTS index, pays one failed `CALL DROP_FTS_INDEX` per configured table on @@ -257,6 +267,32 @@ export async function dropSearchFTSIndexes(indexRows?: IndexCatalogSnapshot): Pr } } +/** + * The configured FTS tables whose index the catalog proves is ABSENT right now. + * + * `undefined` means the catalog could not be read, which proves nothing — the + * same fail-closed reading the sweep above applies. Callers narrowing a rebuild + * to a subset of tables (#3016) must union this in, or must not narrow at all + * when it is `undefined`: a run that rebuilds only the tables it wrote leaves + * keyword search permanently degraded on every table whose index went missing + * earlier (a prior escalation drops all of them, and only the next full rebuild + * would ever put them back). + */ +export async function missingSearchFTSIndexTables( + indexRows?: IndexCatalogSnapshot, +): Promise | undefined> { + const rows = await resolveGateRows(indexRows); + if (rows === undefined) return undefined; + const missing = new Set(); + for (const { table, indexName } of FTS_INDEXES) { + const present = rows.some( + (row) => indexRowTable(row) === table && indexRowName(row) === indexName, + ); + if (!present) missing.add(table); + } + return missing; +} + /** One configured index that could not be (re)built, and why. */ export interface FtsIndexBuildFailure { table: string; @@ -290,6 +326,7 @@ export async function createSearchFTSIndexes( const stemmer = getSearchFTSStemmer(); const failures: FtsIndexBuildFailure[] = []; for (const { table, indexName, properties } of FTS_INDEXES) { + if (options?.tables && !options.tables.has(table)) continue; options?.onIndexStart?.(table, indexName); // Drop first so the live `properties` always win. `createFTSIndex` is // idempotent-by-name (skips when the index already exists), so without the diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index 5c11f800b..960517416 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -15,6 +15,18 @@ export interface PipelineResult { totalFileCount: number; communityResult?: CommunityDetectionResult; processResult?: ProcessDetectionResult; + /** + * Runs the community/process phases that `skipDerivedGraphPhases` held back + * (#3016), against the same graph and phase outputs the pipeline already + * produced, and populates `communityResult`/`processResult` on this object. + * + * Present ONLY when those phases were skipped for that reason, so a caller + * that optimistically skipped them can still get a byte-identical derived + * layer on the paths that turn out to need one (full rebuild, escalated + * write, or an incremental run with deleted files). Absent means the phases + * either already ran or were disabled for an unrelated reason. + */ + runDeferredDerivedPhases?: () => Promise; /** * Additive diagnostics for registry-primary resolution decisions that * deliberately suppress edge emission. Empty means no diagnostic was diff --git a/gitnexus/test/unit/fts-indexes.test.ts b/gitnexus/test/unit/fts-indexes.test.ts index c3cfea224..915abc5db 100644 --- a/gitnexus/test/unit/fts-indexes.test.ts +++ b/gitnexus/test/unit/fts-indexes.test.ts @@ -33,6 +33,7 @@ const { createSearchFTSIndexes, getSearchFTSStemmer, initialiseSearchFTSStemmer, + missingSearchFTSIndexTables, } = await import('../../src/core/search/fts-indexes.js'); const { FTS_INDEXES } = await import('../../src/core/search/fts-schema.js'); const { createFTSIndex } = await import('../../src/core/lbug/lbug-adapter.js'); @@ -65,6 +66,16 @@ describe('createSearchFTSIndexes', () => { expect(calls).toEqual(expected); }); + it('rebuilds only the requested tables when options.tables is set (#3016)', async () => { + await createSearchFTSIndexes({ tables: new Set(['File', 'Function']) }); + expect(calls).toEqual([ + 'drop:File.file_fts', + 'create:File.file_fts:porter', + 'drop:Function.function_fts', + 'create:Function.function_fts:porter', + ]); + }); + it('invokes onIndexStart/onIndexReady once per index', async () => { const started: string[] = []; const ready: string[] = []; @@ -196,6 +207,32 @@ describe('buildSearchIndexesOrDegrade', () => { }); }); +describe('missingSearchFTSIndexTables (#3016)', () => { + const catalogRow = (i: { table: string; indexName: string }) => ({ + table_name: i.table, + index_name: i.indexName, + }); + + it('reports nothing missing when the catalog carries every configured index', async () => { + const missing = await missingSearchFTSIndexTables(FTS_INDEXES.map(catalogRow)); + expect(missing).toEqual(new Set()); + }); + + it('names every table when the catalog is empty (a prior escalation dropped them all)', async () => { + const missing = await missingSearchFTSIndexTables([]); + expect(missing).toEqual(new Set(FTS_INDEXES.map((i) => i.table))); + }); + + it('names only the tables whose index is absent', async () => { + const rows = FTS_INDEXES.filter((i) => i.table !== 'Function').map(catalogRow); + expect(await missingSearchFTSIndexTables(rows)).toEqual(new Set(['Function'])); + }); + + it('answers undefined when the catalog could not be read, so callers do not narrow', async () => { + expect(await missingSearchFTSIndexTables(undefined)).toBeUndefined(); + }); +}); + describe('getSearchFTSStemmer', () => { it('defaults to porter when unset', () => { expect(getSearchFTSStemmer()).toBe('porter'); diff --git a/gitnexus/test/unit/incremental-derived-writeback.test.ts b/gitnexus/test/unit/incremental-derived-writeback.test.ts new file mode 100644 index 000000000..36242eac2 --- /dev/null +++ b/gitnexus/test/unit/incremental-derived-writeback.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; +import type { GraphNode } from 'gitnexus-shared'; +import { NODE_TABLES } from 'gitnexus-shared'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { + ftsTablesAmong, + incrementalFtsTablesFromGraph, + nodeTablesForIncrementalDelete, + shouldPreservePersistedDerivedGraph, +} from '../../src/core/incremental/derived-writeback.js'; + +const node = (id: string, label: string, filePath: string): GraphNode => + ({ + id, + label, + properties: { filePath, name: id }, + }) as unknown as GraphNode; + +describe('shouldPreservePersistedDerivedGraph (#3016)', () => { + const empty = { deleted: [] as string[], added: [] as string[], changed: [] as string[] }; + + it('is true only when the file-hash diff is empty', () => { + expect(shouldPreservePersistedDerivedGraph(empty)).toBe(true); + }); + + it('is false when any file was deleted (old labels are not in the fresh graph)', () => { + expect(shouldPreservePersistedDerivedGraph({ ...empty, deleted: ['gone.ts'] })).toBe(false); + }); + + it('is false when a file was added (new symbols have no persisted membership)', () => { + expect(shouldPreservePersistedDerivedGraph({ ...empty, added: ['new.ts'] })).toBe(false); + }); + + it('is false when a file changed (in-file add/rename/CALLS can change Leiden/flows)', () => { + expect(shouldPreservePersistedDerivedGraph({ ...empty, changed: ['a.ts'] })).toBe(false); + }); +}); + +describe('incrementalFtsTablesFromGraph', () => { + it('returns only FTS tables that have write-set nodes', () => { + const g = createKnowledgeGraph(); + g.addNode(node('f', 'File', 'a.ts')); + g.addNode(node('fn', 'Function', 'a.ts')); + g.addNode(node('tr', 'Trait', 'b.rs')); + const touched = incrementalFtsTablesFromGraph(g, new Set(['a.ts'])); + expect([...touched].sort()).toEqual(['File', 'Function']); + }); + + it('ignores labels that are not FTS-indexed', () => { + const g = createKnowledgeGraph(); + g.addNode(node('folder', 'Folder', 'src')); + const touched = incrementalFtsTablesFromGraph(g, new Set(['src'])); + expect(touched.size).toBe(0); + }); + + it('cannot see a table whose last row the edit removed — hence the DB probe', () => { + // The graph is what the run WILL write. A trait deleted by this edit is + // absent here but still a row in the index, so on its own this answer + // would leave that row behind with a live index over it. run-analyze + // unions this with nodeTablesWithRowsForFiles for exactly that reason. + const g = createKnowledgeGraph(); + g.addNode(node('f', 'File', 'a.rs')); + const touched = incrementalFtsTablesFromGraph(g, new Set(['a.rs'])); + expect(touched.has('Trait')).toBe(false); + }); +}); + +describe('ftsTablesAmong', () => { + it('keeps the FTS-backed tables and drops the rest', () => { + expect([...ftsTablesAmong(['File', 'Folder', 'Function'])].sort()).toEqual([ + 'File', + 'Function', + ]); + }); + + it('is empty for a probe that found only non-indexed tables', () => { + expect(ftsTablesAmong(['Folder']).size).toBe(0); + }); +}); + +describe('nodeTablesForIncrementalDelete', () => { + it('keeps the FTS tables being rebuilt and drops the rest from the delete', () => { + const tables = nodeTablesForIncrementalDelete(NODE_TABLES, new Set(['File', 'Function'])); + expect(tables).toContain('File'); + expect(tables).toContain('Function'); + expect(tables).not.toContain('Trait'); + }); + + it('never withholds a non-FTS table, whatever is being rebuilt', () => { + const tables = nodeTablesForIncrementalDelete(NODE_TABLES, new Set(['File'])); + expect(tables).toContain('Folder'); + }); + + it('targets every FTS table when every FTS index is being rebuilt', () => { + const tables = nodeTablesForIncrementalDelete(NODE_TABLES, new Set(NODE_TABLES)); + expect(tables).toEqual([...NODE_TABLES]); + }); +}); diff --git a/gitnexus/test/unit/incremental-fts-drop-ordering.test.ts b/gitnexus/test/unit/incremental-fts-drop-ordering.test.ts index e2f5c3c87..0fa8c6af3 100644 --- a/gitnexus/test/unit/incremental-fts-drop-ordering.test.ts +++ b/gitnexus/test/unit/incremental-fts-drop-ordering.test.ts @@ -5,8 +5,8 @@ * PREVIOUS run's index. This drives the real `runFullAnalysis` incremental * path (real git repo, real LadybugDB, real FTS extension) and asserts, * at the moment `deleteNodesForFiles` is invoked, that `SHOW_INDEXES()` - * already reports every FTS index absent — proving the drop-before-delete - * ordering end-to-end rather than only unit-testing the call sequence. + * already reports FTS indexes for tables that will be DML'd as absent + * (#2589 drop-before-delete). #3016: empty-language FTS tables may remain. */ import { readFile, writeFile } from 'fs/promises'; import { execSync } from 'child_process'; @@ -67,7 +67,7 @@ describe('runFullAnalysis incremental writeback — FTS drop-before-delete order vi.resetModules(); }); - it('SHOW_INDEXES() reports every FTS index absent by the time deleteNodesForFiles runs', async () => { + it('SHOW_INDEXES() reports the FTS indexes of every table being written as absent by the time deleteNodesForFiles runs', async () => { const lbugAdapter = await import('../../src/core/lbug/lbug-adapter.js'); const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); @@ -128,8 +128,24 @@ describe('runFullAnalysis incremental writeback — FTS drop-before-delete order await runFullAnalysis(repo.dbPath, { skipAgentsMd: true }, { onProgress: () => {} }); expect(indexNamesAtDeleteTime).toBeDefined(); - for (const { indexName } of FTS_INDEXES) { - expect(indexNamesAtDeleteTime).not.toContain(indexName); + // handler.ts writes File/Function/Class/Method. The incremental write set + // also pulls importer-expanded mini-repo files (index.ts re-exports + // handler; validator.ts holds Interface + Property), so those FTS + // indexes must be down before DETACH DELETE (#2589). Every other + // configured FTS index must still be live (#3016 narrowing). + const down = new Set([ + 'file_fts', + 'function_fts', + 'class_fts', + 'method_fts', + 'interface_fts', + 'property_fts', + ]); + const configured = FTS_INDEXES.map((i) => i.indexName); + const ftsAtDelete = indexNamesAtDeleteTime!.filter((name) => configured.includes(name)); + expect([...ftsAtDelete].sort()).toEqual(configured.filter((name) => !down.has(name)).sort()); + for (const name of down) { + expect(ftsAtDelete).not.toContain(name); } } finally { await repo.cleanup(); diff --git a/gitnexus/test/unit/incremental-subgraph-extract.test.ts b/gitnexus/test/unit/incremental-subgraph-extract.test.ts index 4841bba3a..c1b3a2cf5 100644 --- a/gitnexus/test/unit/incremental-subgraph-extract.test.ts +++ b/gitnexus/test/unit/incremental-subgraph-extract.test.ts @@ -74,6 +74,19 @@ describe('extractChangedSubgraph', () => { expect(sub.nodes.map((n) => n.id).sort()).toEqual(['comm-1', 'proc-1']); }); + it('omits Community/Process when includeDerivedGraphWide is false (#3016)', () => { + const g = createKnowledgeGraph(); + g.addNode(makeFileNode('a', '/repo/a.ts')); + g.addNode(makeWideNode('comm-1', 'Community')); + g.addNode(makeWideNode('proc-1', 'Process')); + + const sub = extractChangedSubgraph(g, new Set(['/repo/a.ts']), { + includeDerivedGraphWide: false, + }); + + expect(sub.nodes.map((n) => n.id).sort()).toEqual(['a']); + }); + it('always includes Spring auto-configuration synthetic Class nodes', () => { const g = createKnowledgeGraph(); g.addNode({ diff --git a/gitnexus/test/unit/ingestion/pipeline-phase-registry.test.ts b/gitnexus/test/unit/ingestion/pipeline-phase-registry.test.ts index 5b8c446f9..913ba68a4 100644 --- a/gitnexus/test/unit/ingestion/pipeline-phase-registry.test.ts +++ b/gitnexus/test/unit/ingestion/pipeline-phase-registry.test.ts @@ -109,6 +109,30 @@ describe('buildPhaseList parity (registry refactor, #2080)', () => { WITHOUT_GRAPH_PHASES, ); }); + + it('skipDerivedGraphPhases:true → omits communities/processes but keeps mro/di (#3016)', () => { + const names = buildPhaseList({ skipDerivedGraphPhases: true }).map((p) => p.name); + expect(names).toContain('mro'); + expect(names).toContain('di'); + expect(names).not.toContain('communities'); + expect(names).not.toContain('processes'); + }); + + it('skipDerivedGraphPhases holds back exactly the two derived phases (#3016)', () => { + // runPipelineFromRepo recovers the deferred set by diffing these two lists, + // so anything else the flag removed would be silently un-deferrable. + const skipped = buildPhaseList({ skipDerivedGraphPhases: true }).map((p) => p.name); + const full = buildPhaseList({ skipDerivedGraphPhases: false }).map((p) => p.name); + expect(full.filter((n) => !skipped.includes(n))).toEqual(['communities', 'processes']); + }); + + it('skipDerivedGraphPhases defers nothing that skipGraphPhases already removed (#3016)', () => { + const both = buildPhaseList({ skipGraphPhases: true, skipDerivedGraphPhases: true }).map( + (p) => p.name, + ); + const graphPhasesOnly = buildPhaseList({ skipGraphPhases: true }).map((p) => p.name); + expect(both).toEqual(graphPhasesOnly); + }); }); // --------------------------------------------------------------------------- diff --git a/gitnexus/test/unit/pipeline-runner.test.ts b/gitnexus/test/unit/pipeline-runner.test.ts index bddde7bd1..c29fdd7f7 100644 --- a/gitnexus/test/unit/pipeline-runner.test.ts +++ b/gitnexus/test/unit/pipeline-runner.test.ts @@ -400,6 +400,78 @@ describe('runPipeline', () => { }); }); +describe('runPipeline with seeded results (#3016 deferred derived phases)', () => { + const seeded = (name: string, output: unknown): ReadonlyMap> => + new Map([[name, { phaseName: name, output, durationMs: 0 }]]); + + it('satisfies a dependency from the seed instead of demanding the phase', async () => { + const later: PipelinePhase = { + name: 'later', + deps: ['earlier'], + async execute(_ctx, deps) { + return `${getPhaseOutput(deps, 'earlier')}+later`; + }, + }; + + const results = await runPipeline([later], makeCtx(), seeded('earlier', 'earlierOutput')); + + expect(getPhaseOutput(results, 'later')).toBe('earlierOutput+later'); + }); + + it('returns the seeded results alongside the newly run ones', async () => { + const later: PipelinePhase = { + name: 'later', + deps: ['earlier'], + execute: async () => 'x', + }; + + const results = await runPipeline([later], makeCtx(), seeded('earlier', 'earlierOutput')); + + expect([...results.keys()].sort()).toEqual(['earlier', 'later']); + }); + + it('does not re-run a seeded phase', async () => { + let ran = 0; + const earlier: PipelinePhase = { + name: 'earlier', + deps: [], + async execute() { + ran++; + return 'fresh'; + }, + }; + + await runPipeline([earlier], makeCtx(), seeded('earlier', 'seeded')); + + // The seed already carries this phase's output, so the runner must treat it + // as a duplicate registration rather than silently executing it twice. + expect(ran).toBe(0); + }); + + it('still rejects a dependency that is neither registered nor seeded', async () => { + const later: PipelinePhase = { + name: 'later', + deps: ['missing'], + execute: async () => 'x', + }; + + await expect(runPipeline([later], makeCtx(), seeded('earlier', 'e'))).rejects.toThrow( + /depends on 'missing', which is not registered/, + ); + }); + + it('rejects duplicate phase names even when one copy is also seeded', async () => { + const dup: PipelinePhase = { + name: 'earlier', + deps: [], + async execute() {}, + }; + await expect(runPipeline([dup, dup], makeCtx(), seeded('earlier', 'seed'))).rejects.toThrow( + /Duplicate phase name/, + ); + }); +}); + describe('getPhaseOutput', () => { it('retrieves typed output from dependency map', () => { const deps = new Map>(); From 72edf400871c1589ceb975ad868909389249606a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sun, 30 Aug 2026 21:50:20 +0100 Subject: [PATCH 28/28] perf(store): V8 sidecars plus hardlinked ParsedFile restore (#3099) * perf(store): add best-effort V8 sidecars beside canonical JSON caches Warm ParsedFile and parse-cache loads skip JSON.parse when a sidecar is present. JSON remains authoritative: envelope validation plus v8.deserialize decide the hit, and any failure falls back without reparsing. Co-authored-by: Cursor * fix(store): require generation bind or sidecar drop before cache overwrite A same-length JSON rewrite could accept a leftover V8 sidecar if both generation rotation and unlink failed. Refuse the new generation unless at least one of those invalidations succeeds; skip publishing a sidecar when only the drop succeeded. detect_changes --scope all: 7 files, risk low, no affected processes. tsc --noEmit clean; 115/115 relevant unit tests; cache-related integration tests pass. parse-impl-env-reads worker-ready timeout is pre-existing (same 5 failures with this change set stashed). ESLint 0 errors; remaining warnings are pre-existing and not on changed lines. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command * refactor(store): share V8 overwrite invalidation across persist paths The bind-or-drop gate lived in five writers. One helper keeps the protocol in a single place and lets bind/drop run together on the async path. detect_changes --scope all: 3 files, risk low, no affected processes. Co-authored-by: Cursor * perf(store): hardlink durable ParsedFile shards into the run store Warm restore of parsedfile-cache into parsedfile-store now publishes all four shard files via fs.link, falling back to copy-into-tmp + rename so a leftover dest hardlink can never be written through. JSON remains the canonical cache; V8 sidecars ride the same path. Co-authored-by: Cursor * perf(store): load immutable V8 shards in place, drop JSON fallback Warm analyze was still paying JSON.parse plus a restore copy. One .v8 envelope per shard and SCHEMA_BUMP 81 make a miss re-extract instead of serving a stale JSON twin. Co-authored-by: Cursor * fix(store): validate durable V8 warm-cache restores Reject incomplete or corrupt durable generations and snapshot valid shards before skipping parse workers, preserving ParsedFiles when persistence fails. Co-authored-by: Cursor * refactor(store): drop unused durable load path Load ParsedFiles only from the run-store snapshot and share one checksummed payload reader so inspect and deserialize stay consistent. Co-authored-by: Cursor --------- Co-authored-by: Gergo Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- gitnexus/bench/v8-sidecar/measure.mjs | 91 ++++ .../ingestion/pipeline-phases/parse-impl.ts | 45 +- .../core/ingestion/workers/parse-worker.ts | 6 +- gitnexus/src/storage/fs-atomic.ts | 89 +++- gitnexus/src/storage/parse-cache.ts | 49 +- gitnexus/src/storage/parsedfile-store.ts | 471 ++++++------------ gitnexus/src/storage/v8-sidecar.ts | 363 ++++++++++++++ .../integration/cfg/parse-cache-mixed.test.ts | 27 +- .../test/unit/incremental-parse-cache.test.ts | 147 +++++- ...mpl-warm-cache-parsedfile-coverage.test.ts | 200 ++++++-- gitnexus/test/unit/parsedfile-store.test.ts | 306 +++++------- ...repo-manager-registry-atomic-write.test.ts | 22 +- gitnexus/test/unit/storage/fs-atomic.test.ts | 128 ++++- gitnexus/test/unit/v8-sidecar.test.ts | 204 ++++++++ 14 files changed, 1510 insertions(+), 638 deletions(-) create mode 100644 gitnexus/bench/v8-sidecar/measure.mjs create mode 100644 gitnexus/src/storage/v8-sidecar.ts create mode 100644 gitnexus/test/unit/v8-sidecar.test.ts diff --git a/gitnexus/bench/v8-sidecar/measure.mjs b/gitnexus/bench/v8-sidecar/measure.mjs new file mode 100644 index 000000000..68867b9ba --- /dev/null +++ b/gitnexus/bench/v8-sidecar/measure.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node +/** + * Optional V8 sidecar warm-load bench (#3089). + * + * Not part of `npm test`. Measures repeated warm loads of the `.v8` ParsedFile + * shards already on disk through the production loader. Replay of identical + * shards is throughput-only — it is not unique-object scale. + * + * Copies the store into a temporary workspace first. The source cache is + * never mutated. + * + * Usage (from gitnexus/): + * node --expose-gc --import tsx bench/v8-sidecar/measure.mjs + */ +import { cp, mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { loadParsedFilesForPaths } from '../../src/storage/parsedfile-store.ts'; +import { inspectV8Cache } from '../../src/storage/v8-sidecar.ts'; + +const srcStorage = process.argv[2]; +if (!srcStorage) { + console.error('usage: node --expose-gc --import tsx bench/v8-sidecar/measure.mjs '); + process.exit(2); +} + +const srcStoreDir = path.join(srcStorage, 'parsedfile-store'); +const benchRoot = await mkdtemp(path.join(tmpdir(), 'gnx-v8-bench-')); +const storeDir = path.join(benchRoot, 'parsedfile-store'); +const PATH_SOURCE_SHARDS = 8; +const RUNS = 3; + +try { + await cp(srcStoreDir, storeDir, { recursive: true }); + + const names = (await readdir(storeDir)) + .filter((f) => f.endsWith('.v8') && !f.includes('.v8.')) + .sort(); + if (names.length === 0) { + throw new Error( + `no .v8 ParsedFile shards in ${srcStoreDir} — run an analyze that populates the store first`, + ); + } + + const want = new Set(); + let sourceShards = 0; + for (const name of names) { + const inspected = await inspectV8Cache(path.join(storeDir, name)); + if (!inspected) continue; + sourceShards++; + for (const filePath of inspected.paths) want.add(filePath); + if (sourceShards >= PATH_SOURCE_SHARDS) break; + } + if (want.size === 0) { + throw new Error( + `no file paths readable from ${names.length} shard(s) in ${srcStoreDir} — shards may be from another Node/V8 runtime, so re-analyze with this runtime`, + ); + } + + const rss = () => Math.round(process.memoryUsage().rss / 1024 / 1024); + const heap = () => Math.round(process.memoryUsage().heapUsed / 1024 / 1024); + + const run = async (label) => { + if (typeof globalThis.gc === 'function') globalThis.gc(); + const t0 = performance.now(); + const loaded = await loadParsedFilesForPaths(benchRoot, want); + const ms = Math.round(performance.now() - t0); + if (loaded.size !== want.size) { + throw new Error(`incomplete V8 load: requested ${want.size} paths but loaded ${loaded.size}`); + } + if (typeof globalThis.gc === 'function') globalThis.gc(); + console.log( + JSON.stringify({ + label, + shards: names.length, + wantPaths: want.size, + files: loaded.size, + ms, + rssMiB: rss(), + heapUsedMiB: heap(), + }), + ); + }; + + for (let i = 1; i <= RUNS; i++) { + await run(`v8-load-${i}`); + } +} finally { + await rm(benchRoot, { recursive: true, force: true }); +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index b6ca75ea0..352fd0f2b 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -36,7 +36,7 @@ import { getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, - restoreDurableParsedFileShard, + durableChunkHasShards, } from '../../../storage/parsedfile-store.js'; import type { ParseWorkerResult } from '../workers/parse-worker.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js'; @@ -739,15 +739,15 @@ export async function runChunkedParseAndResolve( // a sibling of the run-scoped store, NOT cleared per run. Workers write a // shard per chunk hash; on a warm parse-cache hit we restore the chunk's // shards into the run-scoped store so scope-resolution streams them without - // re-parsing. `durableHitKeys` is the prior run's index, version-gated by - // PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk re-dispatches, which - // repopulates the durable store — never the main-thread extract fallback). + // re-parsing. `durableHitEntries` is the prior run's path-coverage index, + // version-gated by PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk + // re-dispatches, which repopulates the durable store). const durableParsedFileDir = parsedFileStorePath !== undefined ? getDurableParsedFileDir(parsedFileStorePath) : undefined; - const durableHitKeys = + const durableHitEntries = durableParsedFileDir !== undefined ? await loadDurableParsedFileIndex(durableParsedFileDir, PARSE_CACHE_VERSION) - : new Set(); + : new Map>(); let chunkCacheHits = 0; let chunkCacheMisses = 0; let reparsedFileCount = 0; @@ -825,11 +825,14 @@ export async function runChunkedParseAndResolve( } if (chunkWorkerData.parsedFiles?.length) { if (parsedFileStorePath) { - await persistParsedFileChunk( + const wrote = await persistParsedFileChunk( parsedFileStorePath, `chunk-${chunkIdx}`, chunkWorkerData.parsedFiles, ); + if (!wrote) { + for (const item of chunkWorkerData.parsedFiles) allParsedFiles.push(item); + } } else { for (const item of chunkWorkerData.parsedFiles) allParsedFiles.push(item); } @@ -1019,8 +1022,16 @@ export async function runChunkedParseAndResolve( // store was introduced, or a pruned/version-stale shard — fall through to // a worker re-dispatch to repopulate them. NEVER let scope-resolution // re-extract on the main thread (the #1983 OOM the durable store closes). + const durableExpectedPaths = + chunkHash === null ? undefined : durableHitEntries.get(chunkHash); const durableHit = - chunkHash !== null && durableParsedFileDir !== undefined && durableHitKeys.has(chunkHash); + cachedRaw !== undefined && + cachedRaw.length > 0 && + chunkHash !== null && + durableParsedFileDir !== undefined && + parsedFileStorePath !== undefined && + durableExpectedPaths !== undefined && + (await durableChunkHasShards(parsedFileStorePath, chunkHash, durableExpectedPaths)); if (cachedRaw && cachedRaw.length > 0 && (durableHit || parsedFileStorePath === undefined)) { // Cache hit: replay cached worker output. Finalize any parked worker @@ -1053,22 +1064,8 @@ export async function runChunkedParseAndResolve( nodesCreated: graph.nodeCount, }, }); - // Restore the chunk's durable ParsedFile shards into the run-scoped - // store so scope-resolution finds full coverage with ZERO main-thread - // re-parse. A verbatim byte copy — byte-identical to a cold run. - if (durableHit && durableParsedFileDir && parsedFileStorePath && chunkHash) { - const restored = await restoreDurableParsedFileShard( - durableParsedFileDir, - parsedFileStorePath, - chunkHash, - ); - if (restored === 0) { - logger.warn( - `parsedfile-cache: durable shards missing for cached chunk ` + - `${chunkHash.slice(0, 8)} — scope-resolution will re-extract these files`, - ); - } - } + // The durable gate already snapshotted warm `.v8` shards into the + // run-scoped store for scope resolution. await applyChunkResults(chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs); } else { // Cache miss: dispatch to workers, capture the raw results, store diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 29c94e8ee..ab27cf7ac 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -3243,12 +3243,14 @@ parentPort!.on('message', (msg: WorkerIncomingMessage) => { ); } if (PARSED_FILE_STORE_STORAGE_PATH) { - persistParsedFileShardSync( + const wrote = persistParsedFileShardSync( PARSED_FILE_STORE_STORAGE_PATH, `w${threadId}-${seq}`, accumulated.parsedFiles, ); - accumulated.parsedFiles = []; + if (wrote) { + accumulated.parsedFiles = []; + } } } postResultCloneSafe(accumulated); diff --git a/gitnexus/src/storage/fs-atomic.ts b/gitnexus/src/storage/fs-atomic.ts index 7f3070614..e39053fd5 100644 --- a/gitnexus/src/storage/fs-atomic.ts +++ b/gitnexus/src/storage/fs-atomic.ts @@ -6,7 +6,7 @@ * core/group/ import (the established direction is core/group/ -> storage/, * e.g. core/group/service.ts already imports loadMeta from here). */ -import fsp from 'fs/promises'; +import { closeSync, openSync, promises as fsp, renameSync, unlinkSync, writeSync } from 'node:fs'; import { randomBytes } from 'crypto'; const RETRY_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']); @@ -57,7 +57,7 @@ export async function writeFileAtomic( data: string, attempts?: number, ): Promise { - const tmpPath = `${targetPath}.tmp.${randomBytes(8).toString('hex')}`; + const tmpPath = tmpBeside(targetPath); const handle = await fsp.open(tmpPath, 'wx', 0o600); try { try { @@ -71,3 +71,88 @@ export async function writeFileAtomic( throw err; } } + +function tmpBeside(targetPath: string): string { + return `${targetPath}.tmp.${randomBytes(8).toString('hex')}`; +} + +/** + * Publish `src` at `dst` without ever writing through the inode currently + * named by `dst` (#3090). Fast path is `fs.link`. Any failure (EEXIST, EXDEV, + * EPERM, unsupported hardlinks) copies into a unique sibling and + * {@link retryRename}s over `dst`. POSIX rename of a file over an existing + * file unlinks the old name; it does not open or truncate that inode. + * + * Invariant: may replace the destination directory entry; must never modify + * the inode that entry currently references. There is no dest unlink and no + * in-place `copyFile(src, dst)`. + */ +export async function linkOrCopyFile(src: string, dst: string): Promise { + try { + await fsp.link(src, dst); + return; + } catch { + /* any link failure → publish a new inode via tmp + rename */ + } + const tmp = tmpBeside(dst); + try { + await fsp.copyFile(src, tmp); + await retryRename(tmp, dst); + } catch (err) { + await fsp.unlink(tmp).catch(() => {}); + throw err; + } +} + +/** + * Binary sibling of {@link writeFileAtomic}. Same random tmp + `'wx'` + `0o600` + * + rename contract; used for optional V8 cache sidecars that cannot go through + * a UTF-8 `writeFile`. + */ +export async function writeFileAtomicBytes( + targetPath: string, + data: Uint8Array, + attempts?: number, +): Promise { + const tmpPath = tmpBeside(targetPath); + const handle = await fsp.open(tmpPath, 'wx', 0o600); + try { + try { + await handle.writeFile(data); + } finally { + await handle.close(); + } + await retryRename(tmpPath, targetPath, attempts); + } catch (err) { + await fsp.unlink(tmpPath).catch(() => {}); + throw err; + } +} + +/** + * Sync binary publish for parse workers. Same exclusive-tmp + mode contract as + * {@link writeFileAtomicBytes}; rename is not retried (the worker is not the + * Windows multi-reader case `retryRename` exists for). + */ +export function writeFileAtomicBytesSync(targetPath: string, data: Uint8Array): void { + const tmpPath = tmpBeside(targetPath); + const fd = openSync(tmpPath, 'wx', 0o600); + try { + try { + let offset = 0; + while (offset < data.byteLength) { + offset += writeSync(fd, data, offset); + } + } finally { + closeSync(fd); + } + renameSync(tmpPath, targetPath); + } catch (err) { + try { + unlinkSync(tmpPath); + } catch { + /* leftover tmp is unlinked on the next exclusive create */ + } + throw err; + } +} diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index c99f4d87e..efd17a7f6 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -31,6 +31,7 @@ import path from 'path'; import { fileURLToPath } from 'url'; import { compareCodeUnits } from '../lib/utils.js'; import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.js'; +import { copyV8CacheIfPresent, tryLoadV8Cache, writeV8CacheFile } from './v8-sidecar.js'; /** * Cache version composed of: @@ -663,7 +664,11 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // `route-extractors/` and `workers/` module content — would close the missing- // bump axis without invalidating on unrelated churn, and is the real follow-up. // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING. -const SCHEMA_BUMP = 80; +// 80 -> 81: ParsedFile and parse-cache shards are one immutable `.v8` envelope +// each (no JSON/path/generation siblings). A v80 index still names `.json` +// keys and would skip workers while scope-resolution found nothing — the +// #1983 main-thread reparse. origin/main at allocation is 80. +const SCHEMA_BUMP = 81; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from @@ -899,7 +904,7 @@ const getCacheIndexPath = (storagePath: string): string => path.join(getCacheDirPath(storagePath), CACHE_INDEX_FILENAME); const getCacheChunkPath = (storagePath: string, chunkHash: string): string => - path.join(getCacheDirPath(storagePath), `${chunkHash}.json`); + path.join(getCacheDirPath(storagePath), `${chunkHash}.v8`); /** * Drop fields that are not replayed by `mergeChunkResults` / parse-impl after @@ -930,9 +935,12 @@ const readParseCacheChunkFromDisk = async ( ): Promise => { if (!isValidChunkCacheKey(chunkHash)) return undefined; try { - const chunkRaw = await fs.readFile(getCacheChunkPath(storagePath, chunkHash), 'utf-8'); - const chunkData = JSON.parse(chunkRaw, mapReviver) as ParseWorkerResult[]; - return Array.isArray(chunkData) ? chunkData : undefined; + const chunkPath = getCacheChunkPath(storagePath, chunkHash); + const v8Hit = await tryLoadV8Cache(chunkPath); + if (v8Hit?.kind === 'hit' && Array.isArray(v8Hit.value)) { + return v8Hit.value as ParseWorkerResult[]; + } + return undefined; } catch { return undefined; } @@ -975,17 +983,16 @@ export const persistParseCacheChunk = async ( await fs.mkdir(cacheDir, { recursive: true }); createdCacheDirs.add(cacheDir); } - const payload = JSON.stringify(slim, mapReplacer); const chunkPath = getCacheChunkPath(cache.storagePath, chunkHash); - try { - await fs.writeFile(chunkPath, payload, 'utf-8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - // Long-lived analyze --watch processes can replace the sharded cache - // directory after this process-local memo recorded it as created. + let ok = await writeV8CacheFile(chunkPath, slim); + if (!ok) { await fs.mkdir(cacheDir, { recursive: true }); createdCacheDirs.add(cacheDir); - await fs.writeFile(chunkPath, payload, 'utf-8'); + ok = await writeV8CacheFile(chunkPath, slim); + } + if (!ok) { + cache.entries.set(chunkHash, slim); + return; } cache.onDiskKeys ??= new Set(); cache.onDiskKeys.add(chunkHash); @@ -1089,25 +1096,17 @@ export const saveParseCache = async (storagePath: string, cache: ParseCache): Pr // index from what we persisted, not from the raw usedKeys snapshot. const writtenKeys: string[] = []; for (const chunkHash of keys) { - const chunkPath = path.join(tmpDir, `${chunkHash}.json`); + const chunkPath = path.join(tmpDir, `${chunkHash}.v8`); const inMemory = cache.entries.get(chunkHash); if (inMemory !== undefined) { - let payload: string; - try { - payload = JSON.stringify(inMemory, mapReplacer); - } catch { - continue; + if (await writeV8CacheFile(chunkPath, inMemory)) { + writtenKeys.push(chunkHash); } - await fs.writeFile(chunkPath, payload, 'utf-8'); - writtenKeys.push(chunkHash); continue; } const existingPath = getCacheChunkPath(storagePath, chunkHash); - try { - await fs.copyFile(existingPath, chunkPath); + if (await copyV8CacheIfPresent(existingPath, chunkPath)) { writtenKeys.push(chunkHash); - } catch { - /* shard missing — skip; next run treats as cache miss */ } } diff --git a/gitnexus/src/storage/parsedfile-store.ts b/gitnexus/src/storage/parsedfile-store.ts index 1f34ff083..51e2a9a54 100644 --- a/gitnexus/src/storage/parsedfile-store.ts +++ b/gitnexus/src/storage/parsedfile-store.ts @@ -24,11 +24,11 @@ * * ## Shape * - * `/parsedfile-store/.json` — one shard per parse chunk, - * a JSON array of `ParsedFile` serialized with the same `mapReplacer` the parse - * cache uses (Scope.bindings / Scope.typeBindings are `Map`s). The store is - * cleared at the start of each parse and after scope-resolution consumes it, so - * it never lingers and never goes stale across runs. + * `/parsedfile-store/.v8` — one shard per parse chunk, + * a V8 envelope of `ParsedFile[]` (Scope.bindings / Scope.typeBindings stay + * `Map`s). The store is cleared at the start of each parse and after + * scope-resolution consumes it, so it never lingers and never goes stale + * across runs. * * ## Durable sibling store (`parsedfile-cache/`, warm-cache coverage) * @@ -41,14 +41,14 @@ * that gap we ALSO write the worker's ParsedFiles to a second, CONTENT-ADDRESSED * store keyed by the parse chunk hash (`getDurableParsedFileDir`), which mirrors * the parse cache's lifecycle (persists across runs, pruned by `usedKeys`, - * version-tied via `PARSE_CACHE_VERSION`). On a warm hit the chunk's durable - * shards are byte-COPIED into the run-scoped store (no re-parse, no - * re-serialize → byte-identical), so scope-resolution streams them exactly as - * on a cold run. Content-addressing makes stale reuse impossible: a changed - * file changes its chunk hash, which misses BOTH stores and re-dispatches. + * version-tied via `PARSE_CACHE_VERSION`). On a warm hit the chunk's immutable + * durable shards are hardlinked (or atomically copied) into the run store after + * their envelope metadata proves complete coverage. That pins a stable snapshot + * before workers are skipped, even when another branch refreshes the shared + * durable directory concurrently. */ -import { promises as fs, mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; +import { promises as fs, mkdirSync } from 'node:fs'; import path from 'node:path'; import v8 from 'node:v8'; import vm from 'node:vm'; @@ -60,7 +60,14 @@ import type { } from 'gitnexus-shared'; import { isValidReceiverChain } from '../core/ingestion/utils/receiver-chain-codec.js'; import { logger } from '../core/logger.js'; -import { mapReplacer, mapReviver } from './parse-cache.js'; +import { mapReviver } from './parse-cache.js'; +import { linkOrCopyFile } from './fs-atomic.js'; +import { + inspectV8Cache, + tryLoadV8Cache, + writeV8CacheFile, + writeV8CacheFileSync, +} from './v8-sidecar.js'; const STORE_DIRNAME = 'parsedfile-store'; const DURABLE_DIRNAME = 'parsedfile-cache'; @@ -164,24 +171,13 @@ export const clearParsedFileStore = async (storagePath: string): Promise = await fs.rm(getParsedFileStoreDir(storagePath), { recursive: true, force: true }); }; -/** - * Single source of truth for a shard's bytes. Returns `null` for an empty - * chunk (caller writes nothing). Both the async (`persistParsedFileChunk`) and - * sync (`persistParsedFileShardSync`) writers go through this so the two paths - * are guaranteed byte-identical — the shards must round-trip through the same - * `mapReviver`, and matching bytes by having both authors type the same - * `mapReplacer` call would be a coincidence, not a guarantee. - */ -const serializeParsedFileShard = (parsedFiles: readonly ParsedFile[]): string | null => { - if (parsedFiles.length === 0) return null; - return JSON.stringify(parsedFiles, mapReplacer); -}; +const isV8ShardName = (name: string): boolean => name.endsWith('.v8') && !name.includes('.v8.'); const shardPath = (storagePath: string, shardId: string): string => - path.join(getParsedFileStoreDir(storagePath), `${shardId}.json`); + path.join(getParsedFileStoreDir(storagePath), `${shardId}.v8`); -/** Sidecar listing `filePath`s in a shard; not matched by `endsWith('.json')`. */ -const shardPathsSidecarPath = (jsonPath: string): string => `${jsonPath}.paths`; +const shardFilePaths = (parsedFiles: readonly ParsedFile[]): string[] => + parsedFiles.map((pf) => pf.filePath); const LOAD_YIELD_EVERY_SHARDS = 128; @@ -191,147 +187,26 @@ const LOAD_YIELD_EVERY_SHARDS = 128; */ export const parsedFileLoadGc = { run: forceGc, - /** Raw UTF-8 JSON shard bytes between GCs (#3086). Tests may lower this. */ + /** V8 envelope bytes visited between GCs (#3086). Tests may lower this. */ byteBudget: 128 * 1024 * 1024, }; -const encodeShardPathsSidecar = (parsedFiles: readonly ParsedFile[]): string => { - const paths = parsedFiles.map((pf) => pf.filePath); - return `${paths.length}\n${paths.length === 0 ? '' : `${paths.join('\n')}\n`}`; -}; - /** - * Parse a counted NDJSON path listing. Returns `null` when the sidecar must - * not be trusted to skip the JSON shard: missing trailing newline, CR/NUL, - * a truncated listing that still ends on a complete line, or a count that - * does not match the remaining lines. - */ -const parseShardPathsSidecar = (sidecarRaw: string): string[] | null => { - if (sidecarRaw.includes('\0') || sidecarRaw.includes('\r') || !sidecarRaw.endsWith('\n')) { - return null; - } - const nl = sidecarRaw.indexOf('\n'); - if (nl < 0) return null; - const countToken = sidecarRaw.slice(0, nl); - if (!/^[0-9]+$/.test(countToken)) return null; - const count = Number(countToken); - const body = sidecarRaw.slice(nl + 1); - const listed = body === '' ? [] : body.slice(0, -1).split('\n'); - if (listed.length !== count) return null; - return listed; -}; - -/** NDJSON sidecars cannot encode paths that themselves contain CR/LF/NUL. */ -const shardPathsSidecarSafe = (parsedFiles: readonly ParsedFile[]): boolean => - parsedFiles.every((pf) => !/[\r\n\0]/.test(pf.filePath)); - -const isEnoent = (err: unknown): boolean => (err as NodeJS.ErrnoException).code === 'ENOENT'; - -const warnSidecarIo = (err: unknown, jsonPath: string, msg: string): void => { - logger.warn({ err, jsonPath }, msg); -}; - -const ignoreMissingSidecarUnlink = (err: unknown, jsonPath: string): void => { - if (isEnoent(err)) return; - warnSidecarIo( - err, - jsonPath, - 'parsedfile-store: failed to drop path sidecar; JSON remains authoritative', - ); -}; - -/** Drop a leftover listing before publishing JSON so load cannot skip new paths. */ -const dropPathSidecar = async (jsonPath: string): Promise => { - try { - await fs.unlink(shardPathsSidecarPath(jsonPath)); - } catch (err) { - ignoreMissingSidecarUnlink(err, jsonPath); - } -}; - -const dropPathSidecarSync = (jsonPath: string): void => { - try { - unlinkSync(shardPathsSidecarPath(jsonPath)); - } catch (err) { - ignoreMissingSidecarUnlink(err, jsonPath); - } -}; - -const writeShardPathsSidecar = async ( - jsonPath: string, - parsedFiles: readonly ParsedFile[], -): Promise => { - if (!shardPathsSidecarSafe(parsedFiles)) { - try { - await fs.unlink(shardPathsSidecarPath(jsonPath)); - } catch (err) { - ignoreMissingSidecarUnlink(err, jsonPath); - } - return; - } - try { - await fs.writeFile( - shardPathsSidecarPath(jsonPath), - encodeShardPathsSidecar(parsedFiles), - 'utf-8', - ); - } catch (err) { - warnSidecarIo( - err, - jsonPath, - 'parsedfile-store: path sidecar write failed; JSON shard remains authoritative', - ); - try { - await fs.unlink(shardPathsSidecarPath(jsonPath)); - } catch (unlinkErr) { - ignoreMissingSidecarUnlink(unlinkErr, jsonPath); - } - } -}; - -const writeShardPathsSidecarSync = (jsonPath: string, parsedFiles: readonly ParsedFile[]): void => { - if (!shardPathsSidecarSafe(parsedFiles)) { - try { - unlinkSync(shardPathsSidecarPath(jsonPath)); - } catch (err) { - ignoreMissingSidecarUnlink(err, jsonPath); - } - return; - } - try { - writeFileSync(shardPathsSidecarPath(jsonPath), encodeShardPathsSidecar(parsedFiles), 'utf-8'); - } catch (err) { - warnSidecarIo( - err, - jsonPath, - 'parsedfile-store: path sidecar write failed; JSON shard remains authoritative', - ); - try { - unlinkSync(shardPathsSidecarPath(jsonPath)); - } catch (unlinkErr) { - ignoreMissingSidecarUnlink(unlinkErr, jsonPath); - } - } -}; - -/** - * Write one parse chunk's `ParsedFile[]` to the store as a single shard (async). - * No-op for an empty chunk. `shardId` must be unique within a run. Used by the - * main-thread no-store-disabled fallback and any non-worker writer; the worker - * store path uses {@link persistParsedFileShardSync}. + * Write one parse chunk's `ParsedFile[]` to the store as a single `.v8` shard. + * No-op for an empty chunk. `shardId` must be unique within a run. */ export const persistParsedFileChunk = async ( storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[], -): Promise => { - const payload = serializeParsedFileShard(parsedFiles); - if (payload === null) return; +): Promise => { + if (parsedFiles.length === 0) return true; await fs.mkdir(getParsedFileStoreDir(storagePath), { recursive: true }); - const dest = shardPath(storagePath, shardId); - await dropPathSidecar(dest); - await fs.writeFile(dest, payload, 'utf-8'); - await writeShardPathsSidecar(dest, parsedFiles); + return writeV8CacheFile( + shardPath(storagePath, shardId), + parsedFiles, + shardFilePaths(parsedFiles), + ); }; // Per-process set of store dirs we've already `mkdir`ed, so the sync worker @@ -341,55 +216,42 @@ const createdStoreDirs = new Set(); /** * Synchronous shard writer for use INSIDE a parse worker (#1983 parallel - * serialization). The worker is a dedicated thread, so a blocking write there - * protects the main thread, and a sync write avoids threading `async`/`await` - * through the synchronous per-file extract loop. Produces byte-identical shards - * to {@link persistParsedFileChunk} via the shared {@link serializeParsedFileShard}. - * No-op for an empty chunk. `shardId` must be globally unique for the run (the - * worker uses `w-`); a duplicate would silently overwrite. + * serialization). Returns false on write failure so the worker can keep + * ParsedFiles in the result instead of dropping them. */ export const persistParsedFileShardSync = ( storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[], -): void => { - const payload = serializeParsedFileShard(parsedFiles); - if (payload === null) return; +): boolean => { + if (parsedFiles.length === 0) return true; const dir = getParsedFileStoreDir(storagePath); if (!createdStoreDirs.has(dir)) { mkdirSync(dir, { recursive: true }); createdStoreDirs.add(dir); } - const dest = shardPath(storagePath, shardId); - dropPathSidecarSync(dest); - writeFileSync(dest, payload, 'utf-8'); - writeShardPathsSidecarSync(dest, parsedFiles); + return writeV8CacheFileSync( + shardPath(storagePath, shardId), + parsedFiles, + shardFilePaths(parsedFiles), + ); +}; + +const listV8Shards = async (dir: string): Promise => { + try { + return (await fs.readdir(dir)).filter(isV8ShardName).map((name) => path.join(dir, name)); + } catch { + return []; + } }; -/** - * Stream the store and return the `ParsedFile`s whose `filePath` is in - * `wantPaths`, keyed by path. Loads one shard at a time and retains only the - * matching entries, so peak heap is bounded by (matched set) + (one shard) - * rather than the whole store. Returns an empty map when the store is absent - * (e.g. tests, or a run with no worker pool) — callers fall back to a fresh - * extract for the missing files. - */ export const loadParsedFilesForPaths = async ( storagePath: string, wantPaths: ReadonlySet, ): Promise> => { const out = new Map(); if (wantPaths.size === 0) return out; - const dir = getParsedFileStoreDir(storagePath); - let shards: string[]; - try { - shards = (await fs.readdir(dir)).filter((f) => f.endsWith('.json')); - } catch { - return out; // store absent - } - // Shared interning pool for this load — deduplicates strings ACROSS shards - // (one `int` / one repeated filePath for the whole language), which is where - // most of the saving comes from. Dropped when this function returns. + const shardPaths = await listV8Shards(getParsedFileStoreDir(storagePath)); const pool = new Map(); let droppedSites = 0; let filesWithDroppedSites = 0; @@ -411,51 +273,25 @@ export const loadParsedFilesForPaths = async ( await new Promise((resolve) => setImmediate(resolve)); } }; - for (let i = 0; i < shards.length; i++) { - const jsonName = shards[i]; - const jsonFull = path.join(dir, jsonName); - try { - const sidecarRaw = await fs.readFile(shardPathsSidecarPath(jsonFull), 'utf-8'); - // Fail closed: complete writers emit `\n` plus one path per line - // and a trailing newline, never CR. Stripping CR (or accepting a - // newline-terminated prefix) would let a truncated listing skip JSON. - const listed = parseShardPathsSidecar(sidecarRaw); - if (listed === null) { - throw new Error('corrupt sidecar'); - } - if (listed.length > 0 && !listed.some((p) => wantPaths.has(p))) { - await maybeYieldAndGc(false); - continue; - } - } catch { - // Missing or unreadable sidecar → read the shard (pre-sidecar stores). + for (const shardFull of shardPaths) { + const loaded = await tryLoadV8Cache(shardFull, pool, wantPaths); + if (loaded === undefined) { + await maybeYieldAndGc(false); + continue; } - // Per-shard def pool: a SymbolDefinition's three serialized copies live within - // a single shard (one ParsedFile), so the dedup is shard-local. A cross-shard - // pool would retain defs of files NOT in `wantPaths` (loaded-but-discarded - // shards), reintroducing the leak; per-shard drops them with the shard. - const defPool = new Map(); - const reviver = makeInterningReviver(pool, defPool); - let raw: string; - try { - raw = await fs.readFile(jsonFull, 'utf-8'); - } catch { - continue; // skip a missing shard; missing files fall back to fresh extract + if (loaded.kind === 'skip') { + bytesSinceGc += loaded.bytes; + await maybeYieldAndGc(bytesSinceGc >= parsedFileLoadGc.byteBudget); + continue; } - bytesSinceGc += Buffer.byteLength(raw, 'utf8'); + bytesSinceGc += loaded.bytes; + const parsed = Array.isArray(loaded.value) ? (loaded.value as ParsedFile[]) : undefined; const crossedBudget = bytesSinceGc >= parsedFileLoadGc.byteBudget; - let parsed: ParsedFile[] | undefined; - try { - parsed = JSON.parse(raw, reviver) as ParsedFile[]; - } catch { - parsed = undefined; - } if (Array.isArray(parsed)) { for (const pf of parsed) { if (!pf || typeof pf.filePath !== 'string' || !wantPaths.has(pf.filePath)) continue; const flow = sanitizeCallableFlowSites(pf.callableFlowSites); if (flow === undefined) { - // non-array garbage → distrust the file, re-extract rejectedFiles++; continue; } @@ -481,19 +317,12 @@ export const loadParsedFilesForPaths = async ( await maybeYieldAndGc(crossedBudget); } if (droppedSites > 0 || droppedChains > 0) { - // Facts for the dropped sites are omitted this run (the file itself is - // retained, so no re-extract happens) — surface it so a recurring drop - // on every warm load is observable rather than silent (#2522 review). logger.warn( { droppedSites, droppedChains, files: filesWithDroppedSites }, 'parsedfile-store: dropped malformed/over-bound sites at load; files retained without those facts', ); } if (rejectedFiles > 0) { - // The other half of the same defect. A rejected file silently falls back to - // a fresh extract EVERY load, so a writer that keeps minting what this - // reader keeps refusing is a permanent warm-cache miss that costs real time - // and says nothing about why. logger.warn( { rejectedFiles }, 'parsedfile-store: rejected shard entries at load (untrusted shape); those files re-extract every run', @@ -502,16 +331,6 @@ export const loadParsedFilesForPaths = async ( return out; }; -/** - * Treat the durable ParsedFile store as an untrusted serialization boundary. - * Sanitation is per-SITE, not per-file: one malformed or over-bound fact drops - * only itself (counted, logged by the caller), so a legitimately pathological - * source file cannot push its whole ParsedFile into a permanent, silent - * warm-cache-miss reparse loop (#2522 review). Only a non-array field — - * i.e. garbage that says the serialization itself is untrustworthy — rejects - * the file, and `undefined` (never emitted / no facts) passes through. - * Returns `undefined` for the reject-file case. - */ function sanitizeCallableFlowSites( value: unknown, ): { sites: readonly CallableFlowSite[] | undefined; dropped: number } | undefined { @@ -709,16 +528,15 @@ function isSafeIndex(value: unknown): boolean { // ─── Durable, content-addressed sibling store (warm-cache coverage) ────────── // -// Layout: `//-w-.json` plus a -// top-level `/index.json` = `{version, keys:[chunkHash…]}`. One -// subdir per chunk hash so a chunk's (possibly several) shards collect and -// prune as a unit, and so `readdir(/)` is O(shards-of-this-chunk), -// not O(all-history). Shards are byte-identical to run-scoped shards (same -// `serializeParsedFileShard`); restore is a verbatim copy, never a re-serialize. +// Layout: `//-w-.v8` plus a +// top-level `/index.json` that records each chunk hash's actual +// persisted file-path coverage. One subdir per chunk hash so a chunk's +// (possibly several) shards collect and prune as a unit. Warm hits snapshot +// these files into the run store before worker dispatch is skipped. interface DurableParsedFileIndex { version: string; - keys: string[]; + entries: Record; } /** Durable store dir — a sibling of `parsedfile-store/`, NEVER cleared per run. */ @@ -744,10 +562,6 @@ export const prepareDurableParsedFileChunk = async ( await fs.mkdir(dir, { recursive: true }); }; -// Per-process set of durable chunk subdirs already `mkdir`ed (mirrors -// `createdStoreDirs`) so the worker doesn't `mkdirSync` on every shard. -const createdDurableDirs = new Set(); - /** * Synchronous durable-shard writer for use INSIDE a parse worker, alongside * {@link persistParsedFileShardSync}. Writes the SAME bytes to a content-addressed @@ -757,96 +571,99 @@ const createdDurableDirs = new Set(); * uniqueness that makes the run-scoped `w-` name safe, prefixed by * content. No-op for an empty chunk. */ + +const createdDurableDirs = new Set(); + export const persistDurableParsedFileShardSync = ( durableDir: string, chunkHash: string, threadId: number, shardSeq: number, parsedFiles: readonly ParsedFile[], -): void => { - const payload = serializeParsedFileShard(parsedFiles); - if (payload === null) return; +): boolean => { + if (parsedFiles.length === 0) return true; const dir = durableChunkDir(durableDir, chunkHash); if (!createdDurableDirs.has(dir)) { mkdirSync(dir, { recursive: true }); createdDurableDirs.add(dir); } - const dest = path.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.json`); - dropPathSidecarSync(dest); - writeFileSync(dest, payload, 'utf-8'); - writeShardPathsSidecarSync(dest, parsedFiles); + const dest = path.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.v8`); + return writeV8CacheFileSync(dest, parsedFiles, shardFilePaths(parsedFiles)); }; /** - * Restore a cached chunk's durable shards into the run-scoped store on a warm - * hit. A verbatim byte copy (no parse, no re-serialize), so the restored - * ParsedFiles are byte-identical to a cold run and `loadParsedFilesForPaths` - * (which keys on `filePath`, not shard name) gives scope-resolution full - * coverage. The durable shard names already carry the chunk hash, so they never - * collide with the worker's run-scoped `w-` shards. Returns the number - * of shards restored (0 ⇒ no durable coverage for this chunk; caller treats it - * as a miss). + * Validate and snapshot one durable chunk into the run store. Every envelope + * must be runtime-compatible and integrity-valid, and together they must match + * the path coverage recorded when the durable index was published. Linking + * before returning pins the inodes against concurrent branch-cache rotation. */ -export const restoreDurableParsedFileShard = async ( - durableDir: string, +export const durableChunkHasShards = async ( runStoragePath: string, chunkHash: string, -): Promise => { - const src = durableChunkDir(durableDir, chunkHash); - let shards: string[]; + expectedPaths: ReadonlySet, +): Promise => { + const sourceDir = durableChunkDir(getDurableParsedFileDir(runStoragePath), chunkHash); + const shards = await listV8Shards(sourceDir); + if (shards.length === 0 || expectedPaths.size === 0) return false; + + const runDir = getParsedFileStoreDir(runStoragePath); try { - shards = (await fs.readdir(src)).filter((f) => f.endsWith('.json')); + await fs.mkdir(runDir, { recursive: true }); } catch { - return 0; // no durable shards for this chunk + return false; } - if (shards.length === 0) return 0; - const dst = getParsedFileStoreDir(runStoragePath); - await fs.mkdir(dst, { recursive: true }); - for (const name of shards) { - const srcJson = path.join(src, name); - const dstJson = path.join(dst, name); - await dropPathSidecar(dstJson); - await fs.copyFile(srcJson, dstJson); + const restored: string[] = []; + const covered = new Set(); + const rollback = async (): Promise => { + await Promise.all(restored.map((filePath) => fs.rm(filePath, { force: true }).catch(() => {}))); + return false; + }; + + for (const sourcePath of shards) { + const name = path.basename(sourcePath); + const destinationPath = path.join(runDir, name); try { - await fs.copyFile(shardPathsSidecarPath(srcJson), shardPathsSidecarPath(dstJson)); - } catch (copyErr) { - if (!isEnoent(copyErr)) { - warnSidecarIo( - copyErr, - srcJson, - 'parsedfile-store: durable path sidecar copy failed; JSON remains authoritative', - ); - continue; - } - try { - await fs.unlink(shardPathsSidecarPath(dstJson)); - } catch (err) { - ignoreMissingSidecarUnlink(err, dstJson); - } + await linkOrCopyFile(sourcePath, destinationPath); + restored.push(destinationPath); + } catch { + return rollback(); + } + const inspected = await inspectV8Cache(destinationPath); + if (!inspected) return rollback(); + for (const filePath of inspected.paths) { + if (!expectedPaths.has(filePath)) return rollback(); + covered.add(filePath); } } - return shards.length; + + if (covered.size !== expectedPaths.size) return rollback(); + return true; }; -/** - * Read the durable index and return the set of chunk hashes it vouches for, - * gated on `expectedVersion` (`PARSE_CACHE_VERSION`). A version mismatch or a - * missing/corrupt index returns the empty set — the caller then treats every - * chunk as a durable miss and re-dispatches workers (NEVER the main-thread - * `extractParsedFile` fallback), which rewrites the durable store under the new - * version. Mirrors `loadParseCache`'s version-invalidation contract. - */ export const loadDurableParsedFileIndex = async ( durableDir: string, expectedVersion: string, -): Promise> => { +): Promise>> => { try { const raw = await fs.readFile(path.join(durableDir, DURABLE_INDEX_FILENAME), 'utf-8'); - const idx = JSON.parse(raw) as DurableParsedFileIndex; - if (idx?.version !== expectedVersion || !Array.isArray(idx.keys)) return new Set(); - return new Set(idx.keys); + const idx: unknown = JSON.parse(raw); + if (!isRecord(idx) || idx.version !== expectedVersion || !isRecord(idx.entries)) { + return new Map(); + } + const entries = new Map>(); + for (const [key, paths] of Object.entries(idx.entries)) { + if ( + !Array.isArray(paths) || + paths.length === 0 || + paths.some((filePath) => typeof filePath !== 'string') + ) { + return new Map(); + } + entries.set(key, new Set(paths)); + } + return entries; } catch { - return new Set(); + return new Map(); } }; @@ -855,9 +672,9 @@ export const loadDurableParsedFileIndex = async ( * be the parse cache's surviving on-disk keys (so the two stores stay coherent: * a chunk is "cached" iff BOTH its parse-cache shard and its durable shards * exist; a quarantined chunk — no parse-cache shard — drops its durable subdir - * here and re-dispatches next run). Only subdirs with ≥1 shard are indexed - * (mirrors `saveParseCache`'s written-keys discipline — never vouch for a chunk - * hash with no backing shard). The index write is tmp+rename atomic. + * here and re-dispatches next run). Only chunks whose envelopes all validate + * are indexed, together with their exact persisted path coverage (never vouch + * for a missing/corrupt shard). The index write is tmp+rename atomic. */ export const pruneAndSaveDurableParsedFileStore = async ( durableDir: string, @@ -870,16 +687,28 @@ export const pruneAndSaveDurableParsedFileStore = async ( } catch { return; // nothing written this run } - const survivors: string[] = []; + const survivors: Record = {}; for (const name of entries) { if (name === DURABLE_INDEX_FILENAME) continue; const full = path.join(durableDir, name); if (keepKeys.has(name)) { try { - const shards = (await fs.readdir(full)).filter((f) => f.endsWith('.json')); + const shards = await listV8Shards(full); if (shards.length > 0) { - survivors.push(name); - continue; + const covered = new Set(); + let valid = true; + for (const shard of shards) { + const inspected = await inspectV8Cache(shard); + if (!inspected) { + valid = false; + break; + } + for (const filePath of inspected.paths) covered.add(filePath); + } + if (valid && covered.size > 0) { + survivors[name] = [...covered].sort(); + continue; + } } } catch { /* not a readable dir → drop below */ @@ -887,7 +716,7 @@ export const pruneAndSaveDurableParsedFileStore = async ( } await fs.rm(full, { recursive: true, force: true }); } - const idx: DurableParsedFileIndex = { version, keys: survivors }; + const idx: DurableParsedFileIndex = { version, entries: survivors }; const tmp = path.join(durableDir, `${DURABLE_INDEX_FILENAME}.tmp`); await fs.mkdir(durableDir, { recursive: true }); await fs.writeFile(tmp, JSON.stringify(idx), 'utf-8'); diff --git a/gitnexus/src/storage/v8-sidecar.ts b/gitnexus/src/storage/v8-sidecar.ts new file mode 100644 index 000000000..e3b2f92c9 --- /dev/null +++ b/gitnexus/src/storage/v8-sidecar.ts @@ -0,0 +1,363 @@ +/** + * One-file V8 cache envelope for parse-cache and ParsedFile shards. + * + * Each object-graph shard is a single immutable `.v8` file published by + * tmp+rename. JSON is not a fallback: a missing, corrupt, runtime-incompatible, + * or deserialize-failed envelope is a cache miss (re-extract / re-dispatch). + * Tiny JSON manifests (`index.json`) stay outside this module. + * + * Envelope: magic, format, Node major, V8 version, optional path listing, + * `v8.serialize` of the live graph, then SHA-256 of listing||payload. Path + * bytes live before the payload so a ParsedFile loader can skip a shard whose + * authenticated listing misses `wantPaths` without deserializing. An + * unreadable/invalid listing is fail-closed: deserialize, never skip. + */ +import { promises as fs } from 'node:fs'; +import { createHash } from 'node:crypto'; +import v8 from 'node:v8'; +import { logger } from '../core/logger.js'; +import { linkOrCopyFile, writeFileAtomicBytes, writeFileAtomicBytesSync } from './fs-atomic.js'; + +const MAGIC = Buffer.from('GNXV8CF1', 'ascii'); +/** Envelope version — independent of PARSE_CACHE_VERSION / SCHEMA_BUMP. */ +export const V8_CACHE_FORMAT = 5; +const U32 = 4; +const U16 = 2; +const PAYLOAD_HASH_LEN = 32; +const MAGIC_LEN = 8; +const FIXED_PREFIX = MAGIC_LEN + U32 + U16 + U16; // magic + format + nodeMajor + v8len + +const internString = (value: string, pool: Map): string => { + const hit = pool.get(value); + if (hit !== undefined) return hit; + pool.set(value, value); + return value; +}; + +/** + * Collapse duplicate strings in a live deserialized graph into `pool`, mutating + * in place so object identity (shared `SymbolDefinition`s, Maps) is preserved. + * Required after `v8.deserialize` of ParsedFile shards: V8 does not recreate + * the JSON reviver's cross-shard string intern, and skipping it regresses + * retained heap (~+59% measured vs interned JSON). + */ +export const internGraphStrings = (root: unknown, pool: Map): unknown => { + const seen = new WeakSet(); + const walk = (value: unknown): unknown => { + if (typeof value === 'string') return internString(value, pool); + if (value === null || typeof value !== 'object') return value; + if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) return value; + if (seen.has(value)) return value; + seen.add(value); + if (value instanceof Map) { + const entries = [...value]; + value.clear(); + for (const [k, v] of entries) value.set(walk(k), walk(v)); + return value; + } + if (value instanceof Set) { + const entries = [...value]; + value.clear(); + for (const v of entries) value.add(walk(v)); + return value; + } + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) value[i] = walk(value[i]); + return value; + } + const rec = value as Record; + for (const key of Object.keys(rec)) { + rec[key] = walk(rec[key]); + } + return value; + }; + return walk(root); +}; + +const nodeMajor = (): number => Number.parseInt(process.versions.node.split('.')[0] ?? '0', 10); + +const isEnoent = (err: unknown): boolean => (err as NodeJS.ErrnoException).code === 'ENOENT'; + +const warnCache = (err: unknown, filePath: string, msg: string): void => { + logger.warn({ err, filePath }, msg); +}; + +/** NDJSON listing cannot encode paths that themselves contain CR/LF/NUL. */ +export const encodeCachePathListing = (paths: readonly string[]): Buffer => { + if (paths.length === 0) return Buffer.alloc(0); + if (paths.some((p) => /[\r\n\0]/.test(p))) return Buffer.alloc(0); + return Buffer.from(`${paths.length}\n${paths.join('\n')}\n`, 'utf8'); +}; + +/** + * Parse a counted NDJSON path listing. Returns `null` when it must not be + * trusted to skip the payload: missing trailing newline, CR/NUL, or a count + * that does not match the remaining lines. + */ +export const parseCachePathListing = (raw: Buffer): string[] | null => { + if (raw.byteLength === 0) return []; + let sidecarRaw: string; + try { + sidecarRaw = new TextDecoder('utf-8', { fatal: true }).decode(raw); + } catch { + return null; + } + if (sidecarRaw.includes('\0') || sidecarRaw.includes('\r') || !sidecarRaw.endsWith('\n')) { + return null; + } + const nl = sidecarRaw.indexOf('\n'); + if (nl < 0) return null; + const countToken = sidecarRaw.slice(0, nl); + if (!/^[0-9]+$/.test(countToken)) return null; + const count = Number(countToken); + const body = sidecarRaw.slice(nl + 1); + const listed = body === '' ? [] : body.slice(0, -1).split('\n'); + if (listed.length !== count) return null; + return listed; +}; + +const encodeEnvelope = (graph: unknown, paths: readonly string[]): Buffer | undefined => { + let payload: Buffer; + try { + payload = v8.serialize(graph); + } catch (err) { + warnCache(err, '', 'v8 cache: serialize failed; treating as miss on next load'); + return undefined; + } + const pathListing = encodeCachePathListing(paths); + const listed = parseCachePathListing(pathListing); + const pathCount = listed === null ? 0 : listed.length; + if (payload.byteLength > 0xffff_ffff || pathListing.byteLength > 0xffff_ffff) return undefined; + const v8ver = Buffer.from(process.versions.v8, 'utf8'); + if (v8ver.byteLength > 0xffff) return undefined; + const header = Buffer.allocUnsafe(FIXED_PREFIX + v8ver.length + U32 * 3); + MAGIC.copy(header, 0); + header.writeUInt32LE(V8_CACHE_FORMAT, MAGIC_LEN); + header.writeUInt16LE(nodeMajor(), MAGIC_LEN + U32); + header.writeUInt16LE(v8ver.length, MAGIC_LEN + U32 + U16); + v8ver.copy(header, FIXED_PREFIX); + let off = FIXED_PREFIX + v8ver.length; + header.writeUInt32LE(pathCount, off); + off += U32; + header.writeUInt32LE(pathListing.byteLength, off); + off += U32; + header.writeUInt32LE(payload.byteLength, off); + const payloadHash = createHash('sha256').update(pathListing).update(payload).digest(); + return Buffer.concat([header, pathListing, payload, payloadHash]); +}; + +type EnvelopeMeta = { + recordedNodeMajor: number; + recordedV8: string; + pathCount: number; + pathBytes: number; + payloadLen: number; + pathsOff: number; + payloadOff: number; +}; + +const decodePrefix = (buf: Buffer): EnvelopeMeta | undefined => { + if (buf.byteLength < FIXED_PREFIX) return undefined; + if (!buf.subarray(0, MAGIC_LEN).equals(MAGIC)) return undefined; + if (buf.readUInt32LE(MAGIC_LEN) !== V8_CACHE_FORMAT) return undefined; + const recordedNodeMajor = buf.readUInt16LE(MAGIC_LEN + U32); + const v8len = buf.readUInt16LE(MAGIC_LEN + U32 + U16); + const v8off = FIXED_PREFIX; + const countsOff = v8off + v8len; + if (buf.byteLength < countsOff + U32 * 3) return undefined; + const recordedV8 = buf.subarray(v8off, v8off + v8len).toString('utf8'); + const pathCount = buf.readUInt32LE(countsOff); + const pathBytes = buf.readUInt32LE(countsOff + U32); + const payloadLen = buf.readUInt32LE(countsOff + U32 * 2); + const pathsOff = countsOff + U32 * 3; + const payloadOff = pathsOff + pathBytes; + return { + recordedNodeMajor, + recordedV8, + pathCount, + pathBytes, + payloadLen, + pathsOff, + payloadOff, + }; +}; + +const runtimeCompatible = (meta: EnvelopeMeta): boolean => + meta.recordedNodeMajor === nodeMajor() && meta.recordedV8 === process.versions.v8; + +export type V8CacheHit = { kind: 'hit'; value: unknown; bytes: number }; +export type V8CacheSkip = { kind: 'skip'; bytes: number }; +export type V8CacheLoad = V8CacheHit | V8CacheSkip; +export type V8CacheInspection = { paths: readonly string[] }; + +const readExact = async ( + fh: Awaited>, + offset: number, + length: number, +): Promise => { + const buf = Buffer.allocUnsafe(length); + let got = 0; + while (got < length) { + const { bytesRead } = await fh.read(buf, got, length - got, offset + got); + if (bytesRead === 0) return undefined; + got += bytesRead; + } + return buf; +}; + +const readVerifiedPayload = async ( + fh: Awaited>, + meta: EnvelopeMeta, + pathRaw: Buffer, +): Promise => { + const payloadAndHash = await readExact(fh, meta.payloadOff, meta.payloadLen + PAYLOAD_HASH_LEN); + if (!payloadAndHash) return undefined; + const payload = payloadAndHash.subarray(0, meta.payloadLen); + const expected = payloadAndHash.subarray(meta.payloadLen); + const digest = createHash('sha256').update(pathRaw).update(payload).digest(); + return digest.equals(expected) ? payload : undefined; +}; + +/** + * Validate the immutable envelope metadata needed by the durable ParsedFile + * warm-hit gate without deserializing its payload. Atomic publication means a + * runtime-compatible envelope whose exact file length and counted path listing + * validate is a stable snapshot candidate; malformed/truncated envelopes miss. + */ +export const inspectV8Cache = async (filePath: string): Promise => { + let fh: Awaited> | undefined; + try { + fh = await fs.open(filePath, 'r'); + const st = await fh.stat(); + const prefix = await readExact(fh, 0, Math.min(st.size, FIXED_PREFIX + 256 + U32 * 3)); + if (!prefix) return undefined; + const meta = decodePrefix(prefix); + if (!meta || !runtimeCompatible(meta)) return undefined; + if (meta.payloadOff + meta.payloadLen + PAYLOAD_HASH_LEN !== st.size || meta.pathBytes === 0) { + return undefined; + } + + const pathRaw = + prefix.byteLength >= meta.payloadOff + ? Buffer.from(prefix.subarray(meta.pathsOff, meta.payloadOff)) + : await readExact(fh, meta.pathsOff, meta.pathBytes); + if (!pathRaw) return undefined; + const listed = parseCachePathListing(pathRaw); + if (listed === null || listed.length !== meta.pathCount || listed.length === 0) { + return undefined; + } + if (!(await readVerifiedPayload(fh, meta, pathRaw))) return undefined; + return { paths: listed }; + } catch (err) { + if (!isEnoent(err)) { + logger.debug({ err, filePath }, 'v8 cache: inspection failed; treating as miss'); + } + return undefined; + } finally { + await fh?.close().catch(() => {}); + } +}; + +/** + * Load a cache file. When `wantPaths` is set and a digest-verified non-empty + * path listing has no intersection, returns `{ kind: 'skip' }` without + * deserializing. An authentic listing that does not parse deserializes (fail + * closed). Envelope/runtime/digest failure returns undefined (miss). + */ +export const tryLoadV8Cache = async ( + filePath: string, + internPool?: Map, + wantPaths?: ReadonlySet, +): Promise => { + let fh: Awaited> | undefined; + try { + fh = await fs.open(filePath, 'r'); + const st = await fh.stat(); + const prefix = await readExact(fh, 0, Math.min(st.size, FIXED_PREFIX + 256 + U32 * 3)); + if (!prefix) return undefined; + const meta = decodePrefix(prefix); + if (!meta) return undefined; + if (!runtimeCompatible(meta)) return undefined; + if (meta.payloadOff + meta.payloadLen + PAYLOAD_HASH_LEN !== st.size) return undefined; + + let pathRaw = Buffer.alloc(0); + if (meta.pathBytes > 0) { + if (prefix.byteLength >= meta.payloadOff) { + pathRaw = Buffer.from(prefix.subarray(meta.pathsOff, meta.payloadOff)); + } else { + const raw = await readExact(fh, meta.pathsOff, meta.pathBytes); + if (!raw) return undefined; + pathRaw = Buffer.from(raw); + } + } + + const payload = await readVerifiedPayload(fh, meta, pathRaw); + if (!payload) return undefined; + + if (wantPaths && wantPaths.size > 0 && meta.pathBytes > 0) { + const listed = parseCachePathListing(pathRaw); + if ( + listed !== null && + listed.length === meta.pathCount && + listed.length > 0 && + !listed.some((p) => wantPaths.has(p)) + ) { + return { kind: 'skip', bytes: st.size }; + } + } + const value = v8.deserialize(payload); + if (internPool) internGraphStrings(value, internPool); + return { kind: 'hit', value, bytes: st.size }; + } catch (err) { + if (!isEnoent(err)) { + logger.debug({ err, filePath }, 'v8 cache: load failed; treating as miss'); + } + return undefined; + } finally { + await fh?.close().catch(() => {}); + } +}; + +export const writeV8CacheFile = async ( + filePath: string, + graph: unknown, + paths?: readonly string[], +): Promise => { + const blob = encodeEnvelope(graph, paths ?? []); + if (!blob) return false; + try { + await writeFileAtomicBytes(filePath, blob, 1); + return true; + } catch (err) { + warnCache(err, filePath, 'v8 cache: write failed; treating as miss'); + return false; + } +}; + +export const writeV8CacheFileSync = ( + filePath: string, + graph: unknown, + paths?: readonly string[], +): boolean => { + const blob = encodeEnvelope(graph, paths ?? []); + if (!blob) return false; + try { + writeFileAtomicBytesSync(filePath, blob); + return true; + } catch (err) { + warnCache(err, filePath, 'v8 cache: write failed; treating as miss'); + return false; + } +}; + +export const copyV8CacheIfPresent = async (srcPath: string, dstPath: string): Promise => { + try { + await linkOrCopyFile(srcPath, dstPath); + return true; + } catch (copyErr) { + if (!isEnoent(copyErr)) { + warnCache(copyErr, srcPath, 'v8 cache: copy failed; treating as miss'); + } + return false; + } +}; diff --git a/gitnexus/test/integration/cfg/parse-cache-mixed.test.ts b/gitnexus/test/integration/cfg/parse-cache-mixed.test.ts index 58f183782..b046f422a 100644 --- a/gitnexus/test/integration/cfg/parse-cache-mixed.test.ts +++ b/gitnexus/test/integration/cfg/parse-cache-mixed.test.ts @@ -5,19 +5,19 @@ import path from 'node:path'; import { getDurableParsedFileDir, persistDurableParsedFileShardSync, - restoreDurableParsedFileShard, + durableChunkHasShards, loadParsedFilesForPaths, } from '../../../src/storage/parsedfile-store.js'; import type { ParsedFile } from 'gitnexus-shared'; import type { FunctionCfg } from '../../../src/core/ingestion/cfg/types.js'; // #2082 M2 U5 — the warm/mixed cache seam for statement facts. On a warm (or -// mixed) run the unchanged chunk's ParsedFiles are BYTE-COPIED from the -// durable store instead of re-parsed (#2038); if that copy (or the store's -// interning reviver) dropped or aliased the new `bindings`/`statements` +// mixed) run the unchanged chunk's ParsedFiles are loaded from the +// durable store instead of re-parsed (#2038); if that load (or intern) +// dropped or aliased the new `bindings`/`statements` // fields, reaching-defs would silently degrade to `no-facts` for every cached // file — exactly the field-loss class the #2038 mergeChunkResults lesson -// warns about. This pins the persist → restore → load round-trip at the exact +// warns about. This pins the persist → load round-trip at the exact // seam scope-resolution consumes. const factCfg: FunctionCfg = { @@ -68,7 +68,7 @@ describe('durable ParsedFile store carries M2 statement facts (#2082 U5)', () => if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); }); - it('persist → restore → loadParsedFilesForPaths preserves bindings + statements deep-equal', async () => { + it('persist → loadParsedFilesForPaths preserves bindings + statements deep-equal', async () => { const durableDir = getDurableParsedFileDir(tempDir); const chunkHash = 'c'.repeat(64); const files = ['src/a.ts', 'src/b.ts']; @@ -76,10 +76,9 @@ describe('durable ParsedFile store carries M2 statement facts (#2082 U5)', () => // What a worker writes at flush on a cache MISS (the cold half of a // mixed-mode run)… persistDurableParsedFileShardSync(durableDir, chunkHash, 7, 0, files.map(mkParsedFile)); - // …and what a warm HIT byte-copies into the run-scoped store. - await restoreDurableParsedFileShard(durableDir, tempDir, chunkHash); - - const loaded = await loadParsedFilesForPaths(tempDir, new Set(files)); + const wanted = new Set(files); + expect(await durableChunkHasShards(tempDir, chunkHash, wanted)).toBe(true); + const loaded = await loadParsedFilesForPaths(tempDir, wanted); expect(loaded.size).toBe(2); for (const filePath of files) { const pf = loaded.get(filePath); @@ -106,11 +105,9 @@ describe('durable ParsedFile store carries M2 statement facts (#2082 U5)', () => mkParsedFile('src/same1.ts'), mkParsedFile('src/same2.ts'), ]); - await restoreDurableParsedFileShard(durableDir, tempDir, chunkHash); - const loaded = await loadParsedFilesForPaths( - tempDir, - new Set(['src/same1.ts', 'src/same2.ts']), - ); + const wanted = new Set(['src/same1.ts', 'src/same2.ts']); + expect(await durableChunkHasShards(tempDir, chunkHash, wanted)).toBe(true); + const loaded = await loadParsedFilesForPaths(tempDir, wanted); const c1 = (loaded.get('src/same1.ts') as { cfgSideChannel?: FunctionCfg[] }).cfgSideChannel; const c2 = (loaded.get('src/same2.ts') as { cfgSideChannel?: FunctionCfg[] }).cfgSideChannel; expect(c1?.[0].bindings).toEqual(factCfg.bindings); diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index 14c4c8359..69a366c10 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { mkdtemp, rm } from 'fs/promises'; +import { mkdtemp, rm, readdir, writeFile, readFile } from 'fs/promises'; import { tmpdir } from 'os'; import path from 'path'; import { @@ -17,6 +17,7 @@ import { slimParseWorkerResultsForCache, type ParseCache, } from '../../src/storage/parse-cache.js'; +import { writeV8CacheFile } from '../../src/storage/v8-sidecar.js'; import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; const minimalResult = (overrides: Partial = {}): ParseWorkerResult => ({ @@ -243,11 +244,11 @@ describe('PARSE_CACHE_VERSION', () => { // collided, because each re-checked once and neither re-checked after the // other moved — which is why the rule is re-applied AT MERGE, not when the // number is picked. - it('pins SCHEMA_BUMP to 80 so concurrent bumps cannot silently collide (#2766, #3015, #3088)', () => { - expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(80); + it('pins SCHEMA_BUMP to 81 so concurrent bumps cannot silently collide (#2766, #3015, #3088)', () => { + expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(81); expect(PARSE_CACHE_BUCKET_COUNT).toBe(128); for (const taken of [ - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, ]) { expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).not.toBe(taken); } @@ -450,12 +451,10 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { }), 'utf-8', ); - await fs.writeFile( - path.join(cacheDir, `${goodKey}.json`), - JSON.stringify([minimalResult({ fileCount: 3 })]), - 'utf-8', - ); - await fs.writeFile(path.join(cacheDir, `${badKey}.json`), '{not-json', 'utf-8'); + await writeV8CacheFile(path.join(cacheDir, `${goodKey}.v8`), [ + minimalResult({ fileCount: 3 }), + ]); + await fs.writeFile(path.join(cacheDir, `${badKey}.v8`), '{not-json', 'utf-8'); const loaded = await loadParseCache(dir); expect(loaded.entries.size).toBe(0); @@ -510,7 +509,7 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { await saveParseCache(dir, cache); const persisted = await fs.readdir(path.join(dir, 'parse-cache')); expect(persisted).toContain('index.json'); - expect(persisted).toContain(`${chunkKey}.json`); + expect(persisted).toContain(`${chunkKey}.v8`); const loaded = await loadParseCache(dir); const reloaded = (await loadParseCacheChunk(loaded, chunkKey))?.[0]; expect(reloaded).toBeDefined(); @@ -543,11 +542,9 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { }), 'utf-8', ); - await fs.writeFile( - path.join(cacheDir, `${safeKey}.json`), - JSON.stringify([minimalResult({ fileCount: 9 })]), - 'utf-8', - ); + await writeV8CacheFile(path.join(cacheDir, `${safeKey}.v8`), [ + minimalResult({ fileCount: 9 }), + ]); const loaded = await loadParseCache(dir); expect(loaded.onDiskKeys?.size).toBe(1); const chunk = await loadParseCacheChunk(loaded, safeKey); @@ -577,7 +574,7 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { const cacheDir = path.join(dir, 'parse-cache'); const names = await fs.readdir(cacheDir); expect(names).toContain('index.json'); - expect(names.filter((n) => n.endsWith('.json') && n !== 'index.json').length).toBe(3); + expect(names.filter((n) => n.endsWith('.v8')).length).toBe(3); const loaded = await loadParseCache(dir); expect(loaded.onDiskKeys?.size).toBe(3); } finally { @@ -628,8 +625,8 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { usedKeys: new Set([k2]), }); const names = await fs.readdir(path.join(dir, 'parse-cache')); - expect(names).not.toContain(`${k1}.json`); - expect(names).toContain(`${k2}.json`); + expect(names).not.toContain(`${k1}.v8`); + expect(names).toContain(`${k2}.v8`); const loaded = await loadParseCache(dir); expect(loaded.onDiskKeys?.size).toBe(1); const chunk = await loadParseCacheChunk(loaded, k2); @@ -823,4 +820,116 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { await rm(dir, { recursive: true, force: true }); } }); + + it('writes a V8 shard and loads it with Map-preserving semantics', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-v8-')); + try { + const innerMap = new Map([ + ['k1', 'v1'], + ['k2', 'v2'], + ]); + const innerSet = new Set(['s1', 's2']); + const fake = minimalResult({ + fileCount: 9, + imports: [ + { + typeBindings: innerMap, + extras: innerSet, + } as unknown as ParseWorkerResult['imports'][number], + ], + }); + const key = 'f'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set([key]), + storagePath: dir, + onDiskKeys: new Set(), + }; + await persistParseCacheChunk(cache, key, [fake]); + const names = await readdir(path.join(dir, 'parse-cache')); + expect(names).toEqual(expect.arrayContaining([`${key}.v8`])); + expect(names.some((n) => n.endsWith('.json') && n !== 'index.json')).toBe(false); + const loaded = await loadParseCacheChunk(cache, key); + expect(loaded?.[0]?.fileCount).toBe(9); + const smuggled = loaded?.[0]?.imports[0] as unknown as { + typeBindings?: unknown; + extras?: unknown; + }; + expect(smuggled.typeBindings).toBeInstanceOf(Map); + expect([...(smuggled.typeBindings as Map)]).toEqual([ + ['k1', 'v1'], + ['k2', 'v2'], + ]); + expect(smuggled.extras).toBeInstanceOf(Set); + expect([...(smuggled.extras as Set)].sort()).toEqual(['s1', 's2']); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('treats a corrupt parse-cache V8 shard as a miss', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-v8-fb-')); + try { + const key = 'a'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set([key]), + storagePath: dir, + onDiskKeys: new Set(), + }; + await persistParseCacheChunk(cache, key, [minimalResult({ fileCount: 4 })]); + await writeFile(path.join(dir, 'parse-cache', `${key}.v8`), Buffer.from([1, 2, 3])); + const loaded = await loadParseCacheChunk(cache, key); + expect(loaded).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('saveParseCache copies an existing V8 shard', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-v8-copy-')); + try { + const key = 'c'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set([key]), + storagePath: dir, + onDiskKeys: new Set(), + }; + await persistParseCacheChunk(cache, key, [minimalResult({ fileCount: 42 })]); + const liveV8 = await readFile(path.join(dir, 'parse-cache', `${key}.v8`)); + await saveParseCache(dir, cache); + expect(await readdir(path.join(dir, 'parse-cache'))).toEqual( + expect.arrayContaining([`${key}.v8`, 'index.json']), + ); + expect(await readFile(path.join(dir, 'parse-cache', `${key}.v8`))).toEqual(liveV8); + const loaded = await loadParseCache(dir); + expect((await loadParseCacheChunk(loaded, key))?.[0]?.fileCount).toBe(42); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when the V8 shard is absent', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-v8-legacy-')); + try { + const key = 'b'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set([key]), + storagePath: dir, + onDiskKeys: new Set(), + }; + await persistParseCacheChunk(cache, key, [minimalResult({ fileCount: 7 })]); + await rm(path.join(dir, 'parse-cache', `${key}.v8`), { force: true }); + const loaded = await loadParseCacheChunk(cache, key); + expect(loaded).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts index 77ac0a48c..410d0c048 100644 --- a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts +++ b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts @@ -12,19 +12,19 @@ * OOM. * * The fix: workers ALSO write a durable, content-addressed ParsedFile store - * keyed by chunk hash (`parsedfile-cache/`); a warm hit BYTE-COPIES the chunk's - * durable shards into the run-scoped store so scope-resolution streams them - * exactly as on a cold run — zero re-parse, byte-identical. + * keyed by chunk hash (`parsedfile-cache/`); a warm hit LOADS those shards + * in place (no copy into the run-scoped store) so scope-resolution streams + * them exactly as on a cold run — zero re-parse, byte-identical. * * Two layers of coverage: - * (1) Store-level — the durable persist → restore → `loadParsedFilesForPaths` + * (1) Store-level — the durable persist → `loadParsedFilesForPaths` * round-trip at the EXACT seam scope-resolution consumes (phase.ts:255), * plus the index version gate and the prune-coherence rule. Build-free. * (2) Integration — a two-run `runChunkedParseAndResolve`: run #1 (all miss) * populates the durable store; run #2 (all hits) spawns NO worker and - * restores full coverage; the coherence gate re-dispatches when durable - * shards are absent; and a mixed-mode run (one file changed) hits the - * unchanged chunk while re-parsing the changed one. + * loads full coverage from durable shards; the coherence gate re-dispatches + * when durable shards are absent; and a mixed-mode run (one file changed) + * hits the unchanged chunk while re-parsing the changed one. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import fs from 'node:fs'; @@ -37,6 +37,9 @@ import { pathToFileURL } from 'node:url'; const prepareOverride = vi.hoisted(() => ({ impl: undefined as undefined | (() => Promise), })); +const persistOverride = vi.hoisted(() => ({ + impl: undefined as undefined | (() => Promise), +})); vi.mock('../../src/storage/parsedfile-store.js', async (importOriginal) => { const real = await importOriginal(); return { @@ -45,6 +48,14 @@ vi.mock('../../src/storage/parsedfile-store.js', async (importOriginal) => { prepareOverride.impl ? prepareOverride.impl() : real.prepareDurableParsedFileChunk(durableDir, chunkHash), + persistParsedFileChunk: ( + storagePath: string, + shardId: string, + parsedFiles: readonly ParsedFile[], + ) => + persistOverride.impl + ? persistOverride.impl() + : real.persistParsedFileChunk(storagePath, shardId, parsedFiles), }; }); @@ -57,9 +68,10 @@ import { } from '../../src/storage/parse-cache.js'; import { getDurableParsedFileDir, + getParsedFileStoreDir, prepareDurableParsedFileChunk, persistDurableParsedFileShardSync, - restoreDurableParsedFileShard, + durableChunkHasShards, loadParsedFilesForPaths, loadDurableParsedFileIndex, pruneAndSaveDurableParsedFileStore, @@ -92,29 +104,24 @@ describe('durable ParsedFile store — content-addressed warm-cache coverage', ( if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); }); - it('persist → restore → loadParsedFilesForPaths gives full coverage (the warm seam)', async () => { + it('persist → loadParsedFilesForPaths gives full coverage (the warm seam)', async () => { const durableDir = getDurableParsedFileDir(tempDir); const chunkHash = 'a'.repeat(64); const files = ['src/a.ts', 'src/b.ts']; // A worker would write this at flush on a cache MISS. persistDurableParsedFileShardSync(durableDir, chunkHash, 7, 0, files.map(mkParsedFile)); - - // The run-scoped store is cleared at parse start; a warm hit restores. await clearParsedFileStore(tempDir); - const restored = await restoreDurableParsedFileShard(durableDir, tempDir, chunkHash); - expect(restored).toBe(1); - - // This is the EXACT call scope-resolution makes (phase.ts:255). Full - // coverage ⇒ preExtractedByPath has every file ⇒ no main-thread extract. - const loaded = await loadParsedFilesForPaths(tempDir, new Set(files)); + const wanted = new Set(files); + expect(await durableChunkHasShards(tempDir, chunkHash, wanted)).toBe(true); + const loaded = await loadParsedFilesForPaths(tempDir, wanted); expect([...loaded.keys()].sort()).toEqual([...files].sort()); }); - it('restore returns 0 when the chunk has no durable shards (caller re-dispatches)', async () => { - const durableDir = getDurableParsedFileDir(tempDir); - const restored = await restoreDurableParsedFileShard(durableDir, tempDir, 'b'.repeat(64)); - expect(restored).toBe(0); + it('durableChunkHasShards is false when the chunk has no durable shards', async () => { + expect(await durableChunkHasShards(tempDir, 'b'.repeat(64), new Set(['missing.ts']))).toBe( + false, + ); }); it('prepares a fresh durable generation without retaining old worker shards', async () => { @@ -129,14 +136,14 @@ describe('durable ParsedFile store — content-addressed warm-cache coverage', ( const shards = fs .readdirSync(chunkDir) - .filter((name) => name.endsWith('.json')) + .filter((name) => name.endsWith('.v8')) .sort(); - expect(shards).toEqual([`${chunkHash}-w1-0.json`, `${chunkHash}-w2-0.json`]); - await restoreDurableParsedFileShard(durableDir, tempDir, chunkHash); - const files = await loadParsedFilesForPaths( - tempDir, - new Set(['old.ts', 'new-a.ts', 'new-b.ts']), + expect(shards).toEqual([`${chunkHash}-w1-0.v8`, `${chunkHash}-w2-0.v8`]); + const wanted = new Set(['old.ts', 'new-a.ts', 'new-b.ts']); + expect(await durableChunkHasShards(tempDir, chunkHash, new Set(['new-a.ts', 'new-b.ts']))).toBe( + true, ); + const files = await loadParsedFilesForPaths(tempDir, wanted); expect([...files.keys()].sort()).toEqual(['new-a.ts', 'new-b.ts']); }); @@ -147,10 +154,10 @@ describe('durable ParsedFile store — content-addressed warm-cache coverage', ( await pruneAndSaveDurableParsedFileStore(durableDir, PARSE_CACHE_VERSION, new Set([chunkHash])); expect(await loadDurableParsedFileIndex(durableDir, PARSE_CACHE_VERSION)).toEqual( - new Set([chunkHash]), + new Map([[chunkHash, new Set(['x.ts'])]]), ); // A schema bump (different version) invalidates the whole durable store. - expect(await loadDurableParsedFileIndex(durableDir, '999+9.9.9')).toEqual(new Set()); + expect(await loadDurableParsedFileIndex(durableDir, '999+9.9.9')).toEqual(new Map()); }); it('prune keeps only keepKeys subdirs with ≥1 shard, drops the rest, and re-indexes', async () => { @@ -165,7 +172,7 @@ describe('durable ParsedFile store — content-addressed warm-cache coverage', ( expect(fs.existsSync(path.join(durableDir, keep))).toBe(true); expect(fs.existsSync(path.join(durableDir, drop))).toBe(false); expect(await loadDurableParsedFileIndex(durableDir, PARSE_CACHE_VERSION)).toEqual( - new Set([keep]), + new Map([[keep, new Set(['keep.ts'])]]), ); }); }); @@ -173,22 +180,44 @@ describe('durable ParsedFile store — content-addressed warm-cache coverage', ( // ─── Layer 2: parse-impl integration (injected worker, build-free) ─────────── // A test worker that mirrors the production flush contract: it writes a -// run-scoped shard AND a durable, content-addressed shard (when the flush -// carries a chunk hash) using the SAME directory layout as the real worker. -// Empty-scope ParsedFiles round-trip through plain JSON identically to -// `mapReplacer` (no Map fields), so the store bytes match the production path. +// run-scoped V8 shard AND a durable, content-addressed V8 shard (when the +// flush carries a chunk hash) using the SAME directory layout as the real worker. const writeStoreWorker = (workerPath: string, markerPath: string): void => { fs.writeFileSync( workerPath, ` const fs = require('node:fs'); const path = require('node:path'); +const v8 = require('node:v8'); +const { createHash } = require('node:crypto'); const { parentPort, threadId, workerData } = require('node:worker_threads'); const storePath = workerData && workerData.parsedFileStoreStoragePath; const durablePath = workerData && workerData.durableParsedFileStoragePath; let shardSeq = 0; fs.writeFileSync(${JSON.stringify(markerPath)}, 'spawned'); parentPort.postMessage({ type: 'ready' }); +const writeV8 = (filePath, graph, paths) => { + const payload = v8.serialize(graph); + const listing = paths.some((p) => /[\\r\\n\\0]/.test(p)) + ? Buffer.alloc(0) + : Buffer.from(paths.length + '\\n' + paths.join('\\n') + '\\n', 'utf8'); + const MAGIC = Buffer.from('GNXV8CF1'); + const v8ver = Buffer.from(process.versions.v8, 'utf8'); + const nodeMajor = Number.parseInt(process.versions.node.split('.')[0], 10); + const header = Buffer.allocUnsafe(16 + v8ver.length + 12); + MAGIC.copy(header, 0); + header.writeUInt32LE(5, 8); + header.writeUInt16LE(nodeMajor, 12); + header.writeUInt16LE(v8ver.length, 14); + v8ver.copy(header, 16); + let off = 16 + v8ver.length; + header.writeUInt32LE(listing.length === 0 ? 0 : paths.length, off); + header.writeUInt32LE(listing.length, off + 4); + header.writeUInt32LE(payload.length, off + 8); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const payloadHash = createHash('sha256').update(listing).update(payload).digest(); + fs.writeFileSync(filePath, Buffer.concat([header, listing, payload, payloadHash])); +}; const reset = () => ({ nodes: [], relationships: [], symbols: [], imports: [], calls: [], assignments: [], heritage: [], routes: [], fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], routerIncludes: [], @@ -220,18 +249,27 @@ parentPort.on('message', (msg) => { if (msg && msg.type === 'flush') { if ((storePath || durablePath) && accumulated.parsedFiles.length > 0) { const seq = shardSeq++; - const payload = JSON.stringify(accumulated.parsedFiles); + const paths = accumulated.parsedFiles.map((pf) => pf.filePath); + let wroteStore = false; if (durablePath && typeof msg.chunkHash === 'string') { - const dir = path.join(durablePath, msg.chunkHash); - fs.mkdirSync(dir, { recursive: true }); - fs.writeFileSync(path.join(dir, msg.chunkHash + '-w' + threadId + '-' + seq + '.json'), payload); + writeV8( + path.join(durablePath, msg.chunkHash, msg.chunkHash + '-w' + threadId + '-' + seq + '.v8'), + accumulated.parsedFiles, + paths, + ); } if (storePath) { - const dir = path.join(storePath, 'parsedfile-store'); - fs.mkdirSync(dir, { recursive: true }); - fs.writeFileSync(path.join(dir, 'w' + threadId + '-' + seq + '.json'), payload); - accumulated.parsedFiles = []; + writeV8( + path.join(storePath, 'parsedfile-store', 'w' + threadId + '-' + seq + '.v8'), + accumulated.parsedFiles, + paths, + ); + wroteStore = true; } + const keepForMain = accumulated.parsedFiles.some((pf) => + pf.filePath.includes('persist-fallback') + ); + if (wroteStore && !keepForMain) accumulated.parsedFiles = []; } parentPort.postMessage({ type: 'result', data: accumulated }); accumulated = reset(); @@ -260,6 +298,8 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { }); afterEach(() => { if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); + prepareOverride.impl = undefined; + persistOverride.impl = undefined; }); const writeFile = (rel: string, content: string): { path: string; size: number } => { @@ -329,7 +369,7 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { expect(fs.existsSync(markerPath)).toBe(true); // worker ran (miss) const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); expect(fs.existsSync(chunkDir)).toBe(true); - expect(fs.readdirSync(chunkDir).filter((n) => n.endsWith('.json')).length).toBeGreaterThan(0); + expect(fs.readdirSync(chunkDir).filter((n) => n.endsWith('.v8')).length).toBeGreaterThan(0); expect(cache.usedKeys.has(chunkHash)).toBe(true); }); @@ -343,6 +383,39 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { } }); + it('retains worker ParsedFiles when the main-thread run-store write fails', async () => { + const f = writeFile( + 'src/persist-fallback.ts', + 'export function persistFallback() { return 1; }\n', + ); + persistOverride.impl = () => Promise.resolve(false); + + const result = await run(newCache(), [f]); + + expect(result.parsedFiles.map((parsed) => parsed.filePath)).toContain(f.path); + }); + + it('does not snapshot durable shards when the parse-cache payload is missing', async () => { + const f = writeFile('src/orphan.ts', 'export function orphan() { return 1; }\n'); + const chunkHash = computeChunkHash([ + { + filePath: f.path, + contentHash: fileContentHash(fs.readFileSync(path.join(repoDir, f.path), 'utf-8')), + }, + ]); + const durableDir = getDurableParsedFileDir(storageDir); + persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile(f.path)]); + await pruneAndSaveDurableParsedFileStore(durableDir, PARSE_CACHE_VERSION, new Set([chunkHash])); + + await run(newCache(), [f]); + + const runShards = fs + .readdirSync(getParsedFileStoreDir(storageDir)) + .filter((name) => name.endsWith('.v8')); + expect(runShards.length).toBeGreaterThan(0); + expect(runShards.every((name) => !name.startsWith(chunkHash))).toBe(true); + }); + it('a repeated cache miss replaces the durable chunk generation', async () => { const f = writeFile('src/repeated.ts', 'export function repeated() { return 1; }\n'); const chunkHash = computeChunkHash([ @@ -356,11 +429,15 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { await run(newCache(), [f]); const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); - const shards = fs.readdirSync(chunkDir).filter((name) => name.endsWith('.json')); + const shards = fs.readdirSync(chunkDir).filter((name) => name.endsWith('.v8')); expect(shards).toHaveLength(1); - const parsed = JSON.parse(fs.readFileSync(path.join(chunkDir, shards[0]!), 'utf-8')) as Array<{ - filePath: string; - }>; + const shard = shards[0]; + if (!shard) throw new Error('expected one durable V8 shard'); + const { tryLoadV8Cache } = await import('../../src/storage/v8-sidecar.js'); + const hit = await tryLoadV8Cache(path.join(chunkDir, shard)); + expect(hit?.kind).toBe('hit'); + if (hit?.kind !== 'hit') return; + const parsed = hit.value as Array<{ filePath: string }>; expect(parsed.map((item) => item.filePath)).toEqual(['src/repeated.ts']); }); @@ -420,6 +497,33 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { expect(fs.existsSync(markerPath)).toBe(true); }); + it('coherence gate: a parse-cache hit with a corrupt durable shard re-dispatches', async () => { + const f = writeFile('src/corrupt.ts', 'export function corrupt() { return 1; }\n'); + const cache = newCache(); + + await run(cache, [f]); + await persistCaches(cache); + const chunkHash = computeChunkHash([ + { + filePath: f.path, + contentHash: fileContentHash('export function corrupt() { return 1; }\n'), + }, + ]); + const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); + const shard = fs.readdirSync(chunkDir).find((name) => name.endsWith('.v8')); + expect(shard).toBeDefined(); + if (!shard) return; + fs.writeFileSync(path.join(chunkDir, shard), Buffer.from([0, 1, 2])); + + const { loadParseCache } = await import('../../src/storage/parse-cache.js'); + const warm = await loadParseCache(storageDir); + fs.rmSync(markerPath, { force: true }); + + await run(warm as ReturnType, [f]); + + expect(fs.existsSync(markerPath)).toBe(true); + }); + it('mixed-mode: changing one file re-parses its chunk while the unchanged chunk restores', async () => { // Force one file per chunk (chunkByteBudget: 1) so a and b hash to DISTINCT // chunks — the true mixed-mode the pr-2038 mixed-mode gap warns about. @@ -446,7 +550,7 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { await run(warm as ReturnType, [a, b2], 1); - // The worker spawned (for the changed file b); a was restored from durable. + // The worker spawned (for the changed file b); a was loaded from durable. expect(fs.existsSync(markerPath)).toBe(true); // a's UNCHANGED chunk is still a hit served from the durable store. expect((warm as ReturnType).usedKeys.has(aHash)).toBe(true); diff --git a/gitnexus/test/unit/parsedfile-store.test.ts b/gitnexus/test/unit/parsedfile-store.test.ts index d799f2054..39fd4c9fe 100644 --- a/gitnexus/test/unit/parsedfile-store.test.ts +++ b/gitnexus/test/unit/parsedfile-store.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi } from 'vitest'; import { promises as nodeFsPromises } from 'node:fs'; +import v8 from 'node:v8'; import { mkdtemp, rm, readdir, readFile, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import path from 'path'; @@ -9,7 +10,7 @@ import { persistParsedFileChunk, persistParsedFileShardSync, persistDurableParsedFileShardSync, - restoreDurableParsedFileShard, + durableChunkHasShards, loadParsedFilesForPaths, getParsedFileStoreDir, getDurableParsedFileDir, @@ -246,24 +247,9 @@ describe('parsedfile-store', () => { const files = [makeParsedFile('a.c'), makeParsedFile('b.c')]; await persistParsedFileChunk(asyncDir, 'shard', files); persistParsedFileShardSync(syncDir, 'shard', files); - const asyncBytes = await readFile( - path.join(getParsedFileStoreDir(asyncDir), 'shard.json'), - 'utf-8', - ); - const syncBytes = await readFile( - path.join(getParsedFileStoreDir(syncDir), 'shard.json'), - 'utf-8', - ); - expect(syncBytes).toBe(asyncBytes); - const asyncPaths = await readFile( - path.join(getParsedFileStoreDir(asyncDir), 'shard.json.paths'), - 'utf-8', - ); - const syncPaths = await readFile( - path.join(getParsedFileStoreDir(syncDir), 'shard.json.paths'), - 'utf-8', - ); - expect(syncPaths).toBe(asyncPaths); + const asyncBytes = await readFile(path.join(getParsedFileStoreDir(asyncDir), 'shard.v8')); + const syncBytes = await readFile(path.join(getParsedFileStoreDir(syncDir), 'shard.v8')); + expect(syncBytes.equals(asyncBytes)).toBe(true); } finally { await rm(asyncDir, { recursive: true, force: true }); await rm(syncDir, { recursive: true, force: true }); @@ -629,116 +615,71 @@ describe('parsedfile-store receiverChain sanitation', () => { } }); - it('writes a .json.paths sidecar and skips JSON for non-intersecting shards (#3087)', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-')); + it('writes one .v8 shard per chunk and skips deserialize for non-intersecting listings (#3087)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-v8-skip-')); + const deserialize = vi.spyOn(v8, 'deserialize'); try { await persistParsedFileChunk(dir, 'chunk-0', [makeParsedFile('a.c')]); await persistParsedFileChunk(dir, 'chunk-1', [makeParsedFile('b.c')]); - const storeDir = getParsedFileStoreDir(dir); - const names = await readdir(storeDir); - expect(names.sort()).toEqual([ - 'chunk-0.json', - 'chunk-0.json.paths', - 'chunk-1.json', - 'chunk-1.json.paths', + expect((await readdir(getParsedFileStoreDir(dir))).sort()).toEqual([ + 'chunk-0.v8', + 'chunk-1.v8', ]); - const readSpy = vi.spyOn(nodeFsPromises, 'readFile'); - try { - const loaded = await loadParsedFilesForPaths(dir, new Set(['b.c'])); - expect([...loaded.keys()]).toEqual(['b.c']); - const jsonReads = readSpy.mock.calls.filter(([p]) => { - const n = String(p); - return n.endsWith('.json') && !n.endsWith('.json.paths'); - }); - expect(jsonReads).toHaveLength(1); - expect(String(jsonReads[0][0])).toMatch(/chunk-1\.json$/); - } finally { - readSpy.mockRestore(); - } + deserialize.mockClear(); + const loaded = await loadParsedFilesForPaths(dir, new Set(['b.c'])); + expect([...loaded.keys()]).toEqual(['b.c']); + expect(deserialize).toHaveBeenCalledTimes(1); } finally { + deserialize.mockRestore(); await rm(dir, { recursive: true, force: true }); } }); - it('reads a shard when its sidecar is missing or garbage (#3087)', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-fb-')); + it('misses when the embedded path listing is corrupted', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-listing-fb-')); try { await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')]); - await persistParsedFileChunk(dir, 'bad', [makeParsedFile('b.c')]); - const storeDir = getParsedFileStoreDir(dir); - await rm(path.join(storeDir, 'ok.json.paths'), { force: true }); - await writeFile(path.join(storeDir, 'bad.json.paths'), 'not\x00valid', 'utf-8'); - const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c', 'b.c'])); - expect(loaded.has('a.c')).toBe(true); - expect(loaded.has('b.c')).toBe(true); + const dest = path.join(getParsedFileStoreDir(dir), 'ok.v8'); + const buf = await readFile(dest); + const v8len = buf.readUInt16LE(14); + const pathsOff = 16 + v8len + 12; + buf[pathsOff] = 0; + await writeFile(dest, buf); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); + expect(loaded.has('a.c')).toBe(false); } finally { await rm(dir, { recursive: true, force: true }); } }); - it('reads a shard when its sidecar is truncated without a trailing newline (#3087)', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-trunc-')); - try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('wanted.c')]); - const storeDir = getParsedFileStoreDir(dir); - await writeFile(path.join(storeDir, 'ok.json.paths'), 'unrelated.c', 'utf-8'); - const loaded = await loadParsedFilesForPaths(dir, new Set(['wanted.c'])); - expect(loaded.has('wanted.c')).toBe(true); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it('reads a shard when its sidecar is a newline-terminated partial listing', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-partial-')); - try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('wanted.c')]); - const storeDir = getParsedFileStoreDir(dir); - await writeFile(path.join(storeDir, 'ok.json.paths'), 'unrelated.c\n', 'utf-8'); - const loaded = await loadParsedFilesForPaths(dir, new Set(['wanted.c'])); - expect(loaded.has('wanted.c')).toBe(true); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it('reads a shard when its sidecar contains CR', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-cr-')); - try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('wanted.c')]); - const storeDir = getParsedFileStoreDir(dir); - await writeFile(path.join(storeDir, 'ok.json.paths'), 'unrelated.c\r\n', 'utf-8'); - const loaded = await loadParsedFilesForPaths(dir, new Set(['wanted.c'])); - expect(loaded.has('wanted.c')).toBe(true); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it('omits a sidecar when a filePath contains a newline and still loads JSON', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-nl-')); + it('omits a path listing when a filePath contains a newline and still loads', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-listing-nl-')); const weird = 'weird\nname.c'; + const deserialize = vi.spyOn(v8, 'deserialize'); try { await persistParsedFileChunk(dir, 'ok', [makeParsedFile(weird)]); - const storeDir = getParsedFileStoreDir(dir); - expect(await readdir(storeDir)).toEqual(['ok.json']); - const loaded = await loadParsedFilesForPaths(dir, new Set([weird])); - expect(loaded.has(weird)).toBe(true); + expect(await readdir(getParsedFileStoreDir(dir))).toEqual(['ok.v8']); + deserialize.mockClear(); + const loaded = await loadParsedFilesForPaths(dir, new Set(['unrelated.c'])); + expect(loaded.size).toBe(0); + expect(deserialize).toHaveBeenCalledTimes(1); + expect((await loadParsedFilesForPaths(dir, new Set([weird]))).has(weird)).toBe(true); } finally { + deserialize.mockRestore(); await rm(dir, { recursive: true, force: true }); } }); - it('removes a stale sidecar when a rewritten shard is no longer listing-safe', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-stale-')); + it('rewrites a shard in place when the path listing is no longer safe', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-listing-rewrite-')); const weird = 'weird\nname.c'; try { await persistParsedFileChunk(dir, 'ok', [makeParsedFile('safe.c')]); await persistParsedFileChunk(dir, 'ok', [makeParsedFile(weird)]); - const storeDir = getParsedFileStoreDir(dir); - expect(await readdir(storeDir)).toEqual(['ok.json']); - const loaded = await loadParsedFilesForPaths(dir, new Set([weird])); + expect(await readdir(getParsedFileStoreDir(dir))).toEqual(['ok.v8']); + const loaded = await loadParsedFilesForPaths(dir, new Set([weird, 'safe.c'])); expect(loaded.has(weird)).toBe(true); + expect(loaded.has('safe.c')).toBe(false); } finally { await rm(dir, { recursive: true, force: true }); } @@ -779,101 +720,128 @@ describe('parsedfile-store receiverChain sanitation', () => { } }); - it('restoreDurableParsedFileShard copies sidecars and returns JSON shard count (#3087)', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-restore-')); + it('restores a complete durable chunk into a stable run-store snapshot', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-durable-load-')); try { const durable = getDurableParsedFileDir(dir); persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); - const restored = await restoreDurableParsedFileShard(durable, dir, 'abc'); - expect(restored).toBe(1); - const storeDir = getParsedFileStoreDir(dir); - expect(await readdir(storeDir)).toEqual( - expect.arrayContaining(['abc-w1-0.json', 'abc-w1-0.json.paths']), - ); - expect(await readFile(path.join(storeDir, 'abc-w1-0.json.paths'), 'utf-8')).toBe('1\na.c\n'); + expect(await durableChunkHasShards(dir, 'abc', new Set(['a.c']))).toBe(true); + await rm(path.join(durable, 'abc'), { recursive: true, force: true }); const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); expect(loaded.has('a.c')).toBe(true); + expect(await readdir(getParsedFileStoreDir(dir))).toEqual(['abc-w1-0.v8']); } finally { await rm(dir, { recursive: true, force: true }); } }); - it('restoreDurableParsedFileShard unlinks a stale dest sidecar when the source has none', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-restore-stale-')); + it('rejects a durable chunk with corrupt or incomplete shard coverage', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-durable-partial-')); try { const durable = getDurableParsedFileDir(dir); persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); - const durableShard = path.join(durable, 'abc', 'abc-w1-0.json'); - await rm(`${durableShard}.paths`, { force: true }); - const storeDir = getParsedFileStoreDir(dir); - await nodeFsPromises.mkdir(storeDir, { recursive: true }); - await writeFile(path.join(storeDir, 'abc-w1-0.json.paths'), 'stale.c\n', 'utf-8'); - const restored = await restoreDurableParsedFileShard(durable, dir, 'abc'); - expect(restored).toBe(1); - await expect(readFile(path.join(storeDir, 'abc-w1-0.json.paths'), 'utf-8')).rejects.toThrow(); - const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); - expect(loaded.has('a.c')).toBe(true); + persistDurableParsedFileShardSync(durable, 'abc', 2, 0, [makeParsedFile('b.c')]); + await writeFile(path.join(durable, 'abc', 'abc-w2-0.v8'), Buffer.from([0, 1, 2])); + + expect(await durableChunkHasShards(dir, 'abc', new Set(['a.c', 'b.c']))).toBe(false); + expect(await readdir(getParsedFileStoreDir(dir))).toEqual([]); } finally { await rm(dir, { recursive: true, force: true }); } }); - it('drops a leftover sidecar before overwriting JSON so load cannot skip new paths', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-rewrite-')); + it('rejects valid durable shards that do not cover every indexed path', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-durable-missing-')); try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('stale.c')]); - const origUnlink = nodeFsPromises.unlink.bind(nodeFsPromises); - const origWrite = nodeFsPromises.writeFile.bind(nodeFsPromises); - const order: string[] = []; - const unlinkSpy = vi - .spyOn(nodeFsPromises, 'unlink') - .mockImplementation(async (p, ...rest) => { - order.push(`unlink:${path.basename(String(p))}`); - return origUnlink(p, ...rest); - }); - const writeSpy = vi - .spyOn(nodeFsPromises, 'writeFile') - .mockImplementation(async (p, data, enc) => { - order.push(`write:${path.basename(String(p))}`); - return origWrite(p, data, enc); - }); - try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')]); - } finally { - unlinkSpy.mockRestore(); - writeSpy.mockRestore(); - } - const jsonIdx = order.indexOf('write:ok.json'); - const pathsIdx = order.indexOf('unlink:ok.json.paths'); - expect(pathsIdx).toBeGreaterThanOrEqual(0); - expect(pathsIdx).toBeLessThan(jsonIdx); - const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); - expect(loaded.has('a.c')).toBe(true); + const durable = getDurableParsedFileDir(dir); + persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); + + expect(await durableChunkHasShards(dir, 'abc', new Set(['a.c', 'b.c']))).toBe(false); + expect(await readdir(getParsedFileStoreDir(dir))).toEqual([]); } finally { await rm(dir, { recursive: true, force: true }); } }); - it('persist still succeeds when the sidecar write fails', async () => { - const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-sidecar-enospc-')); + it('run-store shards overlay durable hits for the same path', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-overlay-')); try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('stale.c')]); - const orig = nodeFsPromises.writeFile.bind(nodeFsPromises); - const spy = vi.spyOn(nodeFsPromises, 'writeFile').mockImplementation(async (p, data, enc) => { - if (String(p).endsWith('.paths')) { - throw Object.assign(new Error('ENOSPC'), { code: 'ENOSPC' }); - } - return orig(p, data, enc); - }); - try { - await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')]); - } finally { - spy.mockRestore(); - } + const durable = getDurableParsedFileDir(dir); + persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); + expect(await durableChunkHasShards(dir, 'abc', new Set(['a.c']))).toBe(true); + persistParsedFileShardSync(dir, 'w1-0', [makeParsedFile('other.c')]); + persistParsedFileShardSync(dir, 'w1-1', [makeStoreEntry('a.c', { moduleScope: 'from-run' })]); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c', 'other.c'])); + expect(loaded.get('a.c')?.moduleScope).toBe('from-run'); + expect(loaded.has('other.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('clearParsedFileStore leaves the durable cache intact', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-durable-keep-')); + try { + const durable = getDurableParsedFileDir(dir); + persistDurableParsedFileShardSync(durable, 'abc', 1, 0, [makeParsedFile('a.c')]); + const src = path.join(durable, 'abc', 'abc-w1-0.v8'); + const before = await readFile(src); + persistParsedFileShardSync(dir, 'w1-0', [makeParsedFile('run.c')]); + await clearParsedFileStore(dir); + expect(await readFile(src)).toEqual(before); + expect(await durableChunkHasShards(dir, 'abc', new Set(['a.c']))).toBe(true); + expect((await loadParsedFilesForPaths(dir, new Set(['a.c']))).has('a.c')).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('round-trips Maps and shared def identity through V8 (#3089)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-v8-id-')); + try { + const def = { + nodeId: 'Function:a.c:fn', + filePath: 'a.c', + type: 'Function' as const, + qualifiedName: 'fn', + }; + const pf = makeParsedFile('a.c'); + (pf.localDefs as unknown as object[])[0] = def; + (pf.scopes[0] as { ownedDefs: object[] }).ownedDefs = [def]; + await persistParsedFileChunk(dir, 'ok', [pf]); + const loadedFile = (await loadParsedFilesForPaths(dir, new Set(['a.c']))).get('a.c'); + expect(loadedFile).toBeDefined(); + if (!loadedFile) return; + expect(loadedFile.scopes[0].bindings).toBeInstanceOf(Map); + expect(loadedFile.localDefs[0]).toBe(loadedFile.scopes[0].ownedDefs[0]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('treats a missing or corrupt V8 shard as a miss with no JSON fallback', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-v8-miss-')); + try { + await persistParsedFileChunk(dir, 'gone', [makeParsedFile('a.c')]); + await persistParsedFileChunk(dir, 'junk', [makeParsedFile('b.c')]); const storeDir = getParsedFileStoreDir(dir); - await expect(readFile(path.join(storeDir, 'ok.json.paths'), 'utf-8')).rejects.toThrow(); - const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c'])); - expect(loaded.has('a.c')).toBe(true); + await rm(path.join(storeDir, 'gone.v8')); + await writeFile(path.join(storeDir, 'junk.v8'), Buffer.from([0, 1, 2, 3, 4])); + const loaded = await loadParsedFilesForPaths(dir, new Set(['a.c', 'b.c'])); + expect(loaded.size).toBe(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('returns false when the atomic V8 publish cannot replace the dest', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'pfstore-v8-blocked-')); + try { + const dest = path.join(getParsedFileStoreDir(dir), 'ok.v8'); + await nodeFsPromises.mkdir(dest, { recursive: true }); + await writeFile(path.join(dest, 'occupied'), 'x', 'utf-8'); + expect(await persistParsedFileChunk(dir, 'ok', [makeParsedFile('a.c')])).toBe(false); + expect((await loadParsedFilesForPaths(dir, new Set(['a.c']))).size).toBe(0); } finally { await rm(dir, { recursive: true, force: true }); } diff --git a/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts b/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts index a30005ed3..ba0fbe777 100644 --- a/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts +++ b/gitnexus/test/unit/repo-manager-registry-atomic-write.test.ts @@ -8,9 +8,10 @@ * startup (`LocalBackend.init` -> `refreshRepos`, nothing catches). * * Separate from repo-manager.test.ts: Vitest cannot vi.spyOn ESM namespace - * exports of fs/promises, and these tests must drive `fs.rename` itself — a - * delegating vi.mock is required (same split as repo-manager-rm-failure.test.ts - * and repo-manager-ensure-ignore-readonly.test.ts, #1549). + * exports of `node:fs` promises, and these tests must drive `retryRename`'s + * `fsp.rename` — a delegating vi.mock is required. `writeRegistry` publishes + * via `writeFileAtomic` (`node:fs`), so mocking `fs/promises` never intercepts + * the rename (same split as repo-manager-rm-failure.test.ts, #1549). */ import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; import path from 'path'; @@ -20,16 +21,17 @@ const fsCtx = vi.hoisted(() => ({ realRename: null as ((src: string, dst: string) => Promise) | null, })); -vi.mock('fs/promises', async (importOriginal) => { - const actual = await importOriginal(); - const d = actual.default; - fsCtx.realRename = d.rename.bind(d) as (src: string, dst: string) => Promise; +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + const promises = actual.promises; + fsCtx.realRename = promises.rename.bind(promises) as (src: string, dst: string) => Promise; fsCtx.renameMock.mockImplementation((src: string, dst: string) => fsCtx.realRename!(src, dst)); return { - default: new Proxy(d, { - get(target, prop) { + ...actual, + promises: new Proxy(promises, { + get(target, prop, receiver) { if (prop === 'rename') return fsCtx.renameMock; - const v = Reflect.get(target, prop, target) as unknown; + const v = Reflect.get(target, prop, receiver) as unknown; return typeof v === 'function' ? (v as (...args: unknown[]) => unknown).bind(target) : v; }, }), diff --git a/gitnexus/test/unit/storage/fs-atomic.test.ts b/gitnexus/test/unit/storage/fs-atomic.test.ts index deff8da8c..7e8049dab 100644 --- a/gitnexus/test/unit/storage/fs-atomic.test.ts +++ b/gitnexus/test/unit/storage/fs-atomic.test.ts @@ -4,10 +4,15 @@ * source-text guards in test/unit/group/insecure-tempfile.test.ts used to * approximate by regex, for three separate copies of the sequence. */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import fs from 'node:fs/promises'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; import path from 'node:path'; -import { writeFileAtomic } from '../../../src/storage/fs-atomic.js'; +import { + linkOrCopyFile, + writeFileAtomic, + writeFileAtomicBytes, + writeFileAtomicBytesSync, +} from '../../../src/storage/fs-atomic.js'; import { createTempDir } from '../../helpers/test-db.js'; describe('writeFileAtomic', () => { @@ -71,3 +76,120 @@ describe('writeFileAtomic', () => { expect((await fs.readdir(tmp.dbPath)).sort()).toEqual(['blocked', 'thing.json']); }); }); + +describe('writeFileAtomicBytes', () => { + let tmp: Awaited>; + let target: string; + + beforeEach(async () => { + tmp = await createTempDir('gitnexus-fs-atomic-bin-'); + target = path.join(tmp.dbPath, 'thing.bin'); + }); + + afterEach(async () => { + await tmp.cleanup(); + }); + + it('publishes binary data and leaves no tmp file behind', async () => { + const bytes = Buffer.from([0, 1, 255, 10]); + await writeFileAtomicBytes(target, bytes); + expect(Buffer.from(await fs.readFile(target))).toEqual(bytes); + expect((await fs.readdir(tmp.dbPath)).filter((f) => f !== 'thing.bin')).toEqual([]); + }); + + it('writeFileAtomicBytesSync publishes the same bytes', async () => { + const bytes = Buffer.from('hello'); + writeFileAtomicBytesSync(target, bytes); + expect(Buffer.from(await fs.readFile(target))).toEqual(bytes); + }); +}); + +describe('linkOrCopyFile (#3090)', () => { + let tmp: Awaited>; + + beforeEach(async () => { + tmp = await createTempDir('gitnexus-link-or-copy-'); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await tmp.cleanup(); + }); + + const hardlinksWork = async (): Promise => { + const a = path.join(tmp.dbPath, '.probe-a'); + const b = path.join(tmp.dbPath, '.probe-b'); + await fs.writeFile(a, 'x'); + try { + await fs.link(a, b); + return true; + } catch { + return false; + } + }; + + it('hardlinks when the filesystem allows it', async () => { + if (!(await hardlinksWork())) return; + const src = path.join(tmp.dbPath, 'src.bin'); + const dst = path.join(tmp.dbPath, 'dst.bin'); + await fs.writeFile(src, 'payload'); + await linkOrCopyFile(src, dst); + const dstFd = await fs.open(dst, 'r'); + try { + const [s, d] = await Promise.all([fs.stat(src), dstFd.stat()]); + expect(d.ino).toBe(s.ino); + expect(s.nlink).toBe(2); + expect(await dstFd.readFile('utf-8')).toBe('payload'); + } finally { + await dstFd.close(); + } + }); + + it('falls back to tmp+rename when link reports EXDEV', async () => { + const src = path.join(tmp.dbPath, 'src.bin'); + const dst = path.join(tmp.dbPath, 'dst.bin'); + await fs.writeFile(src, 'payload'); + const err = Object.assign(new Error('cross-device'), { code: 'EXDEV' }); + vi.spyOn(fs, 'link').mockRejectedValue(err); + await linkOrCopyFile(src, dst); + expect(await fs.readFile(dst, 'utf-8')).toBe('payload'); + const [s, d] = await Promise.all([fs.stat(src), fs.stat(dst)]); + if (s.ino !== 0) expect(d.ino).not.toBe(s.ino); + expect(s.nlink).toBe(1); + expect((await fs.readdir(tmp.dbPath)).filter((f) => f.includes('.tmp.'))).toEqual([]); + }); + + it('replaces an existing dest via rename without touching src', async () => { + const src = path.join(tmp.dbPath, 'src.bin'); + const dst = path.join(tmp.dbPath, 'dst.bin'); + await fs.writeFile(src, 'new-bytes'); + await fs.writeFile(dst, 'old-bytes'); + await linkOrCopyFile(src, dst); + expect(await fs.readFile(dst, 'utf-8')).toBe('new-bytes'); + expect(await fs.readFile(src, 'utf-8')).toBe('new-bytes'); + }); + + it('does not write through a dest that is already a hardlink to other durable bytes', async () => { + if (!(await hardlinksWork())) return; + const durable = path.join(tmp.dbPath, 'durable.bin'); + const dst = path.join(tmp.dbPath, 'dst.bin'); + const src = path.join(tmp.dbPath, 'src.bin'); + await fs.writeFile(durable, 'DURABLE'); + await fs.link(durable, dst); + await fs.writeFile(src, 'FRESH'); + const durableFd = await fs.open(durable, 'r'); + try { + const before = await durableFd.stat(); + vi.spyOn(fs, 'link').mockRejectedValue(Object.assign(new Error('exdev'), { code: 'EXDEV' })); + await linkOrCopyFile(src, dst); + const after = await durableFd.stat(); + expect(await durableFd.readFile('utf-8')).toBe('DURABLE'); + expect(after.ino).toBe(before.ino); + expect(after.nlink).toBe(1); + expect(await fs.readFile(dst, 'utf-8')).toBe('FRESH'); + expect((await fs.stat(dst)).ino).not.toBe(before.ino); + } finally { + await durableFd.close(); + } + }); +}); diff --git a/gitnexus/test/unit/v8-sidecar.test.ts b/gitnexus/test/unit/v8-sidecar.test.ts new file mode 100644 index 000000000..fb88d986f --- /dev/null +++ b/gitnexus/test/unit/v8-sidecar.test.ts @@ -0,0 +1,204 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtemp, rm, writeFile, readFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import path from 'path'; +import { + inspectV8Cache, + internGraphStrings, + parseCachePathListing, + tryLoadV8Cache, + writeV8CacheFile, + V8_CACHE_FORMAT, +} from '../../src/storage/v8-sidecar.js'; + +describe('v8 cache envelope', () => { + it('interns duplicate strings in place and keeps Map identity', () => { + const pool = new Map(); + const m = new Map([['k', 'dup']]); + const graph = { a: 'dup', b: 'dup', m }; + internGraphStrings(graph, pool); + expect(graph.a).toBe(graph.b); + expect(graph.m).toBe(m); + expect(graph.m.get('k')).toBe(graph.a); + expect([...pool.keys()].sort()).toEqual(['dup', 'k']); + expect(pool.get('dup')).toBe('dup'); + }); + + it('round-trips a live graph including Maps', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-')); + try { + const filePath = path.join(dir, 'shard.v8'); + const graph = { n: 1, nested: { s: 'x' }, map: new Map([['a', 1]]) }; + expect(await writeV8CacheFile(filePath, graph)).toBe(true); + const hit = await tryLoadV8Cache(filePath); + expect(hit?.kind).toBe('hit'); + if (hit?.kind !== 'hit') return; + const value = hit.value as typeof graph; + expect(value.n).toBe(1); + expect(value.map).toBeInstanceOf(Map); + expect(value.map.get('a')).toBe(1); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('skips deserialize when a valid path listing misses wantPaths', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-skip-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, [{ filePath: 'a.c' }], ['a.c']); + const skip = await tryLoadV8Cache(filePath, undefined, new Set(['other.c'])); + expect(skip?.kind).toBe('skip'); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when the path listing bytes are corrupted', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-failclosed-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, [{ filePath: 'a.c' }], ['a.c']); + const buf = await readFile(filePath); + const v8len = buf.readUInt16LE(14); + const pathBytesOff = 16 + v8len + 4; + const pathBytes = buf.readUInt32LE(pathBytesOff); + const pathsOff = 16 + v8len + 12; + buf.fill(0x00, pathsOff, pathsOff + Math.min(pathBytes, 1)); + await writeFile(filePath, buf); + const hit = await tryLoadV8Cache(filePath, undefined, new Set(['other.c'])); + expect(hit).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when a same-length path listing is rewritten without the digest', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-list-tamper-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, [{ filePath: 'a.c' }], ['a.c']); + const buf = await readFile(filePath); + const v8len = buf.readUInt16LE(14); + const pathsOff = 16 + v8len + 12; + const listing = Buffer.from('1\nb.c\n'); + listing.copy(buf, pathsOff); + await writeFile(filePath, buf); + expect(await inspectV8Cache(filePath)).toBeUndefined(); + expect(await tryLoadV8Cache(filePath, undefined, new Set(['other.c']))).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('treats invalid UTF-8 path listing bytes as untrusted', () => { + expect(parseCachePathListing(Buffer.from([0x31, 0x0a, 0xff, 0x0a]))).toBeNull(); + }); + + it('misses when the path listing is invalid UTF-8 instead of skipping', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-utf8-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, [{ filePath: 'a.c' }], ['a.c']); + const buf = await readFile(filePath); + const v8len = buf.readUInt16LE(14); + const pathBytes = buf.readUInt32LE(16 + v8len + 4); + const pathsOff = 16 + v8len + 12; + buf.fill(0xff, pathsOff, pathsOff + Math.min(pathBytes, 1)); + await writeFile(filePath, buf); + const hit = await tryLoadV8Cache(filePath, undefined, new Set(['other.c'])); + expect(hit).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('deserializes when the envelope path count disagrees with the listing', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-count-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, [{ filePath: 'a.c' }], ['a.c']); + const buf = await readFile(filePath); + const v8len = buf.readUInt16LE(14); + buf.writeUInt32LE(2, 16 + v8len); + await writeFile(filePath, buf); + expect(await inspectV8Cache(filePath)).toBeUndefined(); + const hit = await tryLoadV8Cache(filePath, undefined, new Set(['other.c'])); + expect(hit?.kind).toBe('hit'); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when internGraphStrings throws during materialization', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-intern-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, { s: 'x' }); + const pool = new Map(); + pool.set = () => { + throw new Error('intern boom'); + }; + expect(await tryLoadV8Cache(filePath, pool)).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when the recorded Node major does not match this runtime', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-compat-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, { ok: true }); + const buf = await readFile(filePath); + buf.writeUInt16LE(1, 12); + await writeFile(filePath, buf); + expect(await tryLoadV8Cache(filePath)).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when the recorded V8 version does not match this runtime', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-v8-compat-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, { ok: true }); + const buf = await readFile(filePath); + buf[16] ^= 1; + await writeFile(filePath, buf); + expect(await tryLoadV8Cache(filePath)).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses when the payload checksum rejects same-size corruption', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-payload-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeV8CacheFile(filePath, { ok: true }); + const buf = await readFile(filePath); + expect(buf.readUInt32LE(8)).toBe(V8_CACHE_FORMAT); + const v8len = buf.readUInt16LE(14); + const pathBytes = buf.readUInt32LE(16 + v8len + 4); + const payloadOff = 16 + v8len + 12 + pathBytes; + buf[payloadOff] ^= 0xff; + await writeFile(filePath, buf); + expect(await tryLoadV8Cache(filePath)).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('misses a garbage magic without throwing', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'v8cf-magic-')); + try { + const filePath = path.join(dir, 'shard.v8'); + await writeFile(filePath, Buffer.alloc(64, 7)); + expect(await tryLoadV8Cache(filePath)).toBeUndefined(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +});