From 9772bcd24fdfb872a8ff2e8f5e44e4fc1f4a1834 Mon Sep 17 00:00:00 2001 From: samuel mbabhazi <111171386+samuelmbabhazi@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:29:09 +0200 Subject: [PATCH 1/6] feat(api_impact): list the callers of the requesting function as indirect consumers (#3720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(api_impact): list the callers of the requesting function as indirect consumers The FETCHES edge is file-level, so api_impact saw only the file that sends the request. The files that call its request function never mention the URL and were missing from the report (#3644). The edge's reason now names the callables whose spans contain the request sites, one `|via:` field each, under the containment rule the R3-6 sink set already uses; that rule moves into EnclosingCallableIndex so both stay one. Several sites of one file for one route now share the edge instead of colliding on its id. api_impact follows the CALLS edges into those callables one level up and reports the callers as `indirectConsumers`, apart from the direct consumers and outside the risk level. Direct consumer files and the handler file are left out. An index built before the attribution carries no field. Expo navigation sites carry their real line so they attribute to the component that contains them. Closes #3644 * fix(api_impact): attribute a request site by its AST enclosing callable A helper defined on the line of the request has the smaller span, so the line-only lookup named it and the callers of the real requesting function vanished from indirectConsumers. The parser now records the enclosing callable of each fetch site and of each wrapper call (through the call's position), and attribution prefers that identity; the span lookup remains for sites found by text scan. The same preference applies to the R3-6 sink set. A real analyze of the fixture followed by api_impact pins the helper case and the wrapper case. * test(bench): rebaseline receiver resolution for the indirect-consumers fixture The api-impact-indirect-consumers fixture adds three receiver-unresolved sites to the corpus: two `.then(...)` calls on the untyped return of `fetch(...)` and of the local wrapper `apiFetch(...)`, and one `req.query.path` read inside an inline Express handler with an untyped `req`. Call drops 138 -> 140, reads 30 -> 31, all kinds 168 -> 171. No shape-arm state and no existing site changed. * fix(api_impact): preserve module-scope request attribution Keep AST-confirmed module scope distinct from missing attribution for direct fetches, HTTP clients, and wrapper calls. Prevent overlapping callable spans from inventing indirect consumers or process sinks. Preserve span fallback for callable forms the generic identity walk cannot name. Add eight regression cases, including real analyze/api_impact coverage. Rebaseline receiver-resolution counts for the two then calls and one property read added by this PR's existing fixtures; shape expectations are unchanged. --------- Co-authored-by: Gergo Magyar Co-authored-by: Gergő Magyar --- ARCHITECTURE.md | 7 + .../bench/receiver-resolution/BASELINE.md | 13 +- .../bench/receiver-resolution/baseline.json | 16 +- gitnexus/src/core/ingestion/call-processor.ts | 71 +++++-- .../src/core/ingestion/enclosing-callable.ts | 96 +++++++++ .../core/ingestion/pipeline-phases/routes.ts | 6 +- .../src/core/ingestion/process-processor.ts | 39 +--- .../route-extractors/fetch-wrapper.ts | 7 + .../core/ingestion/workers/parse-worker.ts | 33 ++- gitnexus/src/mcp/local/local-backend.ts | 130 +++++++++++- gitnexus/src/mcp/tools.ts | 2 +- gitnexus/test/fixtures/api-impact-seed.ts | 18 ++ .../api-impact-indirect-consumers/src/api.ts | 4 + .../api-impact-indirect-consumers/src/app.ts | 5 + .../src/client.ts | 15 ++ .../src/header.ts | 3 + .../src/server.ts | 15 ++ .../src/stats-panel.ts | 5 + .../src/stats.ts | 6 + .../src/viewer.ts | 5 + .../test/integration/api-impact-e2e.test.ts | 31 +++ .../api-impact-indirect-consumers-e2e.test.ts | 172 ++++++++++++++++ .../fetch-consumer-attribution.test.ts | 63 ++++++ gitnexus/test/unit/calltool-dispatch.test.ts | 49 +++++ .../unit/fetch-consumer-attribution.test.ts | 188 ++++++++++++++++++ .../test/unit/fetch-reason-parsing.test.ts | 24 +++ .../test/unit/fetch-wrapper-calls.test.ts | 4 + .../test/unit/fetch-wrapper-index.test.ts | 9 +- gitnexus/test/unit/process-processor.test.ts | 34 ++++ gitnexus/test/unit/result-merge.test.ts | 2 + 30 files changed, 1007 insertions(+), 65 deletions(-) create mode 100644 gitnexus/src/core/ingestion/enclosing-callable.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/api.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/app.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/client.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/header.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/server.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats-panel.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats.ts create mode 100644 gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/viewer.ts create mode 100644 gitnexus/test/integration/api-impact-indirect-consumers-e2e.test.ts create mode 100644 gitnexus/test/integration/resolvers/fetch-consumer-attribution.test.ts create mode 100644 gitnexus/test/unit/fetch-consumer-attribution.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 08bc2a14f..26bc36874 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -228,6 +228,13 @@ passes a route's URL to a callee not proven to be a wrapper is counted on the Route as `unverifiedConsumers` (proven consumers and the route's own file excluded), and `api_impact` then reports its consumer count as a lower bound. +A `FETCHES` edge is file-level. Its reason also names the callables whose spans +contain the request sites (`|via:`, under the same containment rule as the +R3-6 sink set). `api_impact` follows their `CALLS` edges one level up and lists +those callers as `indirectConsumers`, apart from the direct count and outside +the risk level: they never mention the URL, so no edge of their own could reach +the route. An index built before this attribution carries no such list. + An inline Express handler resolves to the anonymous Function node the capture layer names by position, so its Route carries a `handlerSymbolId`. In a file with several routes, response keys come from that handler's lines rather than diff --git a/gitnexus/bench/receiver-resolution/BASELINE.md b/gitnexus/bench/receiver-resolution/BASELINE.md index 6283d9ec9..10391e1e3 100644 --- a/gitnexus/bench/receiver-resolution/BASELINE.md +++ b/gitnexus/bench/receiver-resolution/BASELINE.md @@ -13,6 +13,18 @@ entries. The count arm remains 138 call drops and 168 total drops. Earlier entries below retain their original measurement contracts and source pins; member-name-only RESOLVES claims in those entries are historical. +## api_impact indirect-consumers fixture (#3644) + +The new `api-impact-indirect-consumers` fixture adds three `receiver-unresolved` +sites, none of them a resolver change: two `.then(...)` calls whose receiver is +the untyped return of `fetch(...)` (`src/client.ts`, external origin) and of the +local wrapper `apiFetch(...)` (`src/stats.ts`, in-program origin), and one +`req.query.path` read inside an inline Express handler whose `req` parameter is +untyped (`src/server.ts`), the same shape as the `express-inline-handlers` +entry below. Call drops move from 138 to 140 (`.ts` 16 -> 18, `no-chain` +36 -> 38), reads from 30 to 31, all kinds from 168 to 171. Every shape-arm +state is unchanged; no existing site changed state. + ## Typed shape bodies for JavaScript, Ruby, PHP and C (#3498) The JavaScript and Ruby rows had no type information anywhere, so every chain @@ -1478,7 +1490,6 @@ Main c30449394 is merged normally. Package-factory fallback declines inherited r The integrated build, both TypeScript configurations, 345 tests across ten files, Kotlin capture fingerprint and regenerated receiver check pass. The baseline remains 136 call drops, 166 drops across all kinds, 30 reads and zero writes, with unchanged shape states. The 5,521-record corpus is byte-identical to the candidate before the companion-header fix. These tests overlap earlier runs. Exact-head hosted CI remains required. - ### Rust pointer fields on the prepared C#/PHP/Kotlin stack The reviewed C# parent `b74fe72b8` is merged normally, retaining PHP diff --git a/gitnexus/bench/receiver-resolution/baseline.json b/gitnexus/bench/receiver-resolution/baseline.json index 490d49c8f..a973f6f58 100644 --- a/gitnexus/bench/receiver-resolution/baseline.json +++ b/gitnexus/bench/receiver-resolution/baseline.json @@ -202,16 +202,16 @@ } }, "countArm": { - "callDrops": 138, - "totalDropsAllKinds": 168, + "callDrops": 140, + "totalDropsAllKinds": 171, "bySiteKind": { - "call": 138, - "read": 30 + "call": 140, + "read": 31 }, "callDropsByExtension": { ".java": 49, ".py": 24, - ".ts": 16, + ".ts": 18, ".zig": 10, ".kt": 8, ".tsx": 6, @@ -225,15 +225,15 @@ }, "callDropsByShape": { "chain-field": 56, - "no-chain": 36, + "no-chain": 38, "chain-call": 32, "<>": 13, "chain-unwrap": 1 }, "callDropsByOrigin": { - "in-program": 57, + "in-program": 58, "unknown": 49, - "external": 30, + "external": 31, "<>": 2 } } diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 4ca250198..fa69b72e2 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -17,7 +17,7 @@ import { KnowledgeGraph } from '../graph/types.js'; import type { SemanticModel, SymbolTableReader } from './model/index.js'; -import { generateId } from '../../lib/utils.js'; +import { compareCodeUnits, generateId } from '../../lib/utils.js'; import type { ParsedImport, SymbolDefinition } from 'gitnexus-shared'; import { yieldToEventLoop } from './utils/event-loop.js'; import type { ExtractedRoute, ExtractedFetchCall } from './workers/parse-worker.js'; @@ -34,6 +34,7 @@ import { extractReturnTypeName } from './type-extractors/shared.js'; import { DATA_ROUTE_TABLE_SOURCE } from './route-extractors/data-route-table.js'; import { reconcileDispatchGuardRoutes } from './route-extractors/dispatch-guard.js'; import { toZeroBasedLine } from './utils/line-base.js'; +import { EnclosingCallableIndex } from './enclosing-callable.js'; const MAX_EXPORTS_PER_FILE = 500; const MAX_TYPE_NAME_LENGTH = 256; @@ -772,6 +773,14 @@ export const matchFetchRouteKeys = ( * route URL to the list of `routeNodeKey` identities at that URL; a single-verb * (or method-less) URL has a one-element list, keeping edges byte-identical to * the pre-#2289 behavior. + * + * The edge is file-level. Its reason also names the callables that hold the + * request sites, one `|via:` field each, under the same attribution as the + * R3-6 sink set (the parser's enclosing callable, else the innermost span): + * `api_impact` follows their CALLS edges one level up to list the callers of + * the requesting function (#3644). A site outside any callable adds no field, + * so a reason without attribution reads as before. An id that itself contains + * the field separator cannot be carried and is left out rather than split. */ export const processNextjsFetchRoutes = ( graph: KnowledgeGraph, @@ -794,6 +803,15 @@ export const processNextjsFetchRoutes = ( } } + const callables = new EnclosingCallableIndex(graph, new Set(fetchCalls.map((c) => c.filePath))); + + // One edge per (consumer file, route node), in first-seen order. Several + // sites in one file for the same route share the edge, each contributing the + // callable that contains it. + const edges = new Map< + string, + { sourceId: string; targetId: string; filePath: string; via: Set } + >(); for (const call of fetchCalls) { const normalized = normalizeFetchURL(call.fetchURL); if (!normalized) continue; @@ -801,36 +819,49 @@ export const processNextjsFetchRoutes = ( const routeKeys = matchFetchRouteKeys(normalized, routeUrlToKeys); if (!routeKeys) continue; const sourceId = generateId('File', call.filePath); + const callable = callables.of(call); + // Connect to every Route node at this URL (one per verb). + for (const routeKey of routeKeys) { + const targetId = generateId('Route', routeKey); + const id = generateId('FETCHES', `${sourceId}->${targetId}`); + let edge = edges.get(id); + if (edge === undefined) { + edge = { sourceId, targetId, filePath: call.filePath, via: new Set() }; + edges.set(id, edge); + } + if (callable !== undefined && !callable.includes('|')) edge.via.add(callable); + } + } + + for (const [id, edge] of edges) { // Extract consumer accessed keys if file content is available let reason = 'fetch-url-match'; - if (consumerContents) { - const content = consumerContents.get(call.filePath); - if (content) { - const accessedKeys = extractConsumerAccessedKeys(content); - if (accessedKeys.length > 0) { - reason = `fetch-url-match|keys:${accessedKeys.join(',')}`; - } + const content = consumerContents?.get(edge.filePath); + if (content) { + const accessedKeys = extractConsumerAccessedKeys(content); + if (accessedKeys.length > 0) { + reason = `${reason}|keys:${accessedKeys.join(',')}`; } } // Encode multi-fetch count so downstream can set confidence - const fetchCount = routeCountByFile.get(call.filePath) ?? 1; + const fetchCount = routeCountByFile.get(edge.filePath) ?? 1; if (fetchCount > 1) { reason = `${reason}|fetches:${fetchCount}`; } - // Connect to every Route node at this URL (one per verb). - for (const routeKey of routeKeys) { - const routeNodeId = generateId('Route', routeKey); - graph.addRelationship({ - id: generateId('FETCHES', `${sourceId}->${routeNodeId}`), - sourceId, - targetId: routeNodeId, - type: 'FETCHES', - confidence: 0.9, - reason, - }); + for (const via of [...edge.via].sort(compareCodeUnits)) { + reason = `${reason}|via:${via}`; } + + graph.addRelationship({ + id, + sourceId: edge.sourceId, + targetId: edge.targetId, + type: 'FETCHES', + confidence: 0.9, + reason, + }); } }; diff --git a/gitnexus/src/core/ingestion/enclosing-callable.ts b/gitnexus/src/core/ingestion/enclosing-callable.ts new file mode 100644 index 000000000..2830c8a80 --- /dev/null +++ b/gitnexus/src/core/ingestion/enclosing-callable.ts @@ -0,0 +1,96 @@ +import type { NodeLabel } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../graph/types.js'; + +/** Node labels whose span can contain a call site. */ +export const CALLABLE_LABELS: ReadonlySet = new Set([ + 'Function', + 'Method', + 'Constructor', +]); + +interface CallableSpan { + id: string; + start: number; + end: number; +} + +/** A call site: where it is, and, when the parser recorded it, which callable holds it. */ +export interface CallSite { + readonly filePath: string; + readonly lineNumber: number; + /** null means the AST confirmed module scope; undefined means no AST attribution. */ + readonly enclosingId?: string | null; +} + +/** + * Which callable contains a given line of a file. + * + * The parse phase records where the program reaches outward (a `fetch()`, an + * ORM query) as `filePath` + `lineNumber`, while the graph knows callables by + * their line spans. Attributing a site to the innermost callable whose span + * contains it turns the one into the other with no new extraction and no schema + * change. The R3-6 sink set and the FETCHES consumer attribution (#3644) both + * use this rule, through this one index, so they cannot drift apart. + * + * The parser records the enclosing callable of each site it captures from the + * AST (`enclosingId`), and that identity is used first: a line alone cannot + * tell a helper defined on the same line as the request apart from the + * function that makes it. The span lookup remains for sites found by text + * scan and for an identity the graph does not hold. An explicit null records + * module scope and must not fall back to an overlapping callable's line span. + * + * Span rule: innermost wins, a nested closure that performs the call is the + * callable, not the outer function that merely spans it. Among equal spans the + * first node seen keeps the site. Lines are 0-based on both sides, as the graph + * stores them and as tree-sitter reports them. + */ +export class EnclosingCallableIndex { + private readonly byFile = new Map(); + + /** + * One pass over the graph's nodes. `files` limits the index to the files + * that have sites to attribute; without it every file is indexed. + */ + constructor( + private readonly graph: KnowledgeGraph, + files?: ReadonlySet, + ) { + for (const node of graph.iterNodes()) { + if (!CALLABLE_LABELS.has(node.label)) continue; + const props = node.properties as { + filePath?: string; + startLine?: number; + endLine?: number; + }; + if (typeof props.filePath !== 'string') continue; + if (files !== undefined && !files.has(props.filePath)) continue; + if (typeof props.startLine !== 'number' || typeof props.endLine !== 'number') continue; + const span = { id: node.id, start: props.startLine, end: props.endLine }; + const list = this.byFile.get(props.filePath); + if (list === undefined) this.byFile.set(props.filePath, [span]); + else list.push(span); + } + } + + /** The callable that holds `site`: its recorded enclosing callable, else the innermost span. */ + of(site: CallSite): string | undefined { + if (site.enclosingId === null) return undefined; + if (site.enclosingId !== undefined) { + const node = this.graph.getNode(site.enclosingId); + if (node !== undefined && CALLABLE_LABELS.has(node.label)) return node.id; + } + return this.at(site.filePath, site.lineNumber); + } + + /** Id of the innermost callable of `filePath` whose span contains `line`, if any. */ + at(filePath: string, line: number): string | undefined { + const candidates = this.byFile.get(filePath); + if (candidates === undefined) return undefined; + let best: CallableSpan | undefined; + for (const c of candidates) { + if (line < c.start || line > c.end) continue; + if (best === undefined || c.end - c.start < best.end - best.start) best = c; + } + return best?.id; + } +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index c9b614099..10edf83e7 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -554,7 +554,11 @@ export const routesPhase: PipelinePhase = { while ((match = pattern.exec(content)) !== null) { const url = match[2] ?? match[1]; if (url && url.startsWith('/')) { - allFetchCalls.push({ filePath, fetchURL: url, lineNumber: 0 }); + // 0-based like the AST sites, so the navigation call is attributed + // to the component that contains it rather than to whatever spans + // line 0. + const lineNumber = content.slice(0, match.index).split('\n').length - 1; + allFetchCalls.push({ filePath, fetchURL: url, lineNumber }); } } } diff --git a/gitnexus/src/core/ingestion/process-processor.ts b/gitnexus/src/core/ingestion/process-processor.ts index fd75652c7..bc55671c0 100644 --- a/gitnexus/src/core/ingestion/process-processor.ts +++ b/gitnexus/src/core/ingestion/process-processor.ts @@ -20,6 +20,7 @@ import { isDev } from './utils/env.js'; import { PROCESS_DETECTION_BUDGET_DEFAULTS } from './process-detection-budget.js'; import { logger } from '../logger.js'; +import { EnclosingCallableIndex } from './enclosing-callable.js'; // ============================================================================ // CONFIGURATION // ============================================================================ @@ -730,14 +731,10 @@ export const traceFromEntryPoint = ( export interface OutwardActionSite { readonly filePath: string; readonly lineNumber: number; + /** The parser's enclosing callable, null for module scope, or absent without AST attribution. */ + readonly enclosingId?: string | null; } -const CALLABLE_SINK_LABELS: ReadonlySet = new Set([ - 'Function', - 'Method', - 'Constructor', -]); - /** * Functions that DO something outward — issue a request, run a query (R3-6). * @@ -754,10 +751,9 @@ const CALLABLE_SINK_LABELS: ReadonlySet = new Set([ * is too coarse to end a trace on — every function in a file containing one * would qualify. Attributing each site to the function whose range CONTAINS it * turns the same facts into the function-level signal the walk needs, with no - * new extraction, no new relation pair, and no schema change. - * - * Innermost wins: a nested closure that performs the call is the sink, not the - * outer function that merely spans it. + * new extraction, no new relation pair, and no schema change. The containment + * rule lives in {@link EnclosingCallableIndex}, shared with the FETCHES consumer + * attribution. */ export function buildSinkFunctionSet( graph: KnowledgeGraph, @@ -766,27 +762,10 @@ export function buildSinkFunctionSet( const sinks = new Set(); if (sites.length === 0) return sinks; - const byFile = new Map(); - for (const node of graph.iterNodes()) { - if (!CALLABLE_SINK_LABELS.has(node.label)) continue; - const props = node.properties as { filePath?: string; startLine?: number; endLine?: number }; - if (typeof props.filePath !== 'string') continue; - if (typeof props.startLine !== 'number' || typeof props.endLine !== 'number') continue; - const entry = { id: node.id, start: props.startLine, end: props.endLine }; - const list = byFile.get(props.filePath); - if (list === undefined) byFile.set(props.filePath, [entry]); - else list.push(entry); - } - + const callables = new EnclosingCallableIndex(graph); for (const site of sites) { - const candidates = byFile.get(site.filePath); - if (candidates === undefined) continue; - let best: { id: string; start: number; end: number } | undefined; - for (const c of candidates) { - if (site.lineNumber < c.start || site.lineNumber > c.end) continue; - if (best === undefined || c.end - c.start < best.end - best.start) best = c; - } - if (best !== undefined) sinks.add(best.id); + const id = callables.of(site); + if (id !== undefined) sinks.add(id); } return sinks; } diff --git a/gitnexus/src/core/ingestion/route-extractors/fetch-wrapper.ts b/gitnexus/src/core/ingestion/route-extractors/fetch-wrapper.ts index d0ea2bc19..c24e97cc5 100644 --- a/gitnexus/src/core/ingestion/route-extractors/fetch-wrapper.ts +++ b/gitnexus/src/core/ingestion/route-extractors/fetch-wrapper.ts @@ -34,6 +34,12 @@ export type CalleeTarget = export interface FetchWrapperCall extends ExtractedFetchCall { functionName: string; target: CalleeTarget; + /** + * Start of the call node in tree coordinates (0-based, before `lineOffset`), + * so the parse worker can find the node again and record the callable that + * holds the call (#3644). `lineNumber` stays 1-based as its readers expect. + */ + callPosition: { row: number; column: number }; } export interface ForwardedCallee { callee: string; @@ -941,6 +947,7 @@ export function createFetchWrapperContext( fetchURL, lineNumber: call.startPosition.row + lineOffset + 1, target, + callPosition: { row: call.startPosition.row, column: call.startPosition.column }, }); } return calls; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 81d543fa9..5d999ed24 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -22,7 +22,7 @@ import Rust from 'tree-sitter-rust'; import PHP from 'tree-sitter-php'; import Ruby from 'tree-sitter-ruby'; import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js'; -import { SupportedLanguages } from 'gitnexus-shared'; +import { buildPositionIndex, SupportedLanguages, type PositionIndex } from 'gitnexus-shared'; import { getLanguageForFileContent, getProvider } from '../languages/index.js'; import { getTreeSitterBufferSize, @@ -346,6 +346,12 @@ export interface ExtractedFetchCall { filePath: string; fetchURL: string; lineNumber: number; + /** + * generateId of the callable whose body holds the call, from the AST; + * null for a confirmed top-level call, absent when attribution is unknown + * (text scans or callable forms the identity walk cannot name) (#3644). + */ + enclosingId?: string | null; } export interface FetchWrapperDef extends FetchWrapperFacts { @@ -1729,6 +1735,19 @@ const processFileGroup = ( sourceLineMap, ); if (scopeExtractionFailed) (result.scopeExtractionFailures ??= []).push(file.path); + let requestScopeIndex: PositionIndex | undefined; + const requestEnclosingId = (node: SyntaxNode): string | null | undefined => { + const id = findEnclosingFunctionId(node, file.path, provider); + if (id !== null) return id; + if (parsedFile === undefined || scopeExtractionFailed) return undefined; + requestScopeIndex ??= buildPositionIndex( + parsedFile.scopes.filter((scope) => scope.kind === 'Function'), + ); + const { row, column } = node.startPosition; + return requestScopeIndex.atPosition(file.path, mapRow(row) + 1, column) === undefined + ? null + : undefined; + }; if (parsedFile !== undefined) { // Capture-time side-channel (#1983): `extractParsedFile` just ran the // provider's `emitScopeCaptures`, which (for C++ ADL/namespace marks, @@ -2031,10 +2050,12 @@ const processFileGroup = ( // Measured before this: 44 of 47 fetch calls in this repo pass a // variable, so the sink signal was absent from 94% of them and // sink-terminated flows could effectively never fire. + const fetchEnclosingId = requestEnclosingId(captureMap['route.fetch']); result.fetchCalls.push({ filePath: file.path, fetchURL: urlNode ? urlNode.text : '', lineNumber: captureMap['route.fetch'].startPosition.row + lineOffset, + enclosingId: fetchEnclosingId, }); continue; } @@ -2045,10 +2066,12 @@ const processFileGroup = ( const method = captureMap['http_client.method']?.text; const url = captureMap['http_client.url'].text; if (method && HTTP_CLIENT_ONLY_METHODS.has(method) && url.startsWith('/')) { + const clientEnclosingId = requestEnclosingId(captureMap['http_client']); result.fetchCalls.push({ filePath: file.path, fetchURL: url, lineNumber: captureMap['http_client'].startPosition.row + lineOffset, + enclosingId: clientEnclosingId, }); } continue; @@ -3309,7 +3332,13 @@ const processFileGroup = ( const calls = provider.extractFetchWrapperCalls?.(tree, parsedFile, lineOffset); const reexports = provider.extractFetchWrapperReexports?.(tree, parsedFile, lineOffset); const routeFacts = provider.extractRouteFacts?.(tree, parsedFile, lineOffset); - for (const call of calls ?? []) (result.fetchWrapperCalls ??= []).push(call); + for (const call of calls ?? []) { + // The wrapper call is a request site too (#3644): record the callable + // that holds it, as the direct fetch sites do. + const callNode = tree.rootNode.descendantForPosition(call.callPosition); + const callEnclosingId = requestEnclosingId(callNode); + (result.fetchWrapperCalls ??= []).push({ ...call, enclosingId: callEnclosingId }); + } for (const reexport of reexports ?? []) (result.fetchWrapperReexports ??= []).push(reexport); if (routeFacts) diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 401c795e8..dc6cdea20 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -1444,6 +1444,13 @@ interface ApiImpactRoute { middlewareDetection?: 'partial'; middlewareNote?: string; consumers: Array<{ name: string; file: string; accesses: string[]; attributionNote?: string }>; + /** + * Callers of the function that sends the request, one level up (#3644). + * Present only when a FETCHES edge names that function; an index built before + * the attribution has none. Direct consumer files and the handler file are + * left out, so an entry is a file the direct count does not already cover. + */ + indirectConsumers?: ApiImpactIndirectConsumer[]; mismatches?: Array<{ consumer: string; field: string; @@ -1459,9 +1466,64 @@ interface ApiImpactRoute { epistemic?: 'lower-bound'; unverifiedConsumers?: number; boundaries?: string[]; + /** Distinct files among `indirectConsumers`; not part of `riskLevel`. */ + indirectConsumers?: number; }; } +interface ApiImpactIndirectConsumer { + name: string; + file: string; + /** Name of the requesting function this caller reaches the route through. */ + via: string; +} + +/** A CALLS edge into one of the requesting functions a FETCHES reason names. */ +interface RequestFunctionCaller { + viaId: string; + viaName: string; + name: string; + filePath: string; +} + +/** + * The one-level callers of a route's requesting functions that the direct + * consumer list does not already cover. `undefined` when no consumer edge of + * the route names a requesting function, so the field stays absent instead of + * claiming an empty result the index could not have produced. + */ +function indirectConsumersOf( + route: { + filePath: string; + consumers: Array<{ filePath: string; viaSymbolIds?: string[] }>; + }, + callersByVia: ReadonlyMap, +): ApiImpactIndirectConsumer[] | undefined { + const directFiles = new Set(route.consumers.map((c) => c.filePath)); + const seen = new Set(); + const out: ApiImpactIndirectConsumer[] = []; + let attributed = false; + for (const consumer of route.consumers) { + for (const viaId of consumer.viaSymbolIds ?? []) { + attributed = true; + for (const caller of callersByVia.get(viaId) ?? []) { + if (caller.filePath === route.filePath || directFiles.has(caller.filePath)) continue; + const key = `${caller.filePath}\u0000${caller.name}\u0000${caller.viaName}`; + if (seen.has(key)) continue; + seen.add(key); + out.push({ name: caller.name, file: caller.filePath, via: caller.viaName }); + } + } + } + if (!attributed) return undefined; + return out.sort( + (a, b) => + compareCodeUnits(a.file, b.file) || + compareCodeUnits(a.name, b.name) || + compareCodeUnits(a.via, b.via), + ); +} + interface RouteRuntimeEvidence { runtimeConfirmed: boolean; runtimeSource: string | null; @@ -9739,6 +9801,7 @@ export class LocalBackend { filePath: string; accessedKeys?: string[]; fetchCount?: number; + viaSymbolIds?: string[]; }>; }> > { @@ -9797,6 +9860,7 @@ export class LocalBackend { filePath: string; accessedKeys?: string[]; fetchCount?: number; + viaSymbolIds?: string[]; }>; } >(); @@ -9837,9 +9901,11 @@ export class LocalBackend { }); } if (consumerName && consumerFile) { - // Parse accessed keys from reason field: "fetch-url-match|keys:data,pagination|fetches:3" + // Parse accessed keys from reason field: + // "fetch-url-match|keys:data,pagination|fetches:3|via:Function:src/api.ts:getGrants" let accessedKeys: string[] | undefined; let fetchCount: number | undefined; + let viaSymbolIds: string[] | undefined; if (fetchReason) { const keysMatch = fetchReason.match(/\|keys:([^|]+)/); if (keysMatch) { @@ -9849,12 +9915,17 @@ export class LocalBackend { if (fetchesMatch) { fetchCount = parseInt(fetchesMatch[1], 10); } + // One field per requesting function (#3644); an id may hold any + // character but the separator. + const via = [...fetchReason.matchAll(/\|via:([^|]+)/g)].map((m) => m[1]); + if (via.length > 0) viaSymbolIds = via; } routeMap.get(id)!.consumers.push({ name: consumerName, filePath: consumerFile, ...(accessedKeys ? { accessedKeys } : {}), ...(fetchCount && fetchCount > 1 ? { fetchCount } : {}), + ...(viaSymbolIds ? { viaSymbolIds } : {}), }); } } @@ -9901,6 +9972,49 @@ export class LocalBackend { return result; } + /** + * Callers of the requesting functions a route's FETCHES reasons name, one + * CALLS hop up, keyed by the requesting function's id (#3644). One query for + * every route in the report. `undefined` when the query fails, so the caller + * leaves the field out rather than reporting an empty list it did not verify. + */ + private async fetchRequestFunctionCallers( + repoId: string, + viaIds: string[], + ): Promise | undefined> { + const result = new Map(); + if (viaIds.length === 0) return result; + try { + const rows = await executeParameterized( + repoId, + ` + MATCH (caller)-[r:CodeRelation]->(via) + WHERE r.type = 'CALLS' AND list_contains($viaIds, via.id) + RETURN DISTINCT via.id AS viaId, via.name AS viaName, + caller.name AS callerName, caller.filePath AS callerFile + `, + { viaIds }, + ); + for (const row of rows) { + const viaId = row.viaId ?? row[0]; + const viaName = row.viaName ?? row[1]; + const name = row.callerName ?? row[2]; + const filePath = row.callerFile ?? row[3]; + if (!viaId || !viaName || !name || !filePath) continue; + let list = result.get(viaId); + if (!list) { + list = []; + result.set(viaId, list); + } + list.push({ viaId, viaName, name, filePath }); + } + } catch (e) { + logQueryError('api_impact:indirect-consumers', e); + return undefined; + } + return result; + } + /** * Same predicate as HTTP `getSourceAvailability`: full retention plus a live * checkout. Meta is the graph directory so a published shared-store commit @@ -10307,6 +10421,15 @@ export class LocalBackend { routes.map((r) => r.id), ); + // The FETCHES edge names the function that holds each request site; its + // callers, one level up, are consumers the file-level edge cannot see + // (#3644). Nothing to look up on an index built before that attribution. + const viaIds = new Set(); + for (const r of routes) { + for (const c of r.consumers) for (const id of c.viaSymbolIds ?? []) viaIds.add(id); + } + const callersByVia = await this.fetchRequestFunctionCallers(repo.lbugPath, [...viaIds]); + // Count verbs per handler from the FULL match (before the method filter) so a // method-scoped query still flags a multi-verb handler's partial middleware. const routeCountByHandler = new Map(); @@ -10360,6 +10483,9 @@ export class LocalBackend { const flows = flowMap.get(r.id) || []; const consumerCount = r.consumers.length; + const indirectConsumers = + callersByVia === undefined ? undefined : indirectConsumersOf(r, callersByVia); + const indirectFileCount = new Set(indirectConsumers?.map((c) => c.file)).size; // Risk level heuristic let riskLevel: 'LOW' | 'MEDIUM' | 'HIGH'; @@ -10405,10 +10531,12 @@ export class LocalBackend { } : {}), consumers, + ...(indirectConsumers ? { indirectConsumers } : {}), ...(mismatches.length > 0 ? { mismatches } : {}), executionFlows: flows, impactSummary: { directConsumers: consumerCount, + ...(indirectConsumers ? { indirectConsumers: indirectFileCount } : {}), affectedFlows: flows.length, riskLevel, ...(warning ? { warning } : {}), diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 9d30cf4ab..f3fab5b60 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -908,7 +908,7 @@ Returns routes that have both detected response keys AND consumers. Shows top-le WHEN TO USE: BEFORE modifying any API route handler. Shows what consumers depend on, what response fields they access, what middleware protects the route, and what execution flows it triggers. Requires at least "route" or "file" parameter. -Risk levels: LOW (0-3 consumers), MEDIUM (4-9 or any mismatches), HIGH (10+ consumers or mismatches with 4+ consumers). Mismatches with confidence "low" indicate the consumer file fetches multiple routes — property attribution is approximate. When impactSummary.epistemic is "lower-bound", impactSummary.unverifiedConsumers more files pass the route's URL to a function not proven to send HTTP requests, so directConsumers is a floor; riskLevel counts proven consumers only. +Risk levels: LOW (0-3 consumers), MEDIUM (4-9 or any mismatches), HIGH (10+ consumers or mismatches with 4+ consumers). Mismatches with confidence "low" indicate the consumer file fetches multiple routes — property attribution is approximate. When impactSummary.epistemic is "lower-bound", impactSummary.unverifiedConsumers more files pass the route's URL to a function not proven to send HTTP requests, so directConsumers is a floor; riskLevel counts proven consumers only. "indirectConsumers" lists the callers of the function that sends the request, one level up, each with that function's name in "via"; impactSummary.indirectConsumers counts their distinct files. They are reported apart from the direct consumers and do not feed riskLevel. The field is absent on an index built before the requesting function was recorded. Response shape is keyed on how many routes match, not on the data: exactly one match returns a single route object; two or more return { routes: [...], total: N }. The same URL can expose multiple HTTP verbs (e.g. GET and POST /api/orders are distinct routes that share the URL), so a bare-URL lookup may return the wrapped form — every route object carries its own "method" so verbs are distinguishable. Pass "method" to narrow to one verb; the single-object shape is returned only when exactly one route remains after filtering — a substring route/file match spanning several URLs can still return the wrapped form. A URL/file that exists but has no route for the given verb returns an error. Each route's "method" is the literal "*" for method-agnostic routes (e.g. Django function views), which match any "method" selector, or null for method-less routes (filesystem, Laravel resource), which never match a selector. Every route also carries "runtimeEvidence"; treat it as authoritative only when runtimeEvidence.confirmed is true. Combines route_map, shape_check, and impact data.`, annotations: READ_ONLY_TOOL_ANNOTATIONS, diff --git a/gitnexus/test/fixtures/api-impact-seed.ts b/gitnexus/test/fixtures/api-impact-seed.ts index 98f13211b..622f24c9d 100644 --- a/gitnexus/test/fixtures/api-impact-seed.ts +++ b/gitnexus/test/fixtures/api-impact-seed.ts @@ -55,6 +55,24 @@ export const API_IMPACT_SEED_DATA = [ CREATE (fn)-[:CodeRelation {type: 'HANDLES_ROUTE', confidence: 1.0, reason: 'nextjs-app-router', step: 0}]->(r)`, `MATCH (fn:Function), (r:Route) WHERE fn.id = 'func:secure-POST' AND r.id = 'Route:/api/secure' CREATE (fn)-[:CodeRelation {type: 'HANDLES_ROUTE', confidence: 1.0, reason: 'nextjs-app-router', step: 0}]->(r)`, + + // ─── Indirect consumers (#3644) ──────────────────────────────────── + // backend-client.ts is the file that sends the request; the edge names the + // function holding the site. App and Header call that function without ever + // mentioning the URL. useMulti calls it too, but already fetches the route + // itself, so it stays a direct consumer only. + `CREATE (f:File {id: 'file:services/backend-client.ts', name: 'backend-client.ts', filePath: 'services/backend-client.ts', content: 'export const fetchSecure = () => fetch("/api/secure")'})`, + `CREATE (fn:Function {id: 'func:fetchSecure', name: 'fetchSecure', filePath: 'services/backend-client.ts', startLine: 1, endLine: 3, isExported: true, content: 'export const fetchSecure = () => fetch("/api/secure")', description: 'Secure request function'})`, + `CREATE (fn:Function {id: 'func:App', name: 'App', filePath: 'App.tsx', startLine: 1, endLine: 20, isExported: true, content: 'export function App() { fetchSecure() }', description: 'App root'})`, + `CREATE (fn:Function {id: 'func:Header', name: 'Header', filePath: 'components/Header.tsx', startLine: 1, endLine: 12, isExported: true, content: 'export function Header() { fetchSecure() }', description: 'Header'})`, + `MATCH (a:File), (r:Route) WHERE a.id = 'file:services/backend-client.ts' AND r.id = 'Route:/api/secure' + CREATE (a)-[:CodeRelation {type: 'FETCHES', confidence: 0.9, reason: 'fetch-url-match|via:func:fetchSecure', step: 0}]->(r)`, + `MATCH (a:Function), (b:Function) WHERE a.id = 'func:App' AND b.id = 'func:fetchSecure' + CREATE (a)-[:CodeRelation {type: 'CALLS', confidence: 1.0, reason: 'direct', step: 0}]->(b)`, + `MATCH (a:Function), (b:Function) WHERE a.id = 'func:Header' AND b.id = 'func:fetchSecure' + CREATE (a)-[:CodeRelation {type: 'CALLS', confidence: 1.0, reason: 'direct', step: 0}]->(b)`, + `MATCH (a:Function), (b:Function) WHERE a.id = 'func:useMulti' AND b.id = 'func:fetchSecure' + CREATE (a)-[:CodeRelation {type: 'CALLS', confidence: 1.0, reason: 'direct', step: 0}]->(b)`, ]; export const API_IMPACT_FTS_INDEXES: FTSIndexDef[] = [ diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/api.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/api.ts new file mode 100644 index 000000000..95a3b83e7 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/api.ts @@ -0,0 +1,4 @@ +// A wrapper around fetch: its callers reach the route through it. +export function apiFetch(path: string) { + return fetch(path); +} diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/app.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/app.ts new file mode 100644 index 000000000..52789dc1a --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/app.ts @@ -0,0 +1,5 @@ +import { fetchRepos } from './client'; + +export async function loadApp() { + return fetchRepos(); +} diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/client.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/client.ts new file mode 100644 index 000000000..6b1b68d01 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/client.ts @@ -0,0 +1,15 @@ +// The one function that sends the request; its callers never mention the URL. +export async function fetchRepos() { + const res = await fetch('/api/repos'); + return res.json(); +} + +// A helper defined on the same line as the request: the request still belongs +// to fetchFile, not to decode. +export function fetchFile(path: string) { + const decode = (res: Response) => res.json(); return fetch(`/api/file?path=${path}`).then(decode); +} + +export function describeClient() { + return 'repos client'; +} diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/header.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/header.ts new file mode 100644 index 000000000..21b0cb25b --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/header.ts @@ -0,0 +1,3 @@ +import { fetchRepos } from './client'; + +export const refreshHeader = () => fetchRepos(); diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/server.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/server.ts new file mode 100644 index 000000000..a5c04875a --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/server.ts @@ -0,0 +1,15 @@ +import express from 'express'; + +export function createServer() { + const app = express(); + app.get('/api/repos', (_req, res) => { + res.json({ repos: [], total: 0 }); + }); + app.get('/api/file', (req, res) => { + res.json({ content: '', path: String(req.query.path) }); + }); + app.get('/api/stats', (_req, res) => { + res.json({ files: 0, symbols: 0 }); + }); + return app; +} diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats-panel.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats-panel.ts new file mode 100644 index 000000000..31a40dd47 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats-panel.ts @@ -0,0 +1,5 @@ +import { fetchStats } from './stats'; + +export function renderStatsPanel() { + return fetchStats(); +} diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats.ts new file mode 100644 index 000000000..4fd26f70e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/stats.ts @@ -0,0 +1,6 @@ +import { apiFetch } from './api'; + +// The request goes through the wrapper, and a helper shares its line again. +export function fetchStats() { + const pick = (res: Response) => res.json(); return apiFetch('/api/stats').then(pick); +} diff --git a/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/viewer.ts b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/viewer.ts new file mode 100644 index 000000000..1ac77dcee --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/api-impact-indirect-consumers/src/viewer.ts @@ -0,0 +1,5 @@ +import { fetchFile } from './client'; + +export function openFile(path: string) { + return fetchFile(path); +} diff --git a/gitnexus/test/integration/api-impact-e2e.test.ts b/gitnexus/test/integration/api-impact-e2e.test.ts index dad5d4652..f3335be81 100644 --- a/gitnexus/test/integration/api-impact-e2e.test.ts +++ b/gitnexus/test/integration/api-impact-e2e.test.ts @@ -98,6 +98,37 @@ withTestLbugDB( }); }); + // #3644: the callers of the function that sends the request, one level up. + describe('api_impact indirect consumers', () => { + it('lists the callers of the requesting function apart from the direct consumers', async () => { + const result = await backend.callTool('api_impact', { route: '/api/secure' }); + + expect(result.indirectConsumers).toEqual([ + { name: 'App', file: 'App.tsx', via: 'fetchSecure' }, + { name: 'Header', file: 'components/Header.tsx', via: 'fetchSecure' }, + ]); + expect(result.impactSummary.indirectConsumers).toBe(2); + // useMulti calls fetchSecure too, but it fetches the route itself. + expect(result.consumers.map((c: any) => c.file)).toEqual( + expect.arrayContaining(['hooks/useMulti.ts', 'services/backend-client.ts']), + ); + }); + + it('does not let indirect consumers change the risk level', async () => { + const result = await backend.callTool('api_impact', { route: '/api/secure' }); + + expect(result.impactSummary.directConsumers).toBe(2); + expect(result.impactSummary.riskLevel).toBe('LOW'); + }); + + it('leaves the field out when no consumer edge names a requesting function', async () => { + const result = await backend.callTool('api_impact', { route: '/api/grants' }); + + expect(result).not.toHaveProperty('indirectConsumers'); + expect(result.impactSummary).not.toHaveProperty('indirectConsumers'); + }); + }); + // ─── Test 2: Mismatch detection ──────────────────────────────────── describe('mismatch detection e2e', () => { diff --git a/gitnexus/test/integration/api-impact-indirect-consumers-e2e.test.ts b/gitnexus/test/integration/api-impact-indirect-consumers-e2e.test.ts new file mode 100644 index 000000000..934b04c34 --- /dev/null +++ b/gitnexus/test/integration/api-impact-indirect-consumers-e2e.test.ts @@ -0,0 +1,172 @@ +/** + * #3644, end to end: a real analyze of a small repository, then `api_impact` + * through LocalBackend. The requesting functions sit in src/client.ts; their + * callers never mention the URL. One request shares its line with a helper + * (`const decode = …; return fetch(…).then(decode);`), which a line-only + * attribution handed to the helper, dropping the callers of fetchFile. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { commitAll, initGitRepo } from '../helpers/temp-git-repo.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const FIXTURE = path.resolve( + __dirname, + '..', + 'fixtures', + 'lang-resolution', + 'api-impact-indirect-consumers', +); + +describe('api_impact indirect consumers through a real analyze (#3644)', () => { + let temp: Awaited>; + let repo: string; + let backend: LocalBackend; + + beforeAll(async () => { + temp = await createTempDir(); + const root = await fs.realpath(temp.dbPath); + const userHome = path.join(root, 'user-home'); + await fs.mkdir(userHome); + vi.stubEnv('HOME', userHome); + vi.stubEnv('USERPROFILE', userHome); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_STORAGE_PATH', undefined); + vi.stubEnv('GITNEXUS_STORAGE_ROOT', undefined); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + vi.stubEnv('GITNEXUS_CONTENT_RETENTION', 'full'); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '1'); + repo = path.join(root, 'repo'); + await fs.cp(FIXTURE, repo, { recursive: true }); + // A top-level request can share a line with a callable without belonging + // to it. Also put a callable on the next line: wrapper sites use 1-based + // lines, which must not make that callable an indirect consumer either. + for (const [kind, request] of [ + ['fetch', "fetch('/api/top-level-fetch')"], + ['wrapper', "apiFetch('/api/top-level-wrapper')"], + ['client', "axios.head('/api/top-level-client')"], + ]) { + await fs.writeFile( + path.join(repo, 'src', `top-level-${kind}.ts`), + [ + "import { apiFetch } from './api';", + "import axios from 'axios';", + `export function decode(res: Response) { return res.json(); } ${request};`, + "export function afterRequest() { return 'local'; }", + ].join('\n'), + ); + await fs.writeFile( + path.join(repo, 'src', `local-${kind}.ts`), + [ + `import { decode, afterRequest } from './top-level-${kind}';`, + 'export function parseLocal(res: Response) { return decode(res); }', + 'export function describeLocal() { return afterRequest(); }', + ].join('\n'), + ); + } + await fs.writeFile( + path.join(repo, 'src', 'lambda_client.py'), + "import requests\ncheck_health = lambda: requests.head('/api/lambda')\n", + ); + for (const [kind, source] of [ + [ + 'declaration', + "export const select = (r: Response) => r.json(), data = fetch('/api/top-level-declaration');", + ], + [ + 'argument', + "export const select = wrap((r: Response) => r.json(), fetch('/api/top-level-argument'));", + ], + ]) { + await fs.writeFile(path.join(repo, 'src', `top-level-${kind}.ts`), source); + await fs.writeFile( + path.join(repo, 'src', `local-${kind}.ts`), + `import { select } from './top-level-${kind}';\nexport function parseLocal(r: Response) { return select(r); }\n`, + ); + } + await fs.writeFile( + path.join(repo, 'src', 'top-level-server.ts'), + [ + "import express from 'express';", + 'const app = express();', + "app.get('/api/lambda', (_req, res) => res.json({ ok: true }));", + ...['fetch', 'wrapper', 'client', 'declaration', 'argument'].map( + (kind) => `app.get('/api/top-level-${kind}', (_req, res) => res.json({ ok: true }));`, + ), + ].join('\n'), + ); + await fs.writeFile(path.join(repo, '.gitignore'), '.gitnexus/\n'); + initGitRepo(repo); + execFileSync('git', ['config', 'commit.gpgsign', 'false'], { cwd: repo }); + commitAll(repo, 'indirect consumers fixture'); + await runFullAnalysis(repo, { skipAgentsMd: true, skipSkills: true }, { onProgress: () => {} }); + backend = new LocalBackend(); + await backend.init(); + }, 180_000); + + afterAll(async () => { + await backend?.dispose(); + vi.unstubAllEnvs(); + await temp?.cleanup(); + }); + + async function impact(route: string) { + const result = await backend.callTool('api_impact', { route, repo }); + expect(result).not.toHaveProperty('error'); + return result; + } + + it('lists the callers of the requesting function, apart from the direct consumer', async () => { + const result = await impact('/api/repos'); + expect(result.consumers.map((c: { file: string }) => c.file)).toEqual(['src/client.ts']); + expect(result.indirectConsumers).toEqual([ + { name: 'loadApp', file: 'src/app.ts', via: 'fetchRepos' }, + { name: 'refreshHeader', file: 'src/header.ts', via: 'fetchRepos' }, + ]); + expect(result.impactSummary.directConsumers).toBe(1); + expect(result.impactSummary.indirectConsumers).toBe(2); + expect(result.impactSummary.riskLevel).toBe('LOW'); + }); + + it('keeps the callers of a request that shares its line with a helper', async () => { + const result = await impact('/api/file'); + expect(result.indirectConsumers).toEqual([ + { name: 'openFile', file: 'src/viewer.ts', via: 'fetchFile' }, + ]); + expect(result.impactSummary.indirectConsumers).toBe(1); + }); + + it('keeps the callers of a request made through a wrapper function', async () => { + const result = await impact('/api/stats'); + expect(result.consumers.map((c: { file: string }) => c.file)).toEqual(['src/stats.ts']); + expect(result.indirectConsumers).toEqual([ + { name: 'renderStatsPanel', file: 'src/stats-panel.ts', via: 'fetchStats' }, + ]); + }); + + it.each(['fetch', 'wrapper', 'client', 'declaration', 'argument'])( + 'does not invent indirect consumers for a module-scope %s request', + async (kind) => { + const result = await impact(`/api/top-level-${kind}`); + expect(result.consumers.map((c: { file: string }) => c.file)).toEqual([ + `src/top-level-${kind}.ts`, + ]); + expect(result).not.toHaveProperty('indirectConsumers'); + expect(result.impactSummary).not.toHaveProperty('indirectConsumers'); + }, + ); + + it('keeps attribution when a callable definition is known but the enclosing-id walk cannot name it', async () => { + const result = await impact('/api/lambda'); + expect(result.consumers.map((c: { file: string }) => c.file)).toEqual(['src/lambda_client.py']); + // An attributed request with no callers reports an empty list; a request + // incorrectly classified as module scope omits the field altogether. + expect(result.indirectConsumers).toEqual([]); + expect(result.impactSummary.indirectConsumers).toBe(0); + }); +}); diff --git a/gitnexus/test/integration/resolvers/fetch-consumer-attribution.test.ts b/gitnexus/test/integration/resolvers/fetch-consumer-attribution.test.ts new file mode 100644 index 000000000..b9c6227e8 --- /dev/null +++ b/gitnexus/test/integration/resolvers/fetch-consumer-attribution.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import path from 'path'; +import { + FIXTURES, + getNodesByLabelFull, + getRelationships, + runPipelineFromRepo, + type PipelineResult, +} from './helpers.js'; + +// The file that sends the request is the only one a FETCHES edge can name. The +// files that call its request function never mention the URL, so `api_impact` +// reaches them through the function the edge's reason records (#3644). +describe('FETCHES consumer attribution through the pipeline (#3644)', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'api-impact-indirect-consumers'), + () => {}, + ); + }, 60000); + + it('names the function holding the fetch site on the file-level edge', () => { + expect( + getNodesByLabelFull(result, 'Function').some( + (node) => node.name === 'fetchRepos' && node.properties.filePath === 'src/client.ts', + ), + ).toBe(true); + + const edges = getRelationships(result, 'FETCHES').filter((e) => e.target === '/api/repos'); + expect(edges.map((e) => [e.sourceFilePath, e.rel.reason])).toEqual([ + ['src/client.ts', 'fetch-url-match|fetches:2|via:Function:src/client.ts:fetchRepos'], + ]); + }); + + it('attributes a request to the function that makes it, not to a helper on the same line', () => { + // `const decode = (res) => res.json(); return fetch(...).then(decode);` on one + // line: by span alone the helper would win, and the callers of fetchFile + // would vanish from the report. + const edges = getRelationships(result, 'FETCHES').filter((e) => e.target === '/api/file'); + expect(edges.map((e) => [e.sourceFilePath, e.rel.reason])).toEqual([ + ['src/client.ts', 'fetch-url-match|fetches:2|via:Function:src/client.ts:fetchFile'], + ]); + }); + + it('attributes a request made through a wrapper to the function that calls the wrapper', () => { + // apiFetch('/api/stats') inside fetchStats, with a helper on the same line: + // the site comes from the wrapper-call capture, not from a bare fetch(). + const edges = getRelationships(result, 'FETCHES').filter((e) => e.target === '/api/stats'); + expect(edges.map((e) => [e.sourceFilePath, e.rel.reason])).toEqual([ + ['src/stats.ts', 'fetch-url-match|fetches:2|via:Function:src/stats.ts:fetchStats'], + ]); + }); + + it('reaches the callers one level up through CALLS edges into that function', () => { + const callers = getRelationships(result, 'CALLS') + .filter((e) => e.target === 'fetchRepos' && e.targetFilePath === 'src/client.ts') + .map((e) => `${e.sourceFilePath}:${e.source}`) + .sort(); + expect(callers).toEqual(['src/app.ts:loadApp', 'src/header.ts:refreshHeader']); + }); +}); diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index a7ee1d34a..ccf00f74f 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -2685,6 +2685,55 @@ describe('LocalBackend.callTool', () => { expect(result.error).toContain('No routes found'); }); + it('api_impact lists the callers of the requesting function as indirect consumers (#3644)', async () => { + const routeRow = (consumerName: string, consumerFile: string, fetchReason: string) => ({ + routeId: 'Route:/api/grants', + routeName: '/api/grants', + handlerFile: 'app/api/grants/route.ts', + responseKeys: ['data'], + errorKeys: [], + middleware: [], + runtimeConfirmed: false, + runtimeSource: null, + runtimeStatus: null, + consumerName, + consumerFile, + fetchReason, + }); + const via = 'Function:src/services/client.ts:fetchGrants'; + const caller = (callerName: string, callerFile: string) => ({ + viaId: via, + viaName: 'fetchGrants', + callerName, + callerFile, + }); + (executeParameterized as any).mockImplementation(async (_repo: string, cypher: string) => { + if (cypher.includes("r.type = 'CALLS'")) { + return [ + caller('Header', 'src/Header.tsx'), + caller('App', 'src/App.tsx'), + // Already a direct consumer file, and the handler file: neither is indirect. + caller('GrantsList', 'src/GrantsList.tsx'), + caller('GET', 'app/api/grants/route.ts'), + ]; + } + return [ + routeRow('client.ts', 'src/services/client.ts', `fetch-url-match|via:${via}`), + routeRow('GrantsList', 'src/GrantsList.tsx', 'fetch-url-match|keys:data'), + ]; + }); + + const result = await backend.callTool('api_impact', { route: '/api/grants' }); + + expect(result.indirectConsumers).toEqual([ + { name: 'App', file: 'src/App.tsx', via: 'fetchGrants' }, + { name: 'Header', file: 'src/Header.tsx', via: 'fetchGrants' }, + ]); + expect(result.impactSummary.directConsumers).toBe(2); + expect(result.impactSummary.indirectConsumers).toBe(2); + expect(result.impactSummary.riskLevel).toBe('LOW'); + }); + it('api_impact detects mismatches and bumps risk level', async () => { (executeParameterized as any).mockResolvedValue([ { diff --git a/gitnexus/test/unit/fetch-consumer-attribution.test.ts b/gitnexus/test/unit/fetch-consumer-attribution.test.ts new file mode 100644 index 000000000..c8197e98a --- /dev/null +++ b/gitnexus/test/unit/fetch-consumer-attribution.test.ts @@ -0,0 +1,188 @@ +/** + * The FETCHES edge is file-level; its reason names the callables whose spans + * contain the request sites, one `|via:` field each, so `api_impact` can + * follow their CALLS edges one level up (#3644). The containment rule is the + * R3-6 sink rule: innermost span wins, lines 0-based on both sides. + */ +import { describe, it, expect } from 'vitest'; +import type { NodeLabel } from 'gitnexus-shared'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import type { KnowledgeGraph } from '../../src/core/graph/types.js'; +import { processNextjsFetchRoutes } from '../../src/core/ingestion/call-processor.js'; +import type { ExtractedFetchCall } from '../../src/core/ingestion/workers/parse-worker.js'; + +const FILE = 'src/client.ts'; +const REPOS = new Map([['/api/repos', ['GET /api/repos']]]); + +function addCallable( + graph: KnowledgeGraph, + id: string, + startLine: number, + endLine: number, + label: NodeLabel = 'Function', + filePath = FILE, +): void { + graph.addNode({ + id, + label, + properties: { name: id.split(':').pop() ?? id, filePath, startLine, endLine }, + }); +} + +const site = ( + lineNumber: number, + filePath = FILE, + fetchURL = '/api/repos', +): ExtractedFetchCall => ({ + filePath, + fetchURL, + lineNumber, +}); + +const reasons = (graph: KnowledgeGraph): string[] => + [...graph.iterRelationshipsByType('FETCHES')].map((r) => r.reason); + +describe('FETCHES consumer attribution (#3644)', () => { + it('names the callable whose span contains the request site', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + + processNextjsFetchRoutes(graph, [site(3)], REPOS); + + expect(reasons(graph)).toEqual(['fetch-url-match|via:Function:src/client.ts:fetchRepos']); + }); + + it('keeps one edge per file and route, naming every requesting callable in code-unit order', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:refresh', 10, 14); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + + // Two sites in one file for one route used to collide on the edge id, with + // the first one winning; now they share the edge. + processNextjsFetchRoutes(graph, [site(12), site(3)], REPOS); + + const edges = [...graph.iterRelationshipsByType('FETCHES')]; + expect(edges).toHaveLength(1); + expect(edges[0].sourceId).toBe('File:src/client.ts'); + expect(edges[0].targetId).toBe('Route:GET /api/repos'); + // `fetches:` counts the file's matching call sites, as before. + expect(edges[0].reason).toBe( + 'fetch-url-match|fetches:2|via:Function:src/client.ts:fetchRepos|via:Function:src/client.ts:refresh', + ); + }); + + it('picks the innermost callable, a method or constructor as well as a function', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:outer', 10, 30); + addCallable(graph, 'Function:src/client.ts:outer.inner', 12, 14); + addCallable(graph, 'Method:src/client.ts:Client.load', 40, 45, 'Method'); + addCallable(graph, 'Constructor:src/client.ts:Client.constructor', 50, 52, 'Constructor'); + addCallable(graph, 'Class:src/client.ts:Client', 38, 60, 'Class'); + + processNextjsFetchRoutes(graph, [site(13), site(42), site(51)], REPOS); + + expect(reasons(graph)).toEqual([ + 'fetch-url-match|fetches:3' + + '|via:Constructor:src/client.ts:Client.constructor' + + '|via:Function:src/client.ts:outer.inner' + + '|via:Method:src/client.ts:Client.load', + ]); + }); + + it('leaves the reason as before when no callable contains the site', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + + // A module-scope call, and a file the graph has no spans for. + processNextjsFetchRoutes(graph, [site(50), site(3, 'src/other.ts')], REPOS); + + expect(reasons(graph)).toEqual(['fetch-url-match', 'fetch-url-match']); + }); + + it('does not attribute a site to a callable of another file at the same lines', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + addCallable(graph, 'Function:src/other.ts:load', 2, 6, 'Function', 'src/other.ts'); + + processNextjsFetchRoutes(graph, [site(3, 'src/other.ts')], REPOS); + + expect(reasons(graph)).toEqual(['fetch-url-match|via:Function:src/other.ts:load']); + }); + + it('appends the attribution after the multi-fetch count', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + addCallable(graph, 'Function:src/client.ts:readFile', 10, 14); + const routes = new Map([ + ['/api/repos', ['GET /api/repos']], + ['/api/file', ['GET /api/file']], + ]); + + processNextjsFetchRoutes(graph, [site(3), site(12, FILE, '/api/file')], routes); + + expect(reasons(graph)).toEqual([ + 'fetch-url-match|fetches:2|via:Function:src/client.ts:fetchRepos', + 'fetch-url-match|fetches:2|via:Function:src/client.ts:readFile', + ]); + }); + + it('appends the attribution after the accessed keys', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + const contents = new Map([[FILE, 'const { repos, total } = await res.json();']]); + + processNextjsFetchRoutes(graph, [site(3)], REPOS, contents); + + expect(reasons(graph)).toEqual([ + 'fetch-url-match|keys:repos,total|via:Function:src/client.ts:fetchRepos', + ]); + }); + + it('prefers the enclosing callable the parser recorded over the innermost span', () => { + const graph = createKnowledgeGraph(); + // `const decode = (res) => res.json(); return fetch('/api/repos').then(decode);` + // on one line: the helper's span is smaller, but the request belongs to fetchRepos. + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 4); + addCallable(graph, 'Function:src/client.ts:fetchRepos.decode', 3, 3); + + processNextjsFetchRoutes( + graph, + [{ ...site(3), enclosingId: 'Function:src/client.ts:fetchRepos' }], + REPOS, + ); + + expect(reasons(graph)).toEqual(['fetch-url-match|via:Function:src/client.ts:fetchRepos']); + }); + + it('falls back to the span when the recorded callable is not in the graph', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:fetchRepos', 2, 6); + + processNextjsFetchRoutes( + graph, + [{ ...site(3), enclosingId: 'Function:src/client.ts:skipped' }], + REPOS, + ); + + expect(reasons(graph)).toEqual(['fetch-url-match|via:Function:src/client.ts:fetchRepos']); + }); + + it('does not use a same-line callable when the parser identified module scope', () => { + const graph = createKnowledgeGraph(); + addCallable(graph, 'Function:src/client.ts:decode', 3, 3); + + processNextjsFetchRoutes(graph, [{ ...site(3), enclosingId: null }], REPOS); + + expect(reasons(graph)).toEqual(['fetch-url-match']); + }); + + it('leaves out an id that contains the field separator rather than splitting the reason', () => { + const graph = createKnowledgeGraph(); + // A type tag from an overload collision can carry a union type. + addCallable(graph, 'Function:src/client.ts:load#1~string|number', 2, 6); + + processNextjsFetchRoutes(graph, [site(3)], REPOS); + + expect(reasons(graph)).toEqual(['fetch-url-match']); + }); +}); diff --git a/gitnexus/test/unit/fetch-reason-parsing.test.ts b/gitnexus/test/unit/fetch-reason-parsing.test.ts index ffb0fc802..1d7c0ec36 100644 --- a/gitnexus/test/unit/fetch-reason-parsing.test.ts +++ b/gitnexus/test/unit/fetch-reason-parsing.test.ts @@ -14,9 +14,11 @@ import { describe, it, expect } from 'vitest'; function parseReasonField(fetchReason: string | null): { accessedKeys?: string[]; fetchCount?: number; + viaSymbolIds?: string[]; } { let accessedKeys: string[] | undefined; let fetchCount: number | undefined; + let viaSymbolIds: string[] | undefined; if (fetchReason) { const keysMatch = fetchReason.match(/\|keys:([^|]+)/); if (keysMatch) { @@ -26,10 +28,13 @@ function parseReasonField(fetchReason: string | null): { if (fetchesMatch) { fetchCount = parseInt(fetchesMatch[1], 10); } + const via = [...fetchReason.matchAll(/\|via:([^|]+)/g)].map((m) => m[1]); + if (via.length > 0) viaSymbolIds = via; } return { ...(accessedKeys ? { accessedKeys } : {}), ...(fetchCount && fetchCount > 1 ? { fetchCount } : {}), + ...(viaSymbolIds ? { viaSymbolIds } : {}), }; } @@ -80,6 +85,25 @@ describe('fetch reason field parsing', () => { expect(result.accessedKeys).not.toContain('pagination|fetches:3'); expect(result.accessedKeys).toEqual(['data', 'pagination']); }); + + // #3644: one field per requesting function; ids carry ':' and '#'. + it('parses every via field, after keys and fetches', () => { + const result = parseReasonField( + 'fetch-url-match|keys:data|fetches:2|via:Function:src/api.ts:getGrants#0|via:Method:src/api.ts:Api.list#1', + ); + expect(result.accessedKeys).toEqual(['data']); + expect(result.fetchCount).toBe(2); + expect(result.viaSymbolIds).toEqual([ + 'Function:src/api.ts:getGrants#0', + 'Method:src/api.ts:Api.list#1', + ]); + }); + + it('leaves viaSymbolIds out of a reason written before the attribution', () => { + const result = parseReasonField('fetch-url-match|keys:data,pagination|fetches:3'); + expect(result.viaSymbolIds).toBeUndefined(); + expect(result.accessedKeys).toEqual(['data', 'pagination']); + }); }); describe('confidence derivation from fetchCount', () => { diff --git a/gitnexus/test/unit/fetch-wrapper-calls.test.ts b/gitnexus/test/unit/fetch-wrapper-calls.test.ts index 77b5d5181..bca065fc1 100644 --- a/gitnexus/test/unit/fetch-wrapper-calls.test.ts +++ b/gitnexus/test/unit/fetch-wrapper-calls.test.ts @@ -179,6 +179,7 @@ for (const [name, grammar, provider] of grammars) { fetchURL: '/api/one', lineNumber: 12, target: { kind: 'unresolved' }, + callPosition: { row: 1, column: 8 }, }, { filePath: 'consumer.ts', @@ -186,6 +187,7 @@ for (const [name, grammar, provider] of grammars) { fetchURL: '/api/two', lineNumber: 13, target: { kind: 'member' }, + callPosition: { row: 2, column: 8 }, }, { filePath: 'consumer.ts', @@ -193,6 +195,7 @@ for (const [name, grammar, provider] of grammars) { fetchURL: '/api/three', lineNumber: 14, target: { kind: 'member' }, + callPosition: { row: 3, column: 8 }, }, { filePath: 'consumer.ts', @@ -200,6 +203,7 @@ for (const [name, grammar, provider] of grammars) { fetchURL: '/api/four', lineNumber: 15, target: { kind: 'unresolved' }, + callPosition: { row: 4, column: 8 }, }, ]); }); diff --git a/gitnexus/test/unit/fetch-wrapper-index.test.ts b/gitnexus/test/unit/fetch-wrapper-index.test.ts index f3589b866..3b839f3e9 100644 --- a/gitnexus/test/unit/fetch-wrapper-index.test.ts +++ b/gitnexus/test/unit/fetch-wrapper-index.test.ts @@ -26,7 +26,14 @@ function definition(id: string, overrides: Partial = {}): Fetch } function call(target: CalleeTarget, functionName = 'request'): FetchWrapperCall { - return { filePath: 'client.ts', functionName, target, fetchURL: '/api/items', lineNumber: 1 }; + return { + filePath: 'client.ts', + functionName, + target, + fetchURL: '/api/items', + lineNumber: 1, + callPosition: { row: 0, column: 0 }, + }; } describe('fetch wrapper target classification', () => { diff --git a/gitnexus/test/unit/process-processor.test.ts b/gitnexus/test/unit/process-processor.test.ts index 4e8c56dad..7a3e39361 100644 --- a/gitnexus/test/unit/process-processor.test.ts +++ b/gitnexus/test/unit/process-processor.test.ts @@ -781,6 +781,40 @@ describe('sink-terminated flows (R3-6)', () => { expect(sinks.has('func:inner')).toBe(true); expect(sinks.has('func:outer')).toBe(false); }); + + it('prefers the enclosing callable the parser recorded over the innermost span', () => { + const graph = createKnowledgeGraph(); + graph.addNode({ + id: 'func:outer', + label: 'Function', + properties: { name: 'outer', filePath: 'src/a.ts', startLine: 1, endLine: 30 }, + }); + graph.addNode({ + id: 'func:helper', + label: 'Function', + properties: { name: 'helper', filePath: 'src/a.ts', startLine: 15, endLine: 15 }, + }); + const sinks = buildSinkFunctionSet(graph, [ + { filePath: 'src/a.ts', lineNumber: 15, enclosingId: 'func:outer' }, + ]); + expect(sinks.has('func:outer')).toBe(true); + expect(sinks.has('func:helper')).toBe(false); + }); + + it('does not make a same-line callable a sink for a known module-scope request', () => { + const graph = createKnowledgeGraph(); + graph.addNode({ + id: 'func:decode', + label: 'Function', + properties: { name: 'decode', filePath: 'src/a.ts', startLine: 3, endLine: 3 }, + }); + + const sinks = buildSinkFunctionSet(graph, [ + { filePath: 'src/a.ts', lineNumber: 3, enclosingId: null }, + ]); + + expect([...sinks]).toEqual([]); + }); }); describe('process selection diversity (R2-3)', () => { diff --git a/gitnexus/test/unit/result-merge.test.ts b/gitnexus/test/unit/result-merge.test.ts index 70472ac36..68521f7b7 100644 --- a/gitnexus/test/unit/result-merge.test.ts +++ b/gitnexus/test/unit/result-merge.test.ts @@ -64,6 +64,7 @@ describe('mergeResult', () => { fetchURL: '/a', lineNumber: 1, target: { kind: 'unresolved' as const }, + callPosition: { row: 0, column: 0 }, }; const second = { filePath: 'b.ts', @@ -71,6 +72,7 @@ describe('mergeResult', () => { fetchURL: '/b', lineNumber: 2, target: { kind: 'member' as const }, + callPosition: { row: 0, column: 0 }, }; mergeResult(target, { ...emptyResult(), fetchWrapperCalls: [first] }); mergeResult(target, emptyResult()); From f759753794fda8e399d670ef7f04b9d7473d6082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sun, 25 Oct 2026 18:30:17 +0000 Subject: [PATCH 2/6] fix(eval): retain large session measurements (#3722) Co-authored-by: Gergo Magyar --- eval/tests/test_workflow_bench_sessions.py | 47 ++++++++++++++++++++++ eval/workflow_bench/runner_sessions.py | 5 ++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index b599f916b..d49530650 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -1298,6 +1298,53 @@ def test_overflowed_parent_capture_is_ineligible_evidence(monkeypatch, tmp_path) assert "exceeds" in rec["error_detail"]["event_stream_error"] +@pytest.mark.parametrize("overflow", [False, True], ids=["complete-12MiB-stream", "bounded-overflow"]) +def test_large_session_stream_preserves_usage_and_redacted_evidence(monkeypatch, tmp_path, overflow): + # Run a real child through the supervisor: a mocked completed capture misses + # the 8 MiB truncation that invalidated release evaluation 40181565589. + if overflow: + monkeypatch.setattr(runner_sessions, "MAX_TRANSCRIPT_BYTES", 64 * 1024) + secret = "large-stream-test-secret" + child = tmp_path / "large_session.py" + child.write_text( + "import json\n" + f"content = 'x' * (768 * 1024) + {secret!r}\n" + "for i in range(16):\n" + " print(json.dumps({'type': 'user', 'message': {'content': [\n" + " {'type': 'tool_result', 'tool_use_id': f'read-{i}', 'content': content}]}}))\n" + f"print({VALID_REPORT!r})\n" + ) + record = runner_sessions.run_claude( + "task", tmp_path, claude_bin="unused", command_prefix=[sys.executable, str(child)], + timeout=30, transcript_output_dir=tmp_path, transcript_secrets=(secret,), + ) + if overflow: + assert record["ok"] is False + assert record["error_kind"] == "session-error" + assert record["transcript_missing"] is True + assert record["cost_usd"] is None + assert "exceeds" in record["error_detail"]["event_stream_error"] + assert not (tmp_path / "transcripts").exists() + return + + assert record["ok"] is True, record["error_detail"] + assert record["transcript_missing"] is False + assert record["cost_usd"] == 0.1 + assert {field: record[field] for field in runner_sessions.USAGE_FIELDS} == json.loads(VALID_REPORT)["usage"] + artifact = record["transcript_artifact"] + payload = (tmp_path / artifact["path"]).read_bytes() + assert artifact["bytes"] == len(payload) > 8 * 1024 * 1024 + assert artifact["sha256"] == hashlib.sha256(payload).hexdigest() + assert secret.encode() not in payload + assert payload.count(b"[REDACTED]") == 16 + assert json.loads(payload.splitlines()[-1]) == json.loads(VALID_REPORT) + # The same limit must admit the persisted evidence for evolution and reuse. + from workflow_bench import comparator_reuse + + assert evolve._transcript_artifact_metadata(artifact)[2] == len(payload) + assert comparator_reuse._transcript_metadata(artifact)[2] == len(payload) + + def test_final_result_event_must_be_last(monkeypatch, tmp_path): stream = VALID_REPORT + "\n" + json.dumps({"type": "assistant", "message": {"content": []}}) + "\n" monkeypatch.setattr(runner_sessions, "run_managed", lambda *a, **k: fake_cli_result(stream)) diff --git a/eval/workflow_bench/runner_sessions.py b/eval/workflow_bench/runner_sessions.py index f45b2dad8..808629ccf 100644 --- a/eval/workflow_bench/runner_sessions.py +++ b/eval/workflow_bench/runner_sessions.py @@ -36,7 +36,10 @@ USAGE_FIELDS = ( "cache_read_input_tokens", "output_tokens", ) -MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024 +# A successful 142-turn release session exceeded 8 MiB (#3500, run +# 40181565589). Keep capture, redacted artifacts, and reuse bounded together, +# with headroom for verbose tool results in the three concurrent paid cells. +MAX_TRANSCRIPT_BYTES = 64 * 1024 * 1024 # Wall-clock ceiling for one headless session, shared by the runner and the # evolution loop so both CLIs kill a session at the same point. 3600s was too # tight for the `workflow` arm: run 29907431284 lost two investigation-task From 34b873696b3766c3237700b5be23823c05d89374 Mon Sep 17 00:00:00 2001 From: rome-xi <49893941+rome-xi@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:05:02 +0800 Subject: [PATCH 3/6] fix(cli): recover pnpm and Bun invocation paths (#3721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(hooks): skip a pnpm shim that cannot run when picking the runner Fixes #3717 * test(hooks): isolate runner liveness checks Inject pnpm liveness and bunx presence in resolver unit tests so they do not depend on installed package managers. Use fixture-only PATHs and POSIX/Windows launchers to cover failed pnpm probes, banner output, fallback execution, argument forwarding, and child exit codes. * fix(cli): prepare native binary before identity Restore skipped LadybugDB install copies before capturing analyzer identity, while keeping native probes and analyzer imports after it. Exercise real npm, pnpm and Bun tarball installs on Linux, macOS and Windows so runner selection is backed by successful indexing. --------- Co-authored-by: Gergo Magyar Co-authored-by: Gergő Magyar --- .github/workflows/ci-tests.yml | 13 +- .../hooks/resolve-analyze-cmd.cjs | 80 ++++--- gitnexus/hooks/claude/resolve-analyze-cmd.cjs | 80 ++++--- gitnexus/scripts/cross-platform-tests.ts | 6 +- gitnexus/scripts/smoke-package-managers.mjs | 111 ++++++++++ gitnexus/src/cli/lazy-action.ts | 8 +- gitnexus/src/core/lbug/native-check.ts | 16 ++ gitnexus/test/unit/build-web-optin.test.ts | 19 +- gitnexus/test/unit/lazy-action.test.ts | 17 +- gitnexus/test/unit/lbug-native-check.test.ts | 24 ++- gitnexus/test/unit/resolve-invocation.test.ts | 197 +++++++++++++++--- 11 files changed, 475 insertions(+), 96 deletions(-) create mode 100644 gitnexus/scripts/smoke-package-managers.mjs diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 48a108fc5..1da1c4662 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -386,11 +386,12 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, ubuntu-24.04-arm] + os: [windows-latest, macos-latest, ubuntu-latest, ubuntu-24.04-arm] runs-on: ${{ matrix.os }} # Windows pack + web install regularly exceeds 15 minutes when setup also # runs prepare/postinstall/build before prepack compiles the same tree again. - timeout-minutes: 20 + # Also exercise fresh npm, pnpm, and Bun installs on the three desktop OSes. + timeout-minutes: 30 steps: # persist-credentials: false — this job runs npm pack + npm install -g # from a tarball and never pushes back; the token in .git/config would @@ -431,6 +432,14 @@ jobs: run: npm pack working-directory: gitnexus + - name: Index the PR tarball with npm, pnpm, and Bun + if: matrix.os != 'ubuntu-24.04-arm' + shell: bash + run: | + npm install --global pnpm@12.10.1 bun@1.3.14 --no-audit --no-fund + node scripts/smoke-package-managers.mjs + working-directory: gitnexus + # Block native compilation only in the consumer phase. A correct bundle # must install using prebuilds even when the runner has a toolchain. - name: Reject ARM64 consumer compilation diff --git a/gitnexus-claude-plugin/hooks/resolve-analyze-cmd.cjs b/gitnexus-claude-plugin/hooks/resolve-analyze-cmd.cjs index c74f03f5d..c20677261 100644 --- a/gitnexus-claude-plugin/hooks/resolve-analyze-cmd.cjs +++ b/gitnexus-claude-plugin/hooks/resolve-analyze-cmd.cjs @@ -16,8 +16,9 @@ * resolved to `npx` and the emitted command could not run at all. `bunx` is * bun's install-free one-shot runner and needs no allow-build equivalent — bun * skips lifecycle scripts unconditionally for a `bunx` fetch, which the native - * loader recovers from directly (see core/lbug/native-check.ts). Both bun rungs - * gate on `bunx` actually running, not merely existing on PATH — see `hasBun`. + * loader recovers from directly (see core/lbug/native-check.ts). Both pnpm and + * bun rungs gate on the tool actually running, not merely existing on PATH — + * see `hasPnpm` (#3717) and `hasBun`. * * The `--allow-build` flags MUST precede the `dlx` token. pnpm < 10.14 keeps * `dlx` in its argv escape list, so flags placed *after* `dlx` are parsed as @@ -126,10 +127,10 @@ function resolveOnPath( // `ran` is liveness, kept separate from the version because a PATH hit proves a // file exists, not that it works, and the two answers differ: a banner-printing // or oddly-versioned tool is alive with `major: null`, while a stale shim left by -// a partial uninstall is neither. Only the bun rung consults `ran` today (see -// hasBun) — it is the one runner with no version to read, so a dedicated probe is -// its only liveness signal; pnpm gets the same evidence for free from the version -// spawn it must make anyway, and deliberately forgives an unreadable one (#1939). +// a partial uninstall (or an unconfigured NVM shim) is neither. Both pnpm and +// bun consult `ran` (see hasPnpm, hasBun) — pnpm gets the evidence for free +// from the version spawn it must make anyway, deliberately forgiving an +// unreadable version when the process actually ran (#1939, #3717). function probeVersion(command) { try { const output = execFileSync(command, ['--version'], { @@ -214,18 +215,34 @@ function resolveInvocationMode(probe = resolveOnPath, deps = {}) { if (probe('gitnexus', true)) return 'gitnexus'; const npmMajor = getNpmMajorVersion(deps); - // pnpm presence: prefer an explicit `pnpmPresent` flag (set by - // formatAnalyzeCommand, which falls back to a PATH probe when the version is - // unreadable) so a present-but-unparseable pnpm — slow probe, Corepack - // banner — still selects pnpm instead of the npx crash path. Otherwise an - // injected version (a successful `pnpm --version` proves presence) - // short-circuits the `which pnpm` probe; failing both, fall back to PATH. - const hasPnpm = - 'pnpmPresent' in deps - ? deps.pnpmPresent - : 'pnpmMajor' in deps - ? deps.pnpmMajor !== null - : Boolean(probe('pnpm')); + // pnpm usability is resolved lazily, mirroring bun: prefer an explicit + // `pnpmPresent` flag (set by formatAnalyzeCommand, which captures whether + // `pnpm --version` actually ran) so a present-but-unparseable pnpm (Corepack + // banner) still selects pnpm instead of the npx crash path, while an + // unusable shim (e.g. NVM for Windows: non-zero exit) is rejected (#3717). + // Otherwise an injected version/presence short-circuits the spawn; failing + // both, check PATH presence then liveness (`pnpm --version`). + let pnpmCache; + const hasPnpm = () => { + if (pnpmCache === undefined) { + if ('pnpmPresent' in deps) { + pnpmCache = + Boolean(deps.pnpmPresent) && ('pnpmRuns' in deps ? Boolean(deps.pnpmRuns) : true); + } else if ('pnpmRuns' in deps && !deps.pnpmRuns) { + pnpmCache = false; + } else if ('pnpmMajor' in deps && deps.pnpmMajor !== null) { + pnpmCache = true; + } else if ('pnpmRuns' in deps && deps.pnpmRuns) { + pnpmCache = Boolean(probe('pnpm')); + } else if ('pnpmMajor' in deps && deps.pnpmMajor === null) { + pnpmCache = false; + } else { + const present = Boolean(probe('pnpm')); + pnpmCache = present && probeVersion('pnpm').ran; + } + } + return pnpmCache; + }; // bun usability is resolved lazily: only the two branches below can select it, // so a machine with pnpm, or with npm < 11, never pays the PATH scan or the @@ -243,13 +260,13 @@ function resolveInvocationMode(probe = resolveOnPath, deps = {}) { }; // npm 11+ npx install crash (#1939) — prefer pnpm dlx when available. - if (hasPnpm && npmMajor !== null && npmMajor >= 11) return 'pnpm'; + if (npmMajor !== null && npmMajor >= 11 && hasPnpm()) return 'pnpm'; // Same crash, no pnpm to fall back on: bunx is install-free and unaffected. if (npmMajor !== null && npmMajor >= 11 && hasBun()) return 'bun'; // npm 10 and earlier: npx works; prefer it over pnpm dlx when npm is present. if (npmMajor !== null && npmMajor < 11) return 'npx'; // npm absent or unreadable — use pnpm if present (with allow-build flags). - if (hasPnpm) return 'pnpm'; + if (hasPnpm()) return 'pnpm'; // Neither npm nor pnpm — bunx is the only install-free runner left. Without // this rung a bun-only machine fell through to `npx`, which is not installed // there, so the emitted command failed with "npx: command not found". @@ -304,12 +321,23 @@ function formatAnalyzeCommand(options = {}, deps = {}) { const mightUsePnpm = forced === 'pnpm' || (forced !== 'gitnexus' && forced !== 'npx' && forced !== 'bun'); if (mightUsePnpm && (forced === 'pnpm' || !probe('gitnexus', true))) { - const { major, minor } = probeVersion('pnpm'); - // Carry presence separately from version: when the version probe fails - // (timeout, Corepack banner) but pnpm is on PATH, still treat it as - // present so mode resolution picks pnpm over the npx crash path. The - // PATH probe is memoized and only runs when the version is unreadable. - const pnpmPresent = major !== null || Boolean(probe('pnpm')); + let ran; + let major = null; + let minor = null; + if ('pnpmRuns' in deps && !deps.pnpmRuns) { + ran = false; + } else { + const probed = probeVersion('pnpm'); + ran = 'pnpmRuns' in deps ? Boolean(deps.pnpmRuns) : probed.ran; + major = probed.major; + minor = probed.minor; + } + // Carry presence separately from version: when the version probe runs + // but produces an unparseable version (e.g. Corepack banner), still treat + // it as present (ran: true, major: null) so mode resolution picks pnpm + // over the npx crash path. A failed probe (ran: false, non-zero exit or + // absent) marks pnpm not present so it falls through to bunx/npx (#3717). + const pnpmPresent = 'pnpmPresent' in deps ? Boolean(deps.pnpmPresent) : ran; resolved = { ...deps, pnpmMajor: major, pnpmMinor: minor, pnpmPresent }; } } diff --git a/gitnexus/hooks/claude/resolve-analyze-cmd.cjs b/gitnexus/hooks/claude/resolve-analyze-cmd.cjs index c74f03f5d..c20677261 100644 --- a/gitnexus/hooks/claude/resolve-analyze-cmd.cjs +++ b/gitnexus/hooks/claude/resolve-analyze-cmd.cjs @@ -16,8 +16,9 @@ * resolved to `npx` and the emitted command could not run at all. `bunx` is * bun's install-free one-shot runner and needs no allow-build equivalent — bun * skips lifecycle scripts unconditionally for a `bunx` fetch, which the native - * loader recovers from directly (see core/lbug/native-check.ts). Both bun rungs - * gate on `bunx` actually running, not merely existing on PATH — see `hasBun`. + * loader recovers from directly (see core/lbug/native-check.ts). Both pnpm and + * bun rungs gate on the tool actually running, not merely existing on PATH — + * see `hasPnpm` (#3717) and `hasBun`. * * The `--allow-build` flags MUST precede the `dlx` token. pnpm < 10.14 keeps * `dlx` in its argv escape list, so flags placed *after* `dlx` are parsed as @@ -126,10 +127,10 @@ function resolveOnPath( // `ran` is liveness, kept separate from the version because a PATH hit proves a // file exists, not that it works, and the two answers differ: a banner-printing // or oddly-versioned tool is alive with `major: null`, while a stale shim left by -// a partial uninstall is neither. Only the bun rung consults `ran` today (see -// hasBun) — it is the one runner with no version to read, so a dedicated probe is -// its only liveness signal; pnpm gets the same evidence for free from the version -// spawn it must make anyway, and deliberately forgives an unreadable one (#1939). +// a partial uninstall (or an unconfigured NVM shim) is neither. Both pnpm and +// bun consult `ran` (see hasPnpm, hasBun) — pnpm gets the evidence for free +// from the version spawn it must make anyway, deliberately forgiving an +// unreadable version when the process actually ran (#1939, #3717). function probeVersion(command) { try { const output = execFileSync(command, ['--version'], { @@ -214,18 +215,34 @@ function resolveInvocationMode(probe = resolveOnPath, deps = {}) { if (probe('gitnexus', true)) return 'gitnexus'; const npmMajor = getNpmMajorVersion(deps); - // pnpm presence: prefer an explicit `pnpmPresent` flag (set by - // formatAnalyzeCommand, which falls back to a PATH probe when the version is - // unreadable) so a present-but-unparseable pnpm — slow probe, Corepack - // banner — still selects pnpm instead of the npx crash path. Otherwise an - // injected version (a successful `pnpm --version` proves presence) - // short-circuits the `which pnpm` probe; failing both, fall back to PATH. - const hasPnpm = - 'pnpmPresent' in deps - ? deps.pnpmPresent - : 'pnpmMajor' in deps - ? deps.pnpmMajor !== null - : Boolean(probe('pnpm')); + // pnpm usability is resolved lazily, mirroring bun: prefer an explicit + // `pnpmPresent` flag (set by formatAnalyzeCommand, which captures whether + // `pnpm --version` actually ran) so a present-but-unparseable pnpm (Corepack + // banner) still selects pnpm instead of the npx crash path, while an + // unusable shim (e.g. NVM for Windows: non-zero exit) is rejected (#3717). + // Otherwise an injected version/presence short-circuits the spawn; failing + // both, check PATH presence then liveness (`pnpm --version`). + let pnpmCache; + const hasPnpm = () => { + if (pnpmCache === undefined) { + if ('pnpmPresent' in deps) { + pnpmCache = + Boolean(deps.pnpmPresent) && ('pnpmRuns' in deps ? Boolean(deps.pnpmRuns) : true); + } else if ('pnpmRuns' in deps && !deps.pnpmRuns) { + pnpmCache = false; + } else if ('pnpmMajor' in deps && deps.pnpmMajor !== null) { + pnpmCache = true; + } else if ('pnpmRuns' in deps && deps.pnpmRuns) { + pnpmCache = Boolean(probe('pnpm')); + } else if ('pnpmMajor' in deps && deps.pnpmMajor === null) { + pnpmCache = false; + } else { + const present = Boolean(probe('pnpm')); + pnpmCache = present && probeVersion('pnpm').ran; + } + } + return pnpmCache; + }; // bun usability is resolved lazily: only the two branches below can select it, // so a machine with pnpm, or with npm < 11, never pays the PATH scan or the @@ -243,13 +260,13 @@ function resolveInvocationMode(probe = resolveOnPath, deps = {}) { }; // npm 11+ npx install crash (#1939) — prefer pnpm dlx when available. - if (hasPnpm && npmMajor !== null && npmMajor >= 11) return 'pnpm'; + if (npmMajor !== null && npmMajor >= 11 && hasPnpm()) return 'pnpm'; // Same crash, no pnpm to fall back on: bunx is install-free and unaffected. if (npmMajor !== null && npmMajor >= 11 && hasBun()) return 'bun'; // npm 10 and earlier: npx works; prefer it over pnpm dlx when npm is present. if (npmMajor !== null && npmMajor < 11) return 'npx'; // npm absent or unreadable — use pnpm if present (with allow-build flags). - if (hasPnpm) return 'pnpm'; + if (hasPnpm()) return 'pnpm'; // Neither npm nor pnpm — bunx is the only install-free runner left. Without // this rung a bun-only machine fell through to `npx`, which is not installed // there, so the emitted command failed with "npx: command not found". @@ -304,12 +321,23 @@ function formatAnalyzeCommand(options = {}, deps = {}) { const mightUsePnpm = forced === 'pnpm' || (forced !== 'gitnexus' && forced !== 'npx' && forced !== 'bun'); if (mightUsePnpm && (forced === 'pnpm' || !probe('gitnexus', true))) { - const { major, minor } = probeVersion('pnpm'); - // Carry presence separately from version: when the version probe fails - // (timeout, Corepack banner) but pnpm is on PATH, still treat it as - // present so mode resolution picks pnpm over the npx crash path. The - // PATH probe is memoized and only runs when the version is unreadable. - const pnpmPresent = major !== null || Boolean(probe('pnpm')); + let ran; + let major = null; + let minor = null; + if ('pnpmRuns' in deps && !deps.pnpmRuns) { + ran = false; + } else { + const probed = probeVersion('pnpm'); + ran = 'pnpmRuns' in deps ? Boolean(deps.pnpmRuns) : probed.ran; + major = probed.major; + minor = probed.minor; + } + // Carry presence separately from version: when the version probe runs + // but produces an unparseable version (e.g. Corepack banner), still treat + // it as present (ran: true, major: null) so mode resolution picks pnpm + // over the npx crash path. A failed probe (ran: false, non-zero exit or + // absent) marks pnpm not present so it falls through to bunx/npx (#3717). + const pnpmPresent = 'pnpmPresent' in deps ? Boolean(deps.pnpmPresent) : ran; resolved = { ...deps, pnpmMajor: major, pnpmMinor: minor, pnpmPresent }; } } diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 612875b51..b9e98a8d4 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -353,9 +353,11 @@ const WORKER_THREADS = [ 'test/integration/resolvers/fetch-wrapper-persistence.test.ts', ]; -// Tree-sitter native addon smoke tests — verify that native grammars -// load correctly on each platform (binary compatibility, .node loading) +// Native addon smoke tests — verify binary compatibility and recovery on each OS. const NATIVE_ADDON_SMOKE = [ + // bunx skips the native install copy; recover it before recording identity. + 'test/unit/lbug-native-check.test.ts', + 'test/unit/lazy-action.test.ts', 'test/integration/tree-sitter-languages.test.ts', 'test/integration/parsing.test.ts', 'test/integration/pipeline.test.ts', diff --git a/gitnexus/scripts/smoke-package-managers.mjs b/gitnexus/scripts/smoke-package-managers.mjs new file mode 100644 index 000000000..d54411f8a --- /dev/null +++ b/gitnexus/scripts/smoke-package-managers.mjs @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + writeFile, +} from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +// Run from gitnexus/ after npm pack. Each manager installs the actual PR tarball +// without lifecycle scripts, reproducing bunx's missing native install copy. +const tarballs = (await readdir('.')).filter((name) => /^gitnexus-.*\.tgz$/.test(name)); +assert.equal(tarballs.length, 1, 'Expected one freshly packed gitnexus tarball'); +const tarball = path.resolve(tarballs[0]); + +function run(program, args, cwd, env) { + // All shell arguments are fixed literals; the tarball is copied through fs. + // Windows package-manager entrypoints are .cmd shims. + const result = spawnSync(program, args, { + cwd, + env, + shell: process.platform === 'win32', + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024, + }); + if (result.error || result.status !== 0) { + throw new Error( + `${program} ${args.join(' ')} failed (${result.status}): ${result.error?.message ?? ''}\n` + + `${result.stdout ?? ''}\n${result.stderr ?? ''}`, + ); + } + return result.stdout; +} + +for (const manager of ['npm', 'pnpm', 'bun']) { + const root = await mkdtemp(path.join(os.tmpdir(), `gitnexus-${manager}-smoke-`)); + const env = { + ...process.env, + GITNEXUS_HOME: path.join(root, 'home'), + GITNEXUS_STORAGE_PATH: path.join(root, 'repo', '.gitnexus'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_MEMORY: 'off', + SCARF_ANALYTICS: 'false', + }; + try { + const version = run(manager, ['--version'], root, env).trim(); + await copyFile(tarball, path.join(root, 'gitnexus.tgz')); + await writeFile( + path.join(root, 'package.json'), + JSON.stringify({ + name: 'gitnexus-manager-smoke', + version: '1.0.0', + private: true, + dependencies: { gitnexus: 'file:./gitnexus.tgz' }, + }), + ); + run( + manager, + ['install', '--ignore-scripts', ...(manager === 'pnpm' ? ['--store-dir=store'] : [])], + root, + env, + ); + + const installed = await realpath(path.join(root, 'node_modules', 'gitnexus')); + const requireInstalled = createRequire(path.join(installed, 'package.json')); + const binaryPath = path.join( + path.dirname(requireInstalled.resolve('@ladybugdb/core')), + 'lbugjs.node', + ); + assert.equal(existsSync(binaryPath), false, `${manager}: fixture must skip native preparation`); + await mkdir(path.join(root, 'repo')); + await writeFile( + path.join(root, 'repo', 'index.ts'), + 'export function greet(name: string) { return `Hello ${name}`; }\n' + + 'export function main() { return greet("GitNexus"); }\n', + ); + run('git', ['init', '-q', 'repo'], root, env); + const args = ['gitnexus', 'analyze', 'repo', '--index-only', '--workers', '1']; + const [program, prefix] = + manager === 'npm' + ? ['npx', ['--no-install']] + : manager === 'pnpm' + ? ['pnpm', ['exec']] + : ['bunx', ['--no-install']]; + run(program, [...prefix, ...args], root, env); + assert.equal(existsSync(binaryPath), true, `${manager}: native binary was not prepared`); + const metadata = JSON.parse( + await readFile(path.join(root, 'repo', '.gitnexus', 'gitnexus.json'), 'utf8'), + ); + assert.equal(metadata.stats.files, 1); + assert.ok(metadata.stats.nodes >= 4, `${manager}: missing indexed symbols`); + assert.ok(metadata.stats.edges >= 5, `${manager}: missing graph relationships`); + assert.ok( + metadata.runnerIdentity?.dependencyRuntime?.artifactCount > 0, + `${manager}: missing identity receipt`, + ); + console.log( + `PASS ${process.platform}/${process.arch}: ${manager} ${version} installed and indexed the PR tarball`, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +} diff --git a/gitnexus/src/cli/lazy-action.ts b/gitnexus/src/cli/lazy-action.ts index c03f9a543..7bb96272b 100644 --- a/gitnexus/src/cli/lazy-action.ts +++ b/gitnexus/src/cli/lazy-action.ts @@ -4,7 +4,7 @@ * at compile time — catching typos when used with concrete module imports. */ -import { checkLbugNative } from '../core/lbug/native-check.js'; +import { checkLbugNative, prepareLbugNativeBinary } from '../core/lbug/native-check.js'; function isCallable(value: unknown): value is (...args: unknown[]) => unknown { return typeof value === 'function'; @@ -46,8 +46,9 @@ export function createLbugLazyAction< /** * Analyze-specific lazy action. Unlike the generic LadybugDB wrapper, this - * captures the complete analyzer receipt before probing/loading native code or - * evaluating the analyzer module graph. The target export receives that start + * prepares any skipped native install copy, then captures the complete analyzer + * receipt before probing/loading native code or evaluating the analyzer module + * graph. The target export receives that start * receipt as its first argument and threads it to runFullAnalysis. */ export function createAnalyzerLbugLazyAction< @@ -62,6 +63,7 @@ export function createAnalyzerLbugLazyAction< analyzerModuleUrl: string, ): (...args: unknown[]) => Promise { return async (...args: unknown[]): Promise => { + prepareLbugNativeBinary(); const identityModule = await identityLoader(); const prepared = await identityModule.captureAnalyzerIdentityBeforeLoad( analyzerModuleUrl, diff --git a/gitnexus/src/core/lbug/native-check.ts b/gitnexus/src/core/lbug/native-check.ts index 33d7e8e5d..0613c3071 100644 --- a/gitnexus/src/core/lbug/native-check.ts +++ b/gitnexus/src/core/lbug/native-check.ts @@ -202,6 +202,22 @@ function unrestorableBinaryFailure( }; } +/** + * Materialize a skipped install-script copy before recording analyzer identity. + * This only copies the already-installed prebuilt; it never loads native code. + * checkLbugNative still owns load probing and diagnostics after identity capture. + */ +export function prepareLbugNativeBinary(overridePkgDir?: string): void { + try { + const pkgDir = + overridePkgDir ?? path.dirname(createRequire(import.meta.url).resolve('@ladybugdb/core')); + const binaryPath = path.join(pkgDir, 'lbugjs.node'); + if (!fs.existsSync(binaryPath)) restorePrebuiltNativeBinary(pkgDir, binaryPath); + } catch { + // The normal check reports missing packages and failed recovery unchanged. + } +} + export function checkLbugNative(overridePkgDir?: string): NativeCheckResult { let pkgDir: string; diff --git a/gitnexus/test/unit/build-web-optin.test.ts b/gitnexus/test/unit/build-web-optin.test.ts index 4d205311e..95356286f 100644 --- a/gitnexus/test/unit/build-web-optin.test.ts +++ b/gitnexus/test/unit/build-web-optin.test.ts @@ -274,18 +274,19 @@ describe('workflows that need the web UI install it themselves', () => { expect(steps.filter(installsWeb)).toHaveLength(0); }); - it('packaged install smoke skips a pre-pack CLI build and keeps a 20-minute budget', () => { + it('packaged install smoke skips a pre-pack CLI build and allows the manager smoke budget', () => { const job = ciJobs['packaged-install-smoke']; const setup = job?.steps?.find((step) => step.uses === './.github/actions/setup-gitnexus'); - expect(job?.['timeout-minutes']).toBe(20); + expect(job?.['timeout-minutes']).toBe(30); expect(setup?.with?.['lifecycle-scripts']).toBe('false'); expect(setup?.with?.build).toBeUndefined(); }); - it('exercises the reported native ARM64 runtime alongside Windows and Linux x64', () => { + it('exercises the reported native ARM64 runtime alongside all three desktop OSes', () => { const job = ciJobs['packaged-install-smoke']; expect(job?.strategy?.matrix?.os).toEqual([ 'windows-latest', + 'macos-latest', 'ubuntu-latest', 'ubuntu-24.04-arm', ]); @@ -300,6 +301,18 @@ describe('workflows that need the web UI install it themselves', () => { expect(String(assertArch?.run)).toContain("process.version, 'v22.23.1'"); }); + it('indexes the freshly packed tarball with every manager on all three desktop OSes', () => { + const steps = ciJobs['packaged-install-smoke']?.steps ?? []; + const packIdx = stepIndex(steps, (step) => step.name === 'Pack gitnexus tarball'); + const smokeIdx = stepIndex( + steps, + (step) => step.name === 'Index the PR tarball with npm, pnpm, and Bun', + ); + expect(smokeIdx).toBeGreaterThan(packIdx); + expect(steps[smokeIdx]?.if).toBe("matrix.os != 'ubuntu-24.04-arm'"); + expect(String(steps[smokeIdx]?.run)).toContain('node scripts/smoke-package-managers.mjs'); + }); + it('keeps consumer compilation guards after packing and checks both npm lockfile paths', () => { const steps = ciJobs['packaged-install-smoke']?.steps ?? []; const packIdx = stepIndex(steps, (step) => step.name === 'Pack gitnexus tarball'); diff --git a/gitnexus/test/unit/lazy-action.test.ts b/gitnexus/test/unit/lazy-action.test.ts index fcc6ce3ae..c84a79b35 100644 --- a/gitnexus/test/unit/lazy-action.test.ts +++ b/gitnexus/test/unit/lazy-action.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; import { createAnalyzerLbugLazyAction, createLazyAction } from '../../src/cli/lazy-action.js'; -const { checkLbugNativeMock } = vi.hoisted(() => ({ +const { checkLbugNativeMock, prepareLbugNativeBinaryMock } = vi.hoisted(() => ({ + prepareLbugNativeBinaryMock: vi.fn(), checkLbugNativeMock: vi.fn( () => ({ ok: true }), ), @@ -9,6 +10,7 @@ const { checkLbugNativeMock } = vi.hoisted(() => ({ vi.mock('../../src/core/lbug/native-check.js', () => ({ checkLbugNative: checkLbugNativeMock, + prepareLbugNativeBinary: prepareLbugNativeBinaryMock, })); describe('createLazyAction', () => { @@ -64,6 +66,11 @@ describe('createLbugLazyAction', () => { describe('createAnalyzerLbugLazyAction', () => { it('captures identity before probing native code or importing the analyzer graph', async () => { const events: string[] = []; + prepareLbugNativeBinaryMock.mockImplementationOnce(() => events.push('binary-prepared')); + checkLbugNativeMock.mockImplementationOnce(() => { + events.push('native-probed'); + return { ok: true }; + }); const receipt = { schemaVersion: 4 }; const run = vi.fn(async () => undefined); const identityLoader = vi.fn(async () => { @@ -89,7 +96,13 @@ describe('createAnalyzerLbugLazyAction', () => { await action('repo', { force: true }); - expect(events).toEqual(['identity-module', 'receipt-captured', 'analyzer-module']); + expect(events).toEqual([ + 'binary-prepared', + 'identity-module', + 'receipt-captured', + 'native-probed', + 'analyzer-module', + ]); expect(run).toHaveBeenCalledWith(receipt, 'repo', { force: true }); }); diff --git a/gitnexus/test/unit/lbug-native-check.test.ts b/gitnexus/test/unit/lbug-native-check.test.ts index 767b44e43..d2b847c8a 100644 --- a/gitnexus/test/unit/lbug-native-check.test.ts +++ b/gitnexus/test/unit/lbug-native-check.test.ts @@ -2,7 +2,11 @@ import { describe, it, expect } from 'vitest'; import os from 'os'; import path from 'path'; import fs from 'fs/promises'; -import { checkLbugNative, glibcTooOldMessage } from '../../src/core/lbug/native-check.js'; +import { + checkLbugNative, + glibcTooOldMessage, + prepareLbugNativeBinary, +} from '../../src/core/lbug/native-check.js'; describe('checkLbugNative', () => { it('returns ok:true when the real @ladybugdb/core binary is present', () => { @@ -72,6 +76,24 @@ describe('checkLbugNative', () => { return { root, pkgDir, subPkgDir }; } + it('prepares a skipped lifecycle copy without loading the binary before identity capture', async () => { + const { root, pkgDir, subPkgDir } = await prebuiltFixture('lbug-prepare-'); + try { + // Deliberately not a native addon: preparation must only copy bytes. + const bytes = Buffer.from('prebuilt fixture'); + await fs.writeFile(path.join(subPkgDir, 'lbugjs.node'), bytes); + expect(() => prepareLbugNativeBinary(pkgDir)).not.toThrow(); + expect(await fs.readFile(path.join(pkgDir, 'lbugjs.node'))).toEqual(bytes); + + // An installed binary must not be replaced on a later preparation pass. + await fs.writeFile(path.join(subPkgDir, 'lbugjs.node'), 'different prebuilt'); + prepareLbugNativeBinary(pkgDir); + expect(await fs.readFile(path.join(pkgDir, 'lbugjs.node'))).toEqual(bytes); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + it('restores a missing lbugjs.node from the prebuilt platform sub-package', async () => { // The "install lifecycle script was skipped" case is recoverable without a // network fetch: the binary is already on disk in the per-platform optional diff --git a/gitnexus/test/unit/resolve-invocation.test.ts b/gitnexus/test/unit/resolve-invocation.test.ts index 0ea6e9309..5d17736a6 100644 --- a/gitnexus/test/unit/resolve-invocation.test.ts +++ b/gitnexus/test/unit/resolve-invocation.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('node:child_process', () => ({ execFileSync: vi.fn(), @@ -39,7 +39,15 @@ const PLUGIN_CJS = path.resolve( interface CjsModule { formatAnalyzeCommand: ( o?: { embeddings?: boolean; indexOnly?: boolean }, - deps?: { npmMajor?: number | null; pnpmMajor?: number | null; pnpmMinor?: number | null }, + deps?: { + npmMajor?: number | null; + pnpmMajor?: number | null; + pnpmMinor?: number | null; + pnpmPresent?: boolean; + pnpmRuns?: boolean; + bunPresent?: boolean; + bunRuns?: boolean; + }, ) => string; formatBunxCommand: (args: string) => string; formatDocumentationDlxCommand: (args: string, o?: { embeddings?: boolean }) => string; @@ -53,6 +61,7 @@ interface CjsModule { npmMajor?: number | null; pnpmMajor?: number | null; pnpmPresent?: boolean; + pnpmRuns?: boolean; bunPresent?: boolean; bunRuns?: boolean; }, @@ -78,15 +87,23 @@ interface CjsModule { // `bunx --version`). resolveOnPath is now spawn-free — a pure PATH scan — so tests // pin it by passing an injected `{ platform, env }` (never the host PATH). Mode // tests inject a fake `probe` or force GITNEXUS_INVOCATION; version tests inject -// `deps`. Any test whose `probe` reports a bunx PATH hit MUST also inject -// `bunRuns`, or the liveness probe spawns the host's real bunx and the result -// depends on whether bun is installed. +// `deps`. Any test whose `probe` reports a pnpm/bunx PATH hit MUST also inject +// `pnpmRuns`/`bunRuns`, or the liveness probe spawns the host's real tool and +// the result depends on whether that tool is installed. // Keep new tests on one of those paths so results never depend on the host. const cjs = cjsRequire(CANONICAL_CJS) as CjsModule; describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { + beforeEach(() => { + vi.stubEnv('PATH', ''); + if (process.platform !== 'win32') { + vi.stubEnv('Path', ''); + vi.stubEnv('path', ''); + } + vi.stubEnv('GITNEXUS_INVOCATION', undefined); + }); afterEach(() => { - delete process.env.GITNEXUS_INVOCATION; + vi.unstubAllEnvs(); }); it('standardizes the invocation ref on gitnexus@latest', () => { @@ -131,7 +148,7 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { it('auto-selects pnpm on npm 11+ when pnpm is on PATH', () => { const probe = (c: string) => (c === 'pnpm' ? '/usr/local/bin/pnpm' : null); - expect(cjs.resolveInvocationMode(probe, { npmMajor: 11 })).toBe('pnpm'); + expect(cjs.resolveInvocationMode(probe, { npmMajor: 11, pnpmRuns: true })).toBe('pnpm'); }); it('auto-selects npx on npm 10 even when pnpm is on PATH', () => { @@ -143,7 +160,7 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { // npmMajor:null means "npm absent" and must be honored via the `in` seam — // not fall through to the host's real npm (npm 10.x on CI → would route npx). const probe = (c: string) => (c === 'pnpm' ? '/usr/local/bin/pnpm' : null); - expect(cjs.resolveInvocationMode(probe, { npmMajor: null })).toBe('pnpm'); + expect(cjs.resolveInvocationMode(probe, { npmMajor: null, pnpmRuns: true })).toBe('pnpm'); }); it('falls back to npx when npm is null-absent and pnpm is also absent', () => { @@ -159,7 +176,7 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { it('honors pnpmPresent:true — a present-but-unparseable pnpm selects pnpm, not the npx crash path', () => { // Windows headline regression guard: when probeVersion cannot read the - // version (timeout / Corepack banner) but pnpm is on PATH, formatAnalyzeCommand + // version after a successful probe (Corepack banner), formatAnalyzeCommand // sets pnpmPresent:true so npm-11 users still get pnpm rather than the npx crash. expect(cjs.resolveInvocationMode(() => null, { npmMajor: 11, pnpmPresent: true })).toBe('pnpm'); }); @@ -238,7 +255,9 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { it('still prefers pnpm over bunx on npm 11+ when both are present', () => { const probe = (c: string) => c === 'pnpm' ? '/usr/local/bin/pnpm' : c === 'bunx' ? '/usr/local/bin/bunx' : null; - expect(cjs.resolveInvocationMode(probe, { npmMajor: 11, bunRuns: true })).toBe('pnpm'); + expect(cjs.resolveInvocationMode(probe, { npmMajor: 11, pnpmRuns: true, bunRuns: true })).toBe( + 'pnpm', + ); }); it('still prefers npx on npm 10 even when bunx is present (no behavior change)', () => { @@ -250,7 +269,9 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { it('still prefers pnpm over bunx when npm is absent and both are present', () => { const probe = (c: string) => c === 'pnpm' ? '/usr/local/bin/pnpm' : c === 'bunx' ? '/usr/local/bin/bunx' : null; - expect(cjs.resolveInvocationMode(probe, { npmMajor: null, bunRuns: true })).toBe('pnpm'); + expect( + cjs.resolveInvocationMode(probe, { npmMajor: null, pnpmRuns: true, bunRuns: true }), + ).toBe('pnpm'); }); it('honors bunPresent:false as explicit absence (overrides a PATH hit)', () => { @@ -269,6 +290,52 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { expect(cjs.resolveInvocationMode(probe, { npmMajor: 11, bunRuns: false })).toBe('npx'); }); + it('rejects a pnpm that is on PATH but does not run (stale shim, #3717)', () => { + // When pnpm is an unusable shim (e.g. NVM for Windows: `pnpm --version` exits + // non-zero), mode resolution must not pick pnpm. On npm 11+ with bunx + // available, it falls through to bunx; without bunx, to npx. + const probeBoth = (c: string) => + c === 'pnpm' ? '/usr/local/bin/pnpm' : c === 'bunx' ? '/usr/local/bin/bunx' : null; + const probePnpmOnly = (c: string) => (c === 'pnpm' ? '/usr/local/bin/pnpm' : null); + + // With working bunx available -> falls through to bun + expect( + cjs.resolveInvocationMode(probeBoth, { npmMajor: 11, pnpmRuns: false, bunRuns: true }), + ).toBe('bun'); + expect( + cjs.resolveInvocationMode(probeBoth, { npmMajor: null, pnpmRuns: false, bunRuns: true }), + ).toBe('bun'); + + // Without bunx available -> falls through to npx + expect( + cjs.resolveInvocationMode(probePnpmOnly, { npmMajor: 11, pnpmRuns: false, bunRuns: false }), + ).toBe('npx'); + expect( + cjs.resolveInvocationMode(probePnpmOnly, { npmMajor: null, pnpmRuns: false, bunRuns: false }), + ).toBe('npx'); + }); + + it('formatAnalyzeCommand falls through to bunx or npx when pnpm does not run (#3717)', () => { + expect( + cjs.formatAnalyzeCommand(undefined, { + npmMajor: 11, + pnpmRuns: false, + bunPresent: true, + bunRuns: true, + }), + ).toBe(`bunx ${cjs.NPX_REF} analyze`); + expect( + cjs.formatAnalyzeCommand(undefined, { npmMajor: 11, pnpmRuns: false, bunPresent: false }), + ).toBe(`npx ${cjs.NPX_REF} analyze`); + }); + + it('keeps pnpm when pnpmRuns:true even if version is unparseable (#3717)', () => { + const probe = (c: string) => (c === 'pnpm' ? '/usr/local/bin/pnpm' : null); + expect( + cjs.resolveInvocationMode(probe, { npmMajor: 11, pnpmMajor: null, pnpmRuns: true }), + ).toBe('pnpm'); + }); + it('never probes bunx when npm or pnpm already decides the mode', () => { // The bun lookup is lazy so machines with a Node toolchain pay no extra // PATH scan — the stale-index hook budget is tight (PROBE_TIMEOUT_MS). @@ -277,7 +344,7 @@ describe('resolve-analyze-cmd.cjs (canonical invocation resolver)', () => { probed.push(c); return c === 'pnpm' ? '/usr/local/bin/pnpm' : null; }; - expect(cjs.resolveInvocationMode(probe, { npmMajor: 11 })).toBe('pnpm'); + expect(cjs.resolveInvocationMode(probe, { npmMajor: 11, pnpmRuns: true })).toBe('pnpm'); expect(probed).not.toContain('bunx'); }); @@ -606,31 +673,99 @@ describe('resolveOnPath — pure-Node PATH scan (#1938, all-OS, spawn-free)', () }); describe('formatAnalyzeCommand end-to-end via the pure scan (#1938)', () => { - // Exercises the public entry through resolveOnPath against a real PATH (no - // mocks, no GITNEXUS_INVOCATION): with a launcher on PATH the hint resolves to - // `gitnexus analyze`. Because resolveOnPath is now spawn-free, this works - // identically on every OS — there is no `where`/`which` reachability caveat. - const savedPath = process.env.PATH; - let binDir: string | undefined; + // Real probes and runner execution, with every tool controlled by the fixture. + // Do not append the host PATH: npm 10 or a global gitnexus would win selection. + const { spawnSync } = cjsRequire('node:child_process') as typeof import('node:child_process'); + const isWin = process.platform === 'win32'; + let binDir: string; + beforeEach(() => { + binDir = mkdtempSync(path.join(os.tmpdir(), 'gn-runner-probe-')); + vi.stubEnv('PATH', binDir); + if (!isWin) { + vi.stubEnv('Path', ''); + vi.stubEnv('path', ''); + } + vi.stubEnv('PATHEXT', '.COM;.EXE;.BAT;.CMD'); + vi.stubEnv('GITNEXUS_INVOCATION', undefined); + }); afterEach(() => { - if (savedPath === undefined) delete process.env.PATH; - else process.env.PATH = savedPath; - if (binDir) rmSync(binDir, { recursive: true, force: true }); - binDir = undefined; - delete process.env.GITNEXUS_INVOCATION; + vi.unstubAllEnvs(); + rmSync(binDir, { recursive: true, force: true }); }); - it('resolves `gitnexus analyze` when a launcher is the only thing on PATH', () => { - binDir = mkdtempSync(path.join(os.tmpdir(), 'gn-e2e-')); - const isWin = process.platform === 'win32'; - const launcher = path.join(binDir, isWin ? 'gitnexus.cmd' : 'gitnexus'); - writeFileSync(launcher, isWin ? '@echo off\r\nexit /b 0\r\n' : '#!/bin/sh\nexit 0\n'); + function writeTool(name: string, version: string, versionExit = 0): void { + const launcher = path.join(binDir, isWin ? `${name}.cmd` : name); + writeFileSync( + launcher, + isWin + ? `@echo off\r\nif "%~1"=="--version" (\r\n echo ${version}\r\n exit /b ${versionExit}\r\n)\r\necho ${name} %*\r\nexit /b 23\r\n` + : `#!/bin/sh\nif [ "$1" = "--version" ]; then\n echo '${version}'\n exit ${versionExit}\nfi\necho "${name} $@"\nexit 23\n`, + ); if (!isWin) chmodSync(launcher, 0o755); - // PATH reduced to just the launcher dir — the former `where`/`which` resolver - // would have ENOENT'd here; the pure scan finds the launcher directly. - process.env.PATH = binDir; + } + + function expectRunner(program: string, prefix = cjs.NPX_REF): void { + const result = spawnSync(process.execPath, [CANONICAL_CJS, 'analyze', '--index-only'], { + env: process.env, + cwd: binDir, + encoding: 'utf-8', + timeout: 15000, + }); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(23); + expect(result.stdout.trim()).toBe(`${program} ${prefix} analyze --index-only`); + } + + it('resolves `gitnexus analyze` when a launcher is the only thing on PATH', () => { + writeTool('gitnexus', '1.6.12'); expect(cjs.formatAnalyzeCommand()).toBe('gitnexus analyze'); }); + + it.each([11, null])( + 'falls through to bunx when pnpm fails and npm major is %s (#3717)', + (npmMajor) => { + if (npmMajor !== null) writeTool('npm', `${npmMajor}.0.0`); + writeTool('pnpm', 'pnpm is not installed for this Node.js version', 1); + writeTool('bunx', '1.2.0'); + expect(cjs.formatAnalyzeCommand()).toBe(`bunx ${cjs.NPX_REF} analyze`); + expect(cjs.resolveInvocationMode()).toBe('bun'); + expectRunner('bunx'); + }, + ); + + it.each([11, null])( + 'falls through to npx when pnpm fails, bunx is absent, and npm major is %s', + (npmMajor) => { + if (npmMajor !== null) writeTool('npm', `${npmMajor}.0.0`); + writeTool('pnpm', 'pnpm is not installed for this Node.js version', 1); + writeTool('npx', '11.0.0'); + expect(cjs.formatAnalyzeCommand()).toBe(`npx ${cjs.NPX_REF} analyze`); + expect(cjs.resolveInvocationMode()).toBe('npx'); + expectRunner('npx'); + }, + ); + + it.each(['10.2.0', 'Notice: pnpm is active'])( + 'keeps pnpm when a successful probe prints %s (#3717)', + (version) => { + writeTool('npm', '11.0.0'); + writeTool('pnpm', version); + const prefix = `--allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter dlx ${cjs.NPX_REF}`; + expect(cjs.formatAnalyzeCommand()).toBe(`pnpm ${prefix} analyze`); + expect(cjs.resolveInvocationMode()).toBe('pnpm'); + expectRunner('pnpm', prefix); + }, + ); + + it('preserves the npx preference on npm 10 with a broken pnpm and working bunx', () => { + writeTool('npm', '10.9.0'); + writeTool('pnpm', 'pnpm is not installed for this Node.js version', 1); + writeTool('bunx', '1.2.0'); + writeTool('npx', '10.9.0'); + expect(cjs.formatAnalyzeCommand()).toBe(`npx ${cjs.NPX_REF} analyze`); + expect(cjs.resolveInvocationMode()).toBe('npx'); + expectRunner('npx'); + }); }); describe('resolve-analyze-cmd.cjs parity', () => { From 63f2f8e590c812519ca41586efbf3921617e08f5 Mon Sep 17 00:00:00 2001 From: Andrey Golanov <307717+agolanov@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:57:55 +0300 Subject: [PATCH 4/6] fix(ingestion): retain named class-expression owners (#3725) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ingestion): retain class-expression owners Keep named JS/TS expressions, members and scope declarations on matching graph identities. Distinguish expressions that reuse a local class name. Avoid root enumeration when no file-scope types are configured. 1088 focused tests pass. Full suite: 28407 passed, one pre-existing SSH-signing-sensitive golden failure, 91 skipped. * fix(ingestion): preserve type-alias owners Keep method-local type-alias member edges on the existing simple TypeAlias IDs when JS/TS class qualification is enabled. Otherwise native COPY drops both method and property relationships. Cover TS, TSX and Vue workers plus direct native-persistence reads. Refs #3725 * fix(bench): align JS/TS capture baselines Record the fingerprints introduced by qualified class and member identities. Same-corpus comparisons attribute every changed byte to the new qualified-name tags; removing those tags restores both previous hashes. Keep capture counts and scaling budgets unchanged. --------- Co-authored-by: Gergő Magyar Co-authored-by: Gergo Magyar --- gitnexus/bench/scope-capture/baselines.json | 6 +- gitnexus/scripts/cross-platform-tests.ts | 1 + .../configs/typescript-javascript.ts | 49 ++ .../ingestion/class-extractors/generic.ts | 13 +- .../configs/typescript-javascript.ts | 1 + .../ingestion/field-extractors/typescript.ts | 7 +- .../languages/javascript/captures.ts | 2 + .../ingestion/languages/javascript/query.ts | 3 + .../core/ingestion/languages/typescript.ts | 9 +- .../languages/typescript/captures.ts | 2 + .../languages/typescript/class-identity.ts | 54 ++ .../ingestion/languages/typescript/query.ts | 4 + .../languages/typescript/simple-hooks.ts | 3 +- gitnexus/src/core/ingestion/languages/vue.ts | 10 +- .../configs/typescript-javascript.ts | 3 +- .../src/core/ingestion/tree-sitter-queries.ts | 8 + .../class-expression-owners.test.ts | 514 ++++++++++++++++++ .../qualified-class-lookups.test.ts | 32 ++ gitnexus/vitest.config.ts | 2 + 19 files changed, 705 insertions(+), 18 deletions(-) create mode 100644 gitnexus/src/core/ingestion/languages/typescript/class-identity.ts create mode 100644 gitnexus/test/integration/class-expression-owners.test.ts diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 529e905d7..e7f6ebf07 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -234,7 +234,8 @@ }, "typescript": { "_lexical_ownership_3532": "#3532: lexical ownership captures and fixture updates; fixture identities are relative to lang-resolution so serialized scope IDs are independent of checkout location. Full capture content remains hashed; scaling_budget is unchanged.", - "fingerprint": "112cb06df38cb8144aaf4cdb5e1209f5d74ff5fa38005299fb39c853df311a68", + "fingerprint": "c32c400ad55d97605af7d48320194f39387199c34bc78cd4208c69277d479fd6", + "_rebaselined_3725_class_expression_owners": "#3725 adds qualified class/member identities. Comparing base 34b873696 and head 7e0414915 over the identical 174 fixture entries attributes the entire drift to 356 added @declaration.qualified_name tags across 85 entries. Removing only those tags restores 112cb06df38cb8144aaf4cdb5e1209f5d74ff5fa38005299fb39c853df311a68 exactly; no capture groups are added or removed (2939 total). Synthetic counts remain 4503/14403 and the scaling budget is unchanged.", "_rebaselined_3489_inline_handlers": "#3489: Compared main 22a7608 and candidate 7456546 emitters on the same 174-entry TypeScript corpus. Exactly four callback declaration groups were added: two in typescript-hoc-wrapped/src/negative-bare-call.ts, one in typescript-hof-callbacks/src/store.ts, and one in typescript-mcp-tools/src/server.ts. Every added group has @callable.unbound; none was removed or changed. Capture groups 2935 -> 2939. JavaScript is byte-identical. Synthetic counts remain 4503/14403 and scaling budgets are unchanged; CI measured TypeScript 1.048 < 1.5 and JavaScript 0.911 < 1.5.", "_rebaselined_3446_mcp_tools_fixture": "#3446 adds typescript-mcp-tools/src/{handlers,server,tools}.ts: corpus growth only, with fixture_count 171 -> 174 and capture_groups_fp 2753 -> 2894. Excluding only that fixture directory restores the prior fingerprint 77c9b4ea654123a64972db8348190ec250b669472b150db65ea8c7f20b355467 exactly. The scope emitter, synthetic capture counts (4503/14403), scaling budget, and other language baselines are unchanged.", "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", @@ -266,7 +267,8 @@ }, "javascript": { "_lexical_ownership_3532": "#3532: lexical ownership captures and fixture updates; fixture identities are relative to lang-resolution so serialized scope IDs are independent of checkout location. Full capture content remains hashed; scaling_budget is unchanged.", - "fingerprint": "857b1257a001f8741412ea08a4f43d9253cc4addfa63421f3013b3ff77a390c9", + "fingerprint": "ae7b2a3dc4072c8e4cda166cb5ea11b50b8a4b9173144518129e19007f628013", + "_rebaselined_3725_class_expression_owners": "#3725 adds qualified class/member identities. Comparing base 34b873696 and head 7e0414915 over the identical 43 fixture entries attributes the entire drift to 107 added @declaration.qualified_name tags across 21 entries. Removing only those tags restores 857b1257a001f8741412ea08a4f43d9253cc4addfa63421f3013b3ff77a390c9 exactly; no capture groups are added or removed (856 total). Synthetic counts remain 3253/10403 and the scaling budget is unchanged.", "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3 -> 479927409bbdd9852a36172c8260aa56df260e99129a7a9c20a0d1903dd5538b; scaling 1.050 < 1.5.", diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index b9e98a8d4..c5c589a5d 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -344,6 +344,7 @@ const WORKER_THREADS = [ 'test/integration/java-imported-return-types.test.ts', 'test/unit/community-processor.test.ts', 'test/integration/worker-pool.test.ts', + 'test/integration/class-expression-owners.test.ts', 'test/integration/parse-impl-quarantine-cache-skip.test.ts', // Real parse workers and both on-disk parse stores must retain lexical // local-import ownership on each platform's native grammar/runtime pair. diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts index b2262432d..02cb33230 100644 --- a/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts +++ b/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts @@ -2,20 +2,34 @@ import { SupportedLanguages } from 'gitnexus-shared'; import type { ClassExtractionConfig } from '../../class-types.js'; +import type { LanguageProvider } from '../../language-provider.js'; +import { createClassExtractor } from '../generic.js'; +import { positionQualifiedCallableName } from '../../workers/callable-id.js'; const shared: Omit = { + // Nested class expressions and their member-owner edges must use the same + // qualified identity, including when an outer class has the same name. + qualifiedNodeId: true, typeDeclarationNodes: [ + 'class', 'class_declaration', 'abstract_class_declaration', 'interface_declaration', 'enum_declaration', ], ancestorScopeNodeTypes: [ + 'class', 'class_declaration', 'abstract_class_declaration', 'interface_declaration', 'enum_declaration', ], + extractScopeSegments(node) { + const name = node.childForFieldName('name'); + return node.type === 'class' && name !== null + ? [positionQualifiedCallableName(name.text, node.startPosition)] + : undefined; + }, }; export const typescriptClassConfig: ClassExtractionConfig = { @@ -32,3 +46,38 @@ export const vueClassConfig: ClassExtractionConfig = { ...shared, language: SupportedLanguages.Vue, }; + +/** Expressions can reuse a local name in sibling factories or methods. */ +export const createTsJsClassExtractor = (config: ClassExtractionConfig) => { + const extractor = createClassExtractor(config); + const extract: typeof extractor.extract = (node, fallback) => { + const owner = extractor.extract(node, fallback); + return owner !== null && node.type === 'class' + ? { + ...owner, + qualifiedName: positionQualifiedCallableName(owner.qualifiedName, node.startPosition), + } + : owner; + }; + return { + ...extractor, + extract, + extractQualifiedName: (node: Parameters[0], name: string) => + extract(node, { name })?.qualifiedName ?? null, + // TypeAlias nodes keep simple IDs, even inside qualified class owners. + qualifyScopeName: (node: Parameters[0], name: string) => + node.type === 'type_alias_declaration' + ? name + : (extractor.qualifyScopeName?.(node, name) ?? name), + }; +}; + +const ownerExtractor = createTsJsClassExtractor(typescriptClassConfig); + +/** Use the definition's qualified name for both the owner and its member IDs. */ +export const resolveTsJsContainerTypeOwner: NonNullable< + LanguageProvider['resolveContainerTypeOwner'] +> = (node) => { + const owner = ownerExtractor.extract(node); + return owner === null ? null : { name: owner.qualifiedName, label: owner.type }; +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/generic.ts b/gitnexus/src/core/ingestion/class-extractors/generic.ts index e20d84829..57b42c670 100644 --- a/gitnexus/src/core/ingestion/class-extractors/generic.ts +++ b/gitnexus/src/core/ingestion/class-extractors/generic.ts @@ -92,17 +92,18 @@ export function createClassExtractor(config: ClassExtractionConfig): ClassExtrac ]); const buildQualifiedName = (node: SyntaxNode, simpleName: string): string => { - let root = node; - while (root.parent) root = root.parent; - const readScopeSegments = (scopeNode: SyntaxNode): string[] => config.extractScopeSegments?.(scopeNode) ?? extractScopeSegmentsFromNode(scopeNode, scopeNameNodeTypes); const fileScopeSegments: string[] = []; - for (const child of root.namedChildren ?? []) { - if (fileScopeSet.has(child.type)) { - fileScopeSegments.push(...readScopeSegments(child)); + if (fileScopeSet.size > 0) { + let root = node; + while (root.parent) root = root.parent; + for (const child of root.namedChildren ?? []) { + if (fileScopeSet.has(child.type)) { + fileScopeSegments.push(...readScopeSegments(child)); + } } } diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/field-extractors/configs/typescript-javascript.ts index 1ae3e40cb..c12d032d3 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/typescript-javascript.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/typescript-javascript.ts @@ -34,6 +34,7 @@ export function constructedTypeOfFieldInitializer(node: SyntaxNode): string | un const shared: Omit = { typeDeclarationNodes: [ + 'class', 'class_declaration', 'abstract_class_declaration', 'interface_declaration', diff --git a/gitnexus/src/core/ingestion/field-extractors/typescript.ts b/gitnexus/src/core/ingestion/field-extractors/typescript.ts index 6a3d7c307..31263fc1e 100644 --- a/gitnexus/src/core/ingestion/field-extractors/typescript.ts +++ b/gitnexus/src/core/ingestion/field-extractors/typescript.ts @@ -30,6 +30,7 @@ export class TypeScriptFieldExtractor extends BaseFieldExtractor { * Node types that represent type declarations with fields in TypeScript */ private static readonly TYPE_DECLARATION_NODES = new Set([ + 'class', 'class_declaration', 'interface_declaration', 'abstract_class_declaration', @@ -258,7 +259,11 @@ export class TypeScriptFieldExtractor extends BaseFieldExtractor { const nestedTypes: string[] = []; // Handle different declaration types - if (node.type === 'class_declaration' || node.type === 'abstract_class_declaration') { + if ( + node.type === 'class' || + node.type === 'class_declaration' || + node.type === 'abstract_class_declaration' + ) { // Find the class body const bodyNode = node.childForFieldName('body'); if (bodyNode) { diff --git a/gitnexus/src/core/ingestion/languages/javascript/captures.ts b/gitnexus/src/core/ingestion/languages/javascript/captures.ts index e2890c50b..9b2723dde 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/captures.ts @@ -32,6 +32,7 @@ import { recordCommonJsModuleKind } from '../typescript/module-kind.js'; import type { Capture, CaptureMatch } from 'gitnexus-shared'; +import { synthesizeTsJsClassIdentity } from '../typescript/class-identity.js'; import { findNodeAtRange, nodeToCapture, @@ -913,6 +914,7 @@ export function emitJsScopeCaptures( grouped[tag] = nodeToCapture(tag, c.node); groupedNodes[tag] = c.node; } + synthesizeTsJsClassIdentity(grouped, groupedNodes); if (Object.keys(grouped).length === 0) continue; // Decompose ESM import_statement / re-export export_statement. diff --git a/gitnexus/src/core/ingestion/languages/javascript/query.ts b/gitnexus/src/core/ingestion/languages/javascript/query.ts index cdb7ccb54..93d2676e2 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/query.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/query.ts @@ -101,6 +101,9 @@ ${unboundCallableQuery('declaration.function')} (class_declaration name: (identifier) @declaration.name) @declaration.class +(class + name: (identifier) @declaration.name) @declaration.class @class.expression + ;; Declarations — methods (inside class bodies) (method_definition name: (property_identifier) @declaration.name) @declaration.method diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 115a9b486..83b945b94 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -13,10 +13,11 @@ import { unboundCallableName } from './typescript/unbound-callables.js'; import type { CaptureMatch, NodeLabel, ParsedFile } from 'gitnexus-shared'; import { defineLanguage } from '../language-provider.js'; import type { AstFrameworkPatternConfig } from '../language-provider.js'; -import { createClassExtractor } from '../class-extractors/generic.js'; import { typescriptClassConfig, javascriptClassConfig, + resolveTsJsContainerTypeOwner, + createTsJsClassExtractor, } from '../class-extractors/configs/typescript-javascript.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; import { @@ -524,7 +525,8 @@ export const typescriptProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(typescriptVariableConfig), definitionPropertiesExtractor: extractConvexEndpointProperties, - classExtractor: createClassExtractor(typescriptClassConfig), + classExtractor: createTsJsClassExtractor(typescriptClassConfig), + resolveContainerTypeOwner: resolveTsJsContainerTypeOwner, // ── JSDoc → description (issue #2270). An exported decl is captured as the // inner declaration; its JSDoc precedes the wrapping `export_statement`. ── descriptionExtractor: createLeadingDocDescriptionExtractor({ @@ -627,7 +629,8 @@ export const javascriptProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(javascriptVariableConfig), definitionPropertiesExtractor: extractConvexEndpointProperties, - classExtractor: createClassExtractor(javascriptClassConfig), + classExtractor: createTsJsClassExtractor(javascriptClassConfig), + resolveContainerTypeOwner: resolveTsJsContainerTypeOwner, // ── JSDoc → description (issue #2270). An exported decl is captured as the // inner declaration; its JSDoc precedes the wrapping `export_statement`. ── descriptionExtractor: createLeadingDocDescriptionExtractor({ diff --git a/gitnexus/src/core/ingestion/languages/typescript/captures.ts b/gitnexus/src/core/ingestion/languages/typescript/captures.ts index 3ba0f2a94..f70468675 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/captures.ts @@ -28,6 +28,7 @@ import { recordCommonJsModuleKind } from './module-kind.js'; import type { Capture, CaptureMatch } from 'gitnexus-shared'; +import { synthesizeTsJsClassIdentity } from './class-identity.js'; import { findNodeAtRange, nodeToCapture, @@ -420,6 +421,7 @@ export function emitTsScopeCaptures( groupedNodes[tag] = c.node; } if (Object.keys(grouped).length === 0) continue; + synthesizeTsJsClassIdentity(grouped, groupedNodes); // Decompose each `import_statement` / re-export `export_statement` // so `interpretTsImport` sees the kind/source/name/alias markers diff --git a/gitnexus/src/core/ingestion/languages/typescript/class-identity.ts b/gitnexus/src/core/ingestion/languages/typescript/class-identity.ts new file mode 100644 index 000000000..e029017c1 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/typescript/class-identity.ts @@ -0,0 +1,54 @@ +import type { Capture } from 'gitnexus-shared'; +import { resolveTsJsContainerTypeOwner } from '../../class-extractors/configs/typescript-javascript.js'; +import { syntheticCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; + +/** Keep named-expression scope declarations on the graph's owner/member IDs. */ +export function synthesizeTsJsClassIdentity( + captures: Record, + nodes: Record, +): void { + const type = + nodes['@declaration.class'] ?? nodes['@declaration.interface'] ?? nodes['@declaration.enum']; + if (type !== undefined) { + const owner = resolveTsJsContainerTypeOwner(type, ''); + if (owner !== null) { + captures['@declaration.qualified_name'] = syntheticCapture( + '@declaration.qualified_name', + type, + owner.name, + ); + } + return; + } + const member = + nodes['@declaration.method'] ?? + nodes['@declaration.constructor'] ?? + nodes['@declaration.property']; + const name = nodes['@declaration.name']; + if (member === undefined || name === undefined) return; + for (let parent = member.parent; parent !== null; parent = parent.parent) { + // Object members and executable-body locals belong to their own scope. + if ( + parent.type === 'object' || + parent.type === 'object_type' || + parent.type === 'statement_block' + ) + return; + if ( + parent.type === 'class' || + parent.type === 'class_declaration' || + parent.type === 'abstract_class_declaration' || + parent.type === 'interface_declaration' + ) { + const owner = resolveTsJsContainerTypeOwner(parent, ''); + if (owner !== null) { + captures['@declaration.qualified_name'] = syntheticCapture( + '@declaration.qualified_name', + member, + `${owner.name}.${name.text}`, + ); + } + return; + } + } +} diff --git a/gitnexus/src/core/ingestion/languages/typescript/query.ts b/gitnexus/src/core/ingestion/languages/typescript/query.ts index 597aeaefe..6c71985ff 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/query.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/query.ts @@ -161,6 +161,10 @@ ${unboundCallableQuery('declaration.function')} name: (type_identifier) @declaration.name type_parameters: (type_parameters)? @declaration.type-parameters) @declaration.class +(class + name: (type_identifier) @declaration.name + type_parameters: (type_parameters)? @declaration.type-parameters) @declaration.class @class.expression + (abstract_class_declaration name: (type_identifier) @declaration.name type_parameters: (type_parameters)? @declaration.type-parameters) @declaration.class diff --git a/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts b/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts index 453a32533..bfbfba7e0 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts @@ -44,7 +44,8 @@ export function tsBindingScopeFor( ): ScopeId | null { // A named function expression binds only inside itself. Anonymous callables // explicitly suppress their binding through @declaration.binding-name. - if (decl['@callable.unbound'] !== undefined) return innermost.id; + if (decl['@callable.unbound'] !== undefined || decl['@class.expression'] !== undefined) + return innermost.id; // Method return type: hoist to Module (mirrors csharpBindingScopeFor). if (decl['@type-binding.return'] !== undefined) { return walkToScope(innermost, tree, 'Module'); diff --git a/gitnexus/src/core/ingestion/languages/vue.ts b/gitnexus/src/core/ingestion/languages/vue.ts index 0c421724d..ac602ef90 100644 --- a/gitnexus/src/core/ingestion/languages/vue.ts +++ b/gitnexus/src/core/ingestion/languages/vue.ts @@ -13,8 +13,11 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { isBuiltin } from 'node:module'; -import { createClassExtractor } from '../class-extractors/generic.js'; -import { vueClassConfig } from '../class-extractors/configs/typescript-javascript.js'; +import { + vueClassConfig, + resolveTsJsContainerTypeOwner, + createTsJsClassExtractor, +} from '../class-extractors/configs/typescript-javascript.js'; import { defineLanguage } from '../language-provider.js'; import { typeConfig as typescriptConfig } from '../type-extractors/typescript.js'; import { tsExportChecker } from '../export-detection.js'; @@ -77,7 +80,7 @@ const VUE_SPECIFIC_BUILT_INS = [ const VUE_BUILT_INS: ReadonlySet = new Set([...TS_BUILT_INS, ...VUE_SPECIFIC_BUILT_INS]); -const vueClassExtractor = createClassExtractor(vueClassConfig); +const vueClassExtractor = createTsJsClassExtractor(vueClassConfig); export const vueProvider = defineLanguage({ id: SupportedLanguages.Vue, @@ -92,6 +95,7 @@ export const vueProvider = defineLanguage({ fieldExtractor: typescriptFieldExtractor, variableExtractor: createVariableExtractor(typescriptVariableConfig), classExtractor: vueClassExtractor, + resolveContainerTypeOwner: resolveTsJsContainerTypeOwner, builtInNames: VUE_BUILT_INS, // A `` : source, + }, + ]); + for (const [name, property, method] of [ + ['Shape', 'p', 'f'], + ['Local', 'value', 'run'], + ]) { + const owner = `TypeAlias:${file}:${name}`; + expect(graph.getNode(owner)).toBeDefined(); + const members = graph.relationships.filter( + (edge) => + edge.sourceId === owner && (edge.type === 'HAS_PROPERTY' || edge.type === 'HAS_METHOD'), + ); + expect(members.map((edge) => edge.targetId).sort()).toEqual([ + `Method:${file}:${name}.${method}${methodSuffix}`, + `Property:${file}:${name}.${property}`, + ]); + for (const edge of members) expect(graph.getNode(edge.targetId)).toBeDefined(); + } + expect( + graph.relationships.filter( + (edge) => + (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY') && + (!graph.getNode(edge.sourceId) || !graph.getNode(edge.targetId)), + ), + ).toEqual([]); + }, + ); + + it('persists method-local type alias member relationships through native COPY', async () => { + const temp = await createTempDir(); + try { + await fs.writeFile( + path.join(temp.dbPath, 'alias.ts'), + 'class Named { m() { type Shape = { p: number; f(): void }; } }', + ); + const result = await runPipelineFromRepo(temp.dbPath, () => {}, { workerPoolSize: 1 }); + expect(result.usedWorkerPool).toBe(true); + await initLbug(path.join(temp.dbPath, 'lbug'), { skipFts: true }); + await loadGraphToLbug(result.graph, temp.dbPath, path.join(temp.dbPath, 'storage')); + const aliases = await executeQuery('MATCH (a:TypeAlias) RETURN a.id AS id'); + expect(aliases.map((row) => row.id)).toEqual(['TypeAlias:alias.ts:Shape']); + const members = await executeQuery( + "MATCH (a:TypeAlias)-[r:CodeRelation]->(m) WHERE r.type IN ['HAS_METHOD', 'HAS_PROPERTY'] RETURN a.id AS owner, m.id AS member, r.type AS type ORDER BY type", + ); + expect(members).toEqual([ + { + owner: 'TypeAlias:alias.ts:Shape', + member: 'Method:alias.ts:Shape.f#0', + type: 'HAS_METHOD', + }, + { + owner: 'TypeAlias:alias.ts:Shape', + member: 'Property:alias.ts:Shape.p', + type: 'HAS_PROPERTY', + }, + ]); + } finally { + await closeLbug(); + await temp.cleanup(); + } + }); + + it.each(['ts', 'tsx', 'js', 'mjs'])( + 'resolves a declaration shadowing an outer class inside an expression in .%s', + async (extension) => { + const temp = await createTempDir(); + try { + await fs.writeFile( + path.join(temp.dbPath, `shadow.${extension}`), + 'class Local {} const A = class Outer { m() { class Local {} return new Local(); } };', + ); + const result = await runPipelineFromRepo(temp.dbPath, () => {}, { workerPoolSize: 1 }); + const locals = result.graph.nodes.filter( + (node) => node.label === 'Class' && node.properties.name === 'Local', + ); + expect(locals).toHaveLength(2); + const inner = locals.find((node) => node.id.includes('.Local'))!; + expect(inner).toBeDefined(); + const method = result.graph.nodes.find( + (node) => node.label === 'Method' && node.properties.name === 'm', + )!; + const calls = result.graph.relationships.filter( + (edge) => edge.type === 'CALLS' && edge.sourceId === method.id, + ); + expect(calls.map((edge) => edge.targetId)).toEqual([inner.id]); + } finally { + await temp.cleanup(); + } + }, + ); + it.each(['jsx', 'vue'])('preserves owners for the shared .%s provider', async (extension) => { + const source = 'const X = class Named { status = 500; run() { return this.status; } };'; + const { graph } = await parseFilesWithWorkers([ + { + path: `shared.${extension}`, + content: extension === 'vue' ? `` : source, + }, + ]); + const owner = graph.nodes.find( + (node) => node.label === 'Class' && node.properties.name === 'Named', + ); + expect(owner).toBeDefined(); + const edges = graph.relationships.filter( + (edge) => + edge.sourceId === owner!.id && (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY'), + ); + expect(edges.map((edge) => graph.getNode(edge.targetId)?.properties.name).sort()).toEqual([ + 'run', + 'status', + ]); + }); + + it.each(['ts', 'js'])( + 'keeps declaration and anonymous-expression controls in .%s', + async (extension) => { + const { graph } = await parseFilesWithWorkers([ + { + path: `control.${extension}`, + content: + 'class Control { status = 1; run() {} } const Anonymous = class { value = 2; method() {} };', + }, + ]); + expect( + graph.nodes.filter((node) => node.label === 'Class').map((node) => node.properties.name), + ).toEqual(['Control']); + const owner = graph.getNode(`Class:control.${extension}:Control`); + expect(owner).toBeDefined(); + const edges = graph.relationships.filter((edge) => edge.sourceId === owner!.id); + expect(edges.map((edge) => graph.getNode(edge.targetId)?.properties.name).sort()).toEqual([ + 'run', + 'status', + ]); + }, + ); + it.each(['ts', 'tsx', 'js', 'mjs'])( + 'retains the class and its members in .%s', + async (extension) => { + const { graph } = await parseFilesWithWorkers([ + { + path: `mock.${extension}`, + content: `vi.mock('api', () => ({ + ApiError: class ApiError extends Error { + status = 500; + messageForUser() { return 'Failed'; } + }, +}));`, + }, + ]); + const owner = graph.nodes.find( + (node) => node.label === 'Class' && node.properties.name === 'ApiError', + ); + expect(owner).toBeDefined(); + expect(owner!.properties).toMatchObject({ + filePath: `mock.${extension}`, + startLine: 1, + endLine: 4, + }); + const edges = graph.relationships.filter( + (edge) => edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY', + ); + expect(edges).toHaveLength(2); + expect(edges.map((edge) => graph.getNode(edge.targetId)?.properties.name).sort()).toEqual([ + 'messageForUser', + 'status', + ]); + expect( + graph.nodes.find((node) => node.properties.name === 'messageForUser')!.properties + .parameterCount, + ).toBe(0); + expect( + graph.nodes.find((node) => node.properties.name === 'status')!.properties.isStatic, + ).toBe(false); + for (const edge of edges) { + expect(edge.sourceId).toBe(owner!.id); + expect(graph.getNode(edge.sourceId)).toBeDefined(); + expect(graph.getNode(edge.targetId)).toBeDefined(); + } + }, + ); + + it('retains a TypeScript constructor parameter property on its expression owner', async () => { + const { graph } = await parseFilesWithWorkers([ + { + path: 'parameter.ts', + content: `vi.mock('api', () => ({ + ApiError: class ApiError extends Error { + constructor(public readonly status: number) { super('HTTP ' + status); } + }, +}));`, + }, + ]); + const owner = graph.nodes.find( + (node) => node.label === 'Class' && node.properties.name === 'ApiError', + ); + expect(owner).toBeDefined(); + const property = graph.nodes.find( + (node) => node.label === 'Property' && node.properties.name === 'status', + ); + expect(property).toBeDefined(); + expect(property!.id).toBe('Property:parameter.ts:ApiError@1:12.status'); + expect(graph.relationships).toContainEqual( + expect.objectContaining({ + type: 'HAS_PROPERTY', + sourceId: owner!.id, + targetId: property!.id, + }), + ); + for (const edge of graph.relationships.filter( + (edge) => edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY', + )) { + expect(graph.getNode(edge.sourceId)).toBeDefined(); + expect(graph.getNode(edge.targetId)).toBeDefined(); + } + }); + + it.each(['ts', 'tsx', 'js', 'mjs'])( + 'keeps nested expression members distinct from declarations in .%s', + async (extension) => { + const { graph } = await parseFilesWithWorkers([ + { + path: `nested.${extension}`, + content: `class ApiError { status = 200; messageForUser() { return 'outer'; } } +class Holder { + nested = class ApiError { status = 500; messageForUser() { return 'inner'; } }; +}`, + }, + ]); + const owners = graph.nodes.filter( + (node) => node.label === 'Class' && node.properties.name === 'ApiError', + ); + expect(owners).toHaveLength(2); + expect(new Set(owners.map((node) => node.id)).size).toBe(2); + const memberIds = new Set(); + for (const owner of owners) { + const edges = graph.relationships.filter( + (edge) => + edge.sourceId === owner.id && + (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY'), + ); + expect(edges.map((edge) => graph.getNode(edge.targetId)?.properties.name).sort()).toEqual([ + 'messageForUser', + 'status', + ]); + edges.forEach((edge) => memberIds.add(edge.targetId)); + } + expect(memberIds.size).toBe(4); + for (const edge of graph.relationships.filter( + (edge) => edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY', + )) { + expect(graph.getNode(edge.sourceId)).toBeDefined(); + expect(graph.getNode(edge.targetId)).toBeDefined(); + } + }, + ); + + it.each(['ts', 'tsx', 'js', 'mjs'])( + 'distinguishes same-named expressions in sibling methods in .%s', + async (extension) => { + const { graph } = await parseFilesWithWorkers([ + { + path: `siblings.${extension}`, + content: `class Holder { + first() { return class Local { status = 1; run() {} }; } + second() { return class Local { status = 2; run() {} }; } +}`, + }, + ]); + const owners = graph.nodes.filter( + (node) => node.label === 'Class' && node.properties.name === 'Local', + ); + expect(owners).toHaveLength(2); + const targets = new Set(); + for (const owner of owners) { + const edges = graph.relationships.filter( + (edge) => + edge.sourceId === owner.id && + (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY'), + ); + expect(edges.map((edge) => graph.getNode(edge.targetId)?.properties.name).sort()).toEqual([ + 'run', + 'status', + ]); + for (const edge of edges) { + targets.add(edge.targetId); + const member = graph.getNode(edge.targetId)!; + expect(member.properties.startLine).toBe(owner.properties.startLine); + } + } + expect(targets.size).toBe(4); + }, + ); + + it.each(['ts', 'tsx', 'js', 'mjs'])( + 'resolves expression members and keeps self names internal in .%s', + async (extension) => { + const temp = await createTempDir(); + try { + await fs.writeFile( + path.join(temp.dbPath, `self.${extension}`), + `class Holder { child = class Child { p=1; m(){return this.p;} }; } +const X = class Named { static s() {} m() { Named.s(); } }; +Named.s();`, + ); + const result = await runPipelineFromRepo(temp.dbPath, () => {}, { workerPoolSize: 1 }); + const member = (ownerName: string, memberName: string) => { + const owner = result.graph.nodes.find( + (node) => node.label === 'Class' && node.properties.name === ownerName, + )!; + expect(owner).toBeDefined(); + const edge = result.graph.relationships.find( + (edge) => + edge.sourceId === owner.id && + (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY') && + result.graph.getNode(edge.targetId)?.properties.name === memberName, + ); + expect(edge).toBeDefined(); + return edge!.targetId; + }; + expect(result.graph.relationships).toContainEqual( + expect.objectContaining({ + type: 'ACCESSES', + sourceId: member('Child', 'm'), + targetId: member('Child', 'p'), + }), + ); + const staticId = member('Named', 's'); + const calls = result.graph.relationships.filter( + (edge) => edge.type === 'CALLS' && edge.targetId === staticId, + ); + expect(calls).toHaveLength(1); + expect(calls[0]!.sourceId).toBe(member('Named', 'm')); + expect( + findDanglingEdges(result, ['HAS_METHOD', 'HAS_PROPERTY', 'CALLS', 'ACCESSES']), + ).toEqual([]); + } finally { + await temp.cleanup(); + } + }, + ); + + it('preserves owners through the full pipeline, both durable caches, and native COPY', async () => { + const temp = await createTempDir(); + const repo = path.join(temp.dbPath, 'repo'); + const storage = path.join(temp.dbPath, 'storage'); + const identity = await runtimeParseIdentity(); + const extensions = ['ts', 'tsx', 'js', 'mjs']; + const source = `class ApiError { status = 200; messageForUser() { return 'outer'; } } +vi.mock('api', () => ({ + ApiError: class MockError extends Error { status = 500; messageForUser() { return 'mock'; } }, +})); +class Holder { + nested = class ApiError { status = 501; messageForUser() { return 'inner'; } }; +}`; + try { + await fs.mkdir(repo); + for (const extension of extensions) + await fs.writeFile(path.join(repo, `mock.${extension}`), source); + const coldCache = await loadParseCache(storage, identity, storage); + const cold = await runPipelineFromRepo(repo, () => {}, { + workerPoolSize: 1, + parseCache: coldCache, + }); + expect(cold.usedWorkerPool).toBe(true); + expect(cold.reparsedFileCount).toBe(4); + expect(cold.parseCacheHitFileCount).toBe(0); + expect(cold.scopeExtractionFailures).toEqual([]); + + const assertOwners = (result: typeof cold) => { + expect(findDanglingEdges(result, ['HAS_METHOD', 'HAS_PROPERTY'])).toEqual([]); + for (const extension of extensions) { + for (const name of ['ApiError', 'MockError@2:12', 'Holder.ApiError@5:11']) { + const ownerId = `Class:mock.${extension}:${name}`; + expect(result.graph.getNode(ownerId)).toBeDefined(); + const edges = result.graph.relationships.filter( + (edge) => + edge.sourceId === ownerId && + (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY'), + ); + expect( + edges.map((edge) => result.graph.getNode(edge.targetId)?.properties.name).sort(), + ).toEqual(['messageForUser', 'status']); + expect(edges.map((edge) => edge.targetId).sort()).toEqual([ + `Method:mock.${extension}:${name}.messageForUser#0`, + `Property:mock.${extension}:${name}.status`, + ]); + } + } + }; + assertOwners(cold); + pruneCache(coldCache, coldCache.usedKeys); + const keys = await saveParseCache(storage, coldCache); + await pruneAndSaveDurableParsedFileStore( + getDurableParsedFileDir(storage), + identity.base, + new Set(keys), + ); + await clearParsedFileStore(storage); + const durable = await loadDurableParsedFileIndex( + getDurableParsedFileDir(storage), + identity.base, + ); + for (const [key, wanted] of durable) + expect(await durableChunkHasShards(storage, key, wanted)).toBe(true); + const paths = new Set([...durable.values()].flatMap((paths) => [...paths])); + expect([...paths].sort()).toEqual(extensions.map((extension) => `mock.${extension}`).sort()); + const parsed = await loadParsedFilesForPaths(storage, paths); + expect(parsed.size).toBe(4); + for (const file of parsed.values()) { + expect(file.scopes.filter((scope) => scope.kind === 'Class')).toHaveLength(4); + expect( + file.localDefs + .filter((definition) => definition.type === 'Property') + .map((definition) => definition.qualifiedName) + .sort(), + ).toEqual([ + 'ApiError.status', + 'Holder.ApiError@5:11.status', + 'Holder.nested', + 'MockError@2:12.status', + ]); + } + + const warmCache = await loadParseCache(storage, identity, storage); + const chunks = await Promise.all( + [...warmCache.onDiskKeys!].map((key) => loadParseCacheChunk(warmCache, key)), + ); + expect(chunks.length).toBeGreaterThan(0); + // ParsedFiles must come from their durable shards, not worker-result chunks. + expect( + chunks.flatMap((chunk) => chunk ?? []).flatMap((part) => part.parsedFiles ?? []), + ).toEqual([]); + const warm = await runPipelineFromRepo(repo, () => {}, { + workerPoolSize: 1, + parseCache: warmCache, + }); + expect(warm.usedWorkerPool).toBe(false); + expect(warm.reparsedFileCount).toBe(0); + expect(warm.parseCacheHitFileCount).toBe(4); + expect(warm.scopeExtractionFailures).toEqual([]); + assertOwners(warm); + const members = (result: typeof cold) => + result.graph.relationships + .filter((edge) => edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY') + .map((edge) => `${edge.type}:${edge.sourceId}->${edge.targetId}`) + .sort(); + expect(members(warm)).toEqual(members(cold)); + + await initLbug(path.join(temp.dbPath, 'lbug'), { skipFts: true }); + const loaded = await loadGraphToLbug(warm.graph, repo, storage); + expect(loaded.warnings).toEqual([]); + expect(loaded.skippedRels).toBe(0); + const rows = await executeQuery( + "MATCH (c:Class)-[r:CodeRelation]->(m) WHERE r.type IN ['HAS_METHOD', 'HAS_PROPERTY'] RETURN c.id AS owner, m.id AS member, r.type AS type", + ); + expect(rows.map((row) => `${row.type}:${row.owner}->${row.member}`).sort()).toEqual( + members(warm), + ); + const classes = await executeQuery('MATCH (c:Class) RETURN c.id AS id'); + expect(classes).toHaveLength(16); + } finally { + await closeLbug(); + await clearParsedFileStore(storage); + await temp.cleanup(); + } + }, 120000); +}); diff --git a/gitnexus/test/integration/qualified-class-lookups.test.ts b/gitnexus/test/integration/qualified-class-lookups.test.ts index 53563fefb..7dbfecd9b 100644 --- a/gitnexus/test/integration/qualified-class-lookups.test.ts +++ b/gitnexus/test/integration/qualified-class-lookups.test.ts @@ -2,6 +2,38 @@ import { describe, expect, it } from 'vitest'; import { parseFilesWithWorkers } from '../helpers/worker-parse.js'; describe('qualified class lookups', () => { + it('preserves nested names with and without file-level package scopes', async () => { + const { graph, model } = await parseFilesWithWorkers([ + { + path: 'nested.ts', + content: + 'class Outer { make() { class Inner { value = 1; run() { return this.value; } } return Inner; } }', + }, + { + path: 'Nested.java', + content: 'package example.models; class Outer { class Inner {} }', + }, + { + path: 'nested.rb', + content: 'module Root\n class Outer\n class Inner\n end\n end\nend\n', + }, + ]); + for (const name of ['Outer.Inner', 'example.models.Outer.Inner', 'Root.Outer.Inner']) { + expect(model.types.lookupClassByQualifiedName(name)).toHaveLength(1); + } + const ownerId = 'Class:nested.ts:Outer.Inner'; + expect(graph.getNode(ownerId)).toBeDefined(); + const members = graph.relationships.filter( + (edge) => + edge.sourceId === ownerId && (edge.type === 'HAS_METHOD' || edge.type === 'HAS_PROPERTY'), + ); + expect(members.map((edge) => graph.getNode(edge.targetId)?.properties.name).sort()).toEqual([ + 'run', + 'value', + ]); + for (const edge of members) expect(graph.getNode(edge.targetId)).toBeDefined(); + }); + it('derives canonical dot-separated names from namespaces, packages, and modules', async () => { const { model } = await parseFilesWithWorkers([ { diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 1f8179356..a9c3fa2ed 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -74,6 +74,7 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + 'test/integration/class-expression-owners.test.ts', 'test/integration/rename-provider-*.test.ts', 'test/integration/pdg-explicit-owner.test.ts', 'test/integration/skip-fts.test.ts', @@ -182,6 +183,7 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + 'test/integration/class-expression-owners.test.ts', 'test/integration/rename-provider-*.test.ts', 'test/integration/pdg-explicit-owner.test.ts', 'test/integration/skip-fts.test.ts', From 6122657afce0f0089990dd472b9ca0af2fa195fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Mon, 26 Oct 2026 08:53:47 +0000 Subject: [PATCH 5/6] fix(mcp): bound query previews and advertise repo defaults (#3728) * fix(mcp): reduce query payload and retry overhead * fix(mcp): fetch bounded previews at query time --------- Co-authored-by: Gergo Magyar --- eval/workflow_bench/README.md | 29 ++++- gitnexus/scripts/bench/mcp-serialization.ts | 17 ++- gitnexus/src/mcp/local/local-backend.ts | 77 +++++------ gitnexus/src/mcp/repository-policy.ts | 23 +++- gitnexus/src/mcp/server.ts | 10 +- gitnexus/src/mcp/tools.ts | 5 +- .../integration/mcp-index-generation.test.ts | 2 +- .../query-symbol-integrity.test.ts | 57 ++++++++- gitnexus/test/unit/server.test.ts | 121 +++++++++++++----- 9 files changed, 241 insertions(+), 100 deletions(-) diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index 2610816ef..37a57f6c8 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -409,11 +409,11 @@ the paid job. Only these watchdogs have Actions write permission. The protected `gitnexus-evolution` environment holds these four settings (configured and verified by the live probes below): -| Setting | Kind | Purpose | -| --- | --- | --- | -| `GITNEXUS_EVOLUTION_AWS_ROLE_ARN` | Secret | AWS role assumed through GitHub OIDC | -| `GITNEXUS_EVOLUTION_EC2_INSTANCE_ID` | Secret | Dedicated runner instance | -| `GITNEXUS_EVOLUTION_AWS_REGION` | Variable | Instance region | +| Setting | Kind | Purpose | +| -------------------------------------- | -------- | ----------------------------------------------- | +| `GITNEXUS_EVOLUTION_AWS_ROLE_ARN` | Secret | AWS role assumed through GitHub OIDC | +| `GITNEXUS_EVOLUTION_EC2_INSTANCE_ID` | Secret | Dedicated runner instance | +| `GITNEXUS_EVOLUTION_AWS_REGION` | Variable | Instance region | | `GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC` | Variable | Actual weekly EventBridge stop, `DAY HH:MM` UTC | No long-lived AWS credential is stored. The AWS action obtains short-lived @@ -875,6 +875,25 @@ task ID, prompt, verification command, and oracle bytes produce a new task digest. Old agent patches are not available for rescoring; keep historical measurements unchanged and compare only runs with matching task contracts. +## MCP response cost + +With `include_content: true`, queries share a 16,000-character preview allowance +equally across unique symbols on the returned page. The database returns only +those prefixes, after ranking and pagination. `content_truncated: true` marks +incomplete source; every match remains present. Use `context` for a full symbol +or `read_file` for a bounded file range. This applies to CLI, HTTP, and MCP +queries; group queries already request no source content. + +MCP query guidance lives in the JSON `next` field. Explicit `maxTokens` retains +its existing final text cap, which can truncate JSON; it does not change the +query preview allowance. Payload reductions alone do not establish agent cost +or completion-time improvements: previews can require follow-up reads. + +MCP tool schemas expose the configured default repository name and path. The +sandbox already configures `/workspace` as its default, so the schema identifies +`benchmark-target` without adding benchmark-specific instructions to task prompts. +Compare stable and candidate runtimes using the same harness revision. + ## Relation to the SWE-bench harness The rest of `eval/` benchmarks GitNexus _tools_ inside a litellm agent loop diff --git a/gitnexus/scripts/bench/mcp-serialization.ts b/gitnexus/scripts/bench/mcp-serialization.ts index fdfe95ad1..de0d4b3c5 100644 --- a/gitnexus/scripts/bench/mcp-serialization.ts +++ b/gitnexus/scripts/bench/mcp-serialization.ts @@ -113,16 +113,23 @@ for (const entry of cases) { assert.equal(typeof text, 'string'); const currentText = text as string; const hint = getNextStepHint(entry.tool, entry.args, false, entry.result); - assert.ok(currentText.endsWith(hint)); - const body = hint ? currentText.slice(0, -hint.length) : currentText; - if (typeof entry.result === 'string') assert.equal(body, entry.result); - else assert.deepEqual(JSON.parse(body), entry.result); + if (entry.tool === 'query') { + const { next, ...body } = JSON.parse(currentText); + assert.equal(next, hint); + assert.deepEqual(body, entry.result); + } else { + assert.ok(currentText.endsWith(hint)); + const body = hint ? currentText.slice(0, -hint.length) : currentText; + if (typeof entry.result === 'string') assert.equal(body, entry.result); + else assert.deepEqual(JSON.parse(body), entry.result); + } const prettyText = (typeof entry.result === 'string' ? entry.result : JSON.stringify(entry.result, null, 2)) + hint; const baselineBytes = Buffer.byteLength(prettyText); const currentBytes = Buffer.byteLength(currentText); - assert.ok(currentBytes <= baselineBytes); + // Moving the query hint into JSON adds a field even for an empty result. + assert.ok(currentBytes <= baselineBytes + (entry.tool === 'query' ? 64 : 0)); measurements.push({ name: entry.name, baselineBytes, diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index dc6cdea20..8ab5bb786 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -154,6 +154,7 @@ import { QUERY_DEFAULT_MAX_SYMBOLS, QUERY_MAX_LIMIT, QUERY_MAX_MAX_SYMBOLS, + QUERY_CONTENT_MAX_CHARS, CONTEXT_CHAIN_MAX_DEPTH, CHECKOUT_SOURCE_TOOLS, READ_FILE_DEFAULT_MAX_LINES, @@ -3516,21 +3517,10 @@ export class LocalBackend { >(); const definitions: any[] = []; // standalone symbols not in any process - // Batch-fetch process participation, cohesion, and (optionally) content for - // ALL matched symbols in 2-3 graph queries instead of 2-3 *per symbol*. The - // previous per-symbol loop issued up to 3N sequential pool round-trips - // (searchLimit symbols × {STEP_IN_PROCESS, MEMBER_OF, content}); on a warm - // repo the IPC + query-setup overhead of those round-trips dominated query - // latency. Collapsing to `WHERE n.id IN $nodeIds` preserves identical output - // (the aggregation loop below is unchanged) while cutting the round-trips. - // Array params bind through the pool exactly as bm25Search's - // `WHERE n.id IN $nodeIds` already does. (Ported from gitnexus-enterprise - // PR #222 — N+1 → 2-3 batched queries.) const nodeIds = merged.map(([, m]) => m.data?.nodeId).filter((id): id is string => !!id); const processRowsByNode = new Map(); const cohesionByNode = new Map(); - const contentByNode = new Map(); // Set when a batched enrichment query throws a REAL failure (timeout, lock, // native fault) — NOT the benign "no Process/Community table" case, which is // a normal config (a repo analyzed without processes/communities) and must @@ -3611,30 +3601,6 @@ export class LocalBackend { logQueryError('query:cluster-info', e); if (!isBenignMissingTableError(e)) enrichmentDegraded = true; } - - // Optionally fetch content for every matched symbol. - if (includeContent) { - try { - const rows = await executeParameterized( - repo.lbugPath, - ` - MATCH (n) - WHERE n.id IN $nodeIds - RETURN n.id AS nodeId, n.content AS content - `, - { nodeIds: ids }, - ); - for (const row of rows) { - assertQueryIdentity(row, 'nodeId', 0); - const nid = row.nodeId ?? row[0]; - contentByNode.set(nid, row.content ?? row[1]); - } - } catch (e) { - rethrowSymbolIdentityError(e); - logQueryError('query:content-fetch', e); - if (!isBenignMissingTableError(e)) enrichmentDegraded = true; - } - } } // Aggregation is unchanged from the per-symbol version — it now reads the @@ -3657,7 +3623,6 @@ export class LocalBackend { const coh = cohesionByNode.get(sym.nodeId); const cohesion = coh?.cohesion ?? 0; const module = coh?.module; - const content = includeContent ? contentByNode.get(sym.nodeId) : undefined; const symbolEntry = { id: sym.nodeId, @@ -3667,7 +3632,6 @@ export class LocalBackend { startLine: toDisplayLine(sym.startLine), endLine: toDisplayLine(sym.endLine), ...(module ? { module } : {}), - ...(includeContent && content ? { content } : {}), }; if (processRows.length === 0) { @@ -3941,8 +3905,45 @@ export class LocalBackend { maxSymbolsPerProcess, chainByProcessId, }); + const returnedDefinitions = definitions.slice(0, 20); timer.stop(); // formatting + if (includeContent) { + timer.start('content'); + const targets = new Map< + string, + { id: string; content?: string; content_truncated?: boolean } + >(); + for (const symbol of [...process_symbols, ...returnedDefinitions]) { + if (symbol.id && !targets.has(symbol.id)) targets.set(symbol.id, symbol); + } + const previewChars = Math.floor(QUERY_CONTENT_MAX_CHARS / Math.max(1, targets.size)); + for (const ids of chunk([...targets.keys()], LBUG_QUERY_BATCH_SIZE)) { + try { + const rows = await executeParameterized( + repo.lbugPath, + `MATCH (n) WHERE n.id IN $nodeIds + RETURN n.id AS nodeId, substring(n.content, 1, $previewChars) AS content, + size(n.content) > $previewChars AS contentTruncated`, + { nodeIds: ids, previewChars }, + ); + for (const row of rows) { + assertQueryIdentity(row, 'nodeId', 0); + const target = targets.get(row.nodeId ?? row[0]); + if (!target) continue; + const content = row.content ?? row[1]; + if (content) target.content = content; + if (row.contentTruncated ?? row[2]) target.content_truncated = true; + } + } catch (e) { + rethrowSymbolIdentityError(e); + logQueryError('query:content-fetch', e); + if (!isBenignMissingTableError(e)) enrichmentDegraded = true; + } + } + timer.stop(); + } + // End-to-end wall time — deliberately a separate mark so callers can // compare sum(phases) vs wall to see how much Promise.all concurrency // saved. Must come before summary() so it's included. @@ -4134,7 +4135,7 @@ export class LocalBackend { return { processes, process_symbols, - definitions: definitions.slice(0, 20), // cap standalone definitions + definitions: returnedDefinitions, timing, ...(contentAvailability ? { contentAvailability } : {}), ...(warnings.length > 0 && { warning: warnings.join(' ') }), diff --git a/gitnexus/src/mcp/repository-policy.ts b/gitnexus/src/mcp/repository-policy.ts index 332e1b18f..f59e6fa6b 100644 --- a/gitnexus/src/mcp/repository-policy.ts +++ b/gitnexus/src/mcp/repository-policy.ts @@ -333,18 +333,31 @@ export class McpRepositoryPolicy { } toolForMcp(tool: GitNexusTool): GitNexusTool { - if (!this.restricted) return tool; + const defaultRepo = + this.defaultRepo ?? + (this.restricted && this.allowed.length === 1 ? this.allowed[0] : undefined); + if (!this.restricted && !defaultRepo) return tool; const properties = { ...tool.inputSchema.properties }; const repo = properties.repo; if (repo && typeof repo === 'object') { + const description = this.restricted + ? 'Allowed indexed repository name or path. Group-mode values are unavailable.' + : repo.description; properties.repo = { ...repo, - description: 'Allowed indexed repository name or path. Group-mode values are unavailable.', + description: defaultRepo + ? `${description} Default repository: ${JSON.stringify(defaultRepo.name)}.` + : description, + ...(defaultRepo ? { default: defaultRepo.path } : {}), }; } - delete properties.subgroup; - delete properties.crossDepth; - const description = scrubGroupDescription(tool.description); + if (this.restricted) { + delete properties.subgroup; + delete properties.crossDepth; + } + const description = this.restricted + ? scrubGroupDescription(tool.description) + : tool.description; return { ...tool, description, inputSchema: { ...tool.inputSchema, properties } }; } diff --git a/gitnexus/src/mcp/server.ts b/gitnexus/src/mcp/server.ts index de79d7530..5a39f642b 100644 --- a/gitnexus/src/mcp/server.ts +++ b/gitnexus/src/mcp/server.ts @@ -73,7 +73,7 @@ export function getNextStepHint( return `\n\n---\n**Next:** READ gitnexus://repo/{name}/context for any repo above to get its overview and check staleness. If pagination.hasMore is true, call list_repos again with offset set to pagination.nextOffset to fetch the rest.`; case 'query': - return `\n\n---\n**Next:** To understand a specific symbol in depth, use context({name: ""${repoParam}}) to see categorized refs and process participation.${args?.include_content === true ? ` For source of a process_symbols row without content, use context({uid: "", include_content: true${repoParam}}).` : ''}`; + return `Use context({uid: ""${repoParam}}) for a symbol's references and processes.${args?.include_content === true ? ` Source previews marked content_truncated are incomplete; use context({uid: "", include_content: true${repoParam}}) or read_file with a bounded line range for more.` : ''}`; case 'context': return `\n\n---\n**Next:** If planning changes, use impact({target: "${args?.name || ''}", direction: "upstream"${repoParam}}) to check blast radius. To see execution flows, READ gitnexus://repo/${repoPath}/processes.`; @@ -284,15 +284,17 @@ export function createMCPServer( await scopedBackend.callTool(name, backendArgs), responseFilter, ); - // Formatting whitespace costs model input tokens; keep every value and hint. - const resultText = typeof result === 'string' ? result : JSON.stringify(result); const hint = getNextStepHint(name, args as Record | undefined, readOnly, result); + const resultText = + name === 'query' && result !== null && typeof result === 'object' && !Array.isArray(result) + ? JSON.stringify({ ...result, next: hint }) + : (typeof result === 'string' ? result : JSON.stringify(result)) + hint; return { content: [ { type: 'text', - text: applyMcpMaxTokens(resultText + hint, maxTokens), + text: applyMcpMaxTokens(resultText, maxTokens), }, ], }; diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index f3fab5b60..2bee9f457 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -101,6 +101,8 @@ export const QUERY_DEFAULT_MAX_SYMBOLS = 25; /** Advertised query page maxima (schema + LocalBackend.query reject, not clamp). */ export const QUERY_MAX_LIMIT = 100; export const QUERY_MAX_MAX_SYMBOLS = 200; +/** Source preview characters shared across unique symbols on a query result page. */ +export const QUERY_CONTENT_MAX_CHARS = 16_000; export const CONTEXT_CHAIN_MAX_DEPTH = 3; const CWD_AWARE_REPO_OMISSION = @@ -222,8 +224,7 @@ ${HOT_READ_STALENESS_NOTE}`, }, include_content: { type: 'boolean', - description: - 'Include source text retained for matching symbols (default: false). The response reports contentAvailability; indexes built with content retention "none" explicitly report unavailable content. Content is sent once per symbol id, on its first process_symbols row; context({uid: "", include_content: true}) returns it for any row.', + description: `Include source text retained for matching symbols as previews (default: false), sharing ${QUERY_CONTENT_MAX_CHARS} characters equally across unique returned symbols per repository. content_truncated marks incomplete previews; all matches remain present. Content is sent once per symbol id, on its first process_symbols row; context({uid: "", include_content: true}) returns it for any row. Use read_file line ranges for large sources. contentAvailability reports index retention. Query next is JSON guidance.`, default: false, }, chain_depth: { diff --git a/gitnexus/test/integration/mcp-index-generation.test.ts b/gitnexus/test/integration/mcp-index-generation.test.ts index 643da2f56..463ed05af 100644 --- a/gitnexus/test/integration/mcp-index-generation.test.ts +++ b/gitnexus/test/integration/mcp-index-generation.test.ts @@ -96,7 +96,7 @@ describe('MCP index generation freshness', () => { const text = (result.content as Array<{ type: string; text: string }>).find( (item) => item.type === 'text', )!.text; - const hint = getNextStepHint(name, args); + const hint = name === 'query' ? '' : getNextStepHint(name, args); return JSON.parse(hint ? text.slice(0, -hint.length) : text); } diff --git a/gitnexus/test/integration/query-symbol-integrity.test.ts b/gitnexus/test/integration/query-symbol-integrity.test.ts index 6bc49ac23..d22f07c5a 100644 --- a/gitnexus/test/integration/query-symbol-integrity.test.ts +++ b/gitnexus/test/integration/query-symbol-integrity.test.ts @@ -9,7 +9,7 @@ import * as adapter from '../../src/core/lbug/lbug-adapter.js'; import { runFullAnalysis } from '../../src/core/run-analyze.js'; import { LocalBackend } from '../../src/mcp/local/local-backend.js'; import { SYMBOL_IDENTITY_RECOVERY_SUGGESTION } from '../../src/mcp/local/query-result-integrity.js'; -import { createMCPServer, getNextStepHint } from '../../src/mcp/server.js'; +import { createMCPServer } from '../../src/mcp/server.js'; import { getStoragePaths } from '../../src/storage/repo-manager.js'; import { commitAll, initGitRepo } from '../helpers/temp-git-repo.js'; import { createTempDir } from '../helpers/test-db.js'; @@ -87,7 +87,12 @@ describe('query identity boundary through the real MCP server', () => { await temp?.cleanup(); }); - async function query(mutate?: () => Promise, legacy = false, target = repo): Promise { + async function query( + mutate?: () => Promise, + legacy = false, + target = repo, + includeContent = true, + ): Promise { await fs.copyFile(snapshot, lbugPath); if (mutate) { await adapter.initLbug(lbugPath); @@ -105,7 +110,7 @@ describe('query identity boundary through the real MCP server', () => { const args = { repo: target, [legacy ? 'query' : 'search_query']: 'integrityNeedle', - include_content: true, + include_content: includeContent, chain_depth: 1, }; try { @@ -114,8 +119,7 @@ describe('query identity boundary through the real MCP server', () => { expect(result.isError).not.toBe(true); const content = result.content as Array<{ type: string; text: string }>; const text = content.find((item) => item.type === 'text')!.text; - const hint = getNextStepHint('query', args); - return JSON.parse(hint ? text.slice(0, -hint.length) : text); + return JSON.parse(text); } finally { await client.close(); await server.close(); @@ -140,6 +144,49 @@ describe('query identity boundary through the real MCP server', () => { expect(result.processes.length).toBeGreaterThan(0); }); + it.each(['x'.repeat(538_895), '"\\\n😀'.repeat(30_000)])( + 'bounds source previews in native queries without losing matches (%#)', + async (source) => { + const setSource = async () => { + for (const label of ['Function', 'File']) { + await adapter.executePrepared(`MATCH (n:${label}) SET n.content = $source`, { source }); + } + }; + const before = await query(setSource, false, repo, false); + const result = await query(setSource); + expect(result).not.toHaveProperty('error'); + expect(result).not.toHaveProperty('partial'); + expect(result.processes).toEqual(before.processes); + const rows = [...result.process_symbols, ...result.definitions]; + expect(rows.map(({ id, process_id }) => ({ id, process_id }))).toEqual( + [...before.process_symbols, ...before.definitions].map(({ id, process_id }) => ({ + id, + process_id, + })), + ); + const previews = rows.filter((row) => row.content !== undefined); + expect(previews.some((row) => row.id === UID)).toBe(true); + expect(new Set(previews.map((row) => row.id)).size).toBe(previews.length); + expect(previews.reduce((sum, row) => sum + [...row.content].length, 0)).toBeLessThanOrEqual( + 16_000, + ); + for (const row of previews) { + expect(row.content.length).toBeGreaterThan(0); + expect(source.startsWith(row.content)).toBe(true); + expect(row.content).not.toContain('\uFFFD'); + expect(row.content_truncated).toBe(true); + } + const backend = new LocalBackend(); + try { + await backend.init(); + const full = await backend.callTool('context', { repo, uid: UID, include_content: true }); + expect(full.symbol.content).toBe(source); + } finally { + await backend.dispose(); + } + }, + ); + it.each([ ['NUL symbol name', 'MATCH (n:Function {id: $uid}) SET n.name = $value', 'invalid\0name'], ['NUL symbol path', 'MATCH (n:Function {id: $uid}) SET n.filePath = $value', 'invalid\0path'], diff --git a/gitnexus/test/unit/server.test.ts b/gitnexus/test/unit/server.test.ts index 65626d856..a1d58b223 100644 --- a/gitnexus/test/unit/server.test.ts +++ b/gitnexus/test/unit/server.test.ts @@ -187,31 +187,44 @@ describe('createMCPServer', () => { } }); - it('keeps repo optional when a default repo is configured', async () => { - const backend = createMockBackend({ - listRepos: vi.fn().mockResolvedValue([ - { name: 'alpha', path: '/tmp/alpha' }, - { name: 'beta', path: '/tmp/beta' }, - ]), - }); - const repositoryPolicy = await createMcpRepositoryPolicy(backend, { - GITNEXUS_MCP_DEFAULT_REPO: 'alpha', - }); - const server = createMCPServer(backend, { repositoryPolicy }); - const client = new Client({ name: 'default-repo-client', version: '0.0.0' }); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + it.each([{ GITNEXUS_MCP_DEFAULT_REPO: 'alpha' }, { GITNEXUS_MCP_ALLOWED_REPOS: 'alpha' }])( + 'advertises the resolved repository default in tool schemas (%j)', + async (env) => { + const backend = createMockBackend({ + listRepos: vi.fn().mockResolvedValue([ + { name: 'alpha', path: '/tmp/alpha' }, + { name: 'beta', path: '/tmp/beta' }, + ]), + }); + const repositoryPolicy = await createMcpRepositoryPolicy(backend, env); + const server = createMCPServer(backend, { repositoryPolicy }); + const client = new Client({ name: 'default-repo-client', version: '0.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - try { - await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); - const tools = await client.listTools(); - const query = tools.tools.find((tool) => tool.name === 'query'); + try { + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + const tools = await client.listTools(); + const query = tools.tools.find((tool) => tool.name === 'query'); - expect(query?.inputSchema.required).not.toContain('repo'); - } finally { - await client.close(); - await server.close(); - } - }); + expect(query?.inputSchema.required).not.toContain('repo'); + expect(query?.inputSchema.properties?.repo).toMatchObject({ + default: '/tmp/alpha', + description: expect.stringContaining('"alpha"'), + }); + await client.callTool({ name: 'query', arguments: { search_query: 'auth' } }); + expect(backend.callTool).toHaveBeenLastCalledWith('query', { + search_query: 'auth', + repo: '/tmp/alpha', + }); + expect( + GITNEXUS_TOOLS.find((tool) => tool.name === 'query')?.inputSchema.properties.repo, + ).not.toHaveProperty('default'); + } finally { + await client.close(); + await server.close(); + } + }, + ); it('requires repo when multiple allowed repositories are visible without a default', async () => { const backend = createMockBackend({ @@ -261,9 +274,9 @@ describe('getNextStepHint (via tool call response)', () => { }); const QUERY_HINT_BASE = - '\n\n---\n**Next:** To understand a specific symbol in depth, use context({name: "", repo: "demo"}) to see categorized refs and process participation.'; + 'Use context({uid: "", repo: "demo"}) for a symbol\'s references and processes.'; - it('query hint is unchanged when include_content is not set', async () => { + it('query hint uses the returned identity and repository', async () => { const backend = createMockBackend({ callTool: vi.fn().mockResolvedValue({ processes: [] }), }); @@ -271,8 +284,8 @@ describe('getNextStepHint (via tool call response)', () => { search_query: 'auth', repo: 'demo', }); - expect(text.endsWith(QUERY_HINT_BASE)).toBe(true); - expect(text).not.toContain('context({uid:'); + expect(JSON.parse(text).next).toBe(QUERY_HINT_BASE); + expect(text).not.toContain('content_truncated'); }); it('query hint names the context({uid, include_content}) fallback when include_content is true', async () => { @@ -284,11 +297,9 @@ describe('getNextStepHint (via tool call response)', () => { repo: 'demo', include_content: true, }); - expect( - text.endsWith( - `${QUERY_HINT_BASE} For source of a process_symbols row without content, use context({uid: "", include_content: true, repo: "demo"}).`, - ), - ).toBe(true); + expect(JSON.parse(text).next).toBe( + `${QUERY_HINT_BASE} Source previews marked content_truncated are incomplete; use context({uid: "", include_content: true, repo: "demo"}) or read_file with a bounded line range for more.`, + ); }); it('grep hint converts the 1-based hit line to a 0-based read_file window', async () => { @@ -384,7 +395,45 @@ describe('getNextStepHint rename', () => { }); describe('MCP output budgets', () => { - it('leaves the complete formatted response unchanged when no budget is configured', async () => { + it('serializes query previews and diagnostics as one JSON document', async () => { + const result = { + processes: [{ id: 'process-1', symbol_count: 2 }], + process_symbols: [ + { id: 'Function:small', process_id: 'process-1', content: 'function small() {}' }, + { + id: 'Function:large', + process_id: 'process-1', + content: 'large preview', + content_truncated: true, + }, + ], + definitions: [ + { + id: 'File:large.ts', + filePath: 'large.ts', + content: 'file preview', + content_truncated: true, + }, + ], + warning: 'Some keyword indexes are unavailable', + partial: true, + staleness: { status: 'behind', commitsBehind: 2 }, + contentAvailability: 'available', + }; + const backend = createMockBackend({ callTool: vi.fn().mockResolvedValue(result) }); + const { text, isError } = await callToolThroughServer(backend, 'query', { + search_query: 'large', + include_content: true, + }); + + expect(isError).toBe(false); + const { next, ...body } = JSON.parse(text); + expect(body).toEqual(result); + expect(next).toContain('read_file'); + expect(result).not.toHaveProperty('next'); + }); + + it('preserves small query data and guidance without source omissions', async () => { const previous = process.env.GITNEXUS_MCP_DEFAULT_MAX_TOKENS; delete process.env.GITNEXUS_MCP_DEFAULT_MAX_TOKENS; try { @@ -395,8 +444,10 @@ describe('MCP output budgets', () => { search_query: 'auth', }); expect(isError).toBe(false); - expect(text).toContain('"payload":"complete"'); - expect(text).toContain('**Next:**'); + expect(JSON.parse(text)).toMatchObject({ + payload: 'complete', + next: expect.stringContaining('context'), + }); expect(text.endsWith('\n…')).toBe(false); } finally { if (previous === undefined) delete process.env.GITNEXUS_MCP_DEFAULT_MAX_TOKENS; From c417ccfc81574462a11c00780ae6ee9458bd0967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Mon, 26 Oct 2026 15:13:48 +0000 Subject: [PATCH 6/6] perf(ingestion): reduce indexing overhead and declaration memory (#3726) * refactor(ingestion): share callable declaration facts Reuse pure parameter and signature facts between structure and scope extraction, with tree-owned caches and read-only parameter contracts. Preserve provider-specific wrappers and consumer projections. Add characterization tests, real-worker parity checks, and reproducible performance measurements. Keep the overall indexing effect explicit. Refs #3723 * fix(bench): create result files exclusively Address CodeQL review feedback on #3726. Use a private default directory and an exclusive file handle so existing files and symlinks cannot be overwritten. * perf: speed analyzer identity guard probes * perf(ingestion): reuse callable type and overload facts * ci: retain callable time and memory benchmarks * chore: use established CI performance measurements * fix(bench): check callable baseline in CI * perf(ingestion): guard callable scaling and reuse export checks * fix(ci): budget for callable scaling matrix * refactor(ingestion): simplify callable handling --------- Co-authored-by: Gergo Magyar --- .github/workflows/ci-tests.yml | 16 +- ARCHITECTURE.md | 11 + .../bench/method-declaration-facts/README.md | 155 ++++ .../bench/method-declaration-facts/RESULTS.md | 93 ++ .../method-declaration-facts/baseline.json | 541 ++++++++++++ .../method-declaration-facts/measure.mjs | 799 ++++++++++++++++++ .../bench/method-declaration-facts/memory.mjs | 105 +++ .../method-declaration-facts/scaling.mjs | 28 + .../trace-parameters.mjs | 47 ++ gitnexus/src/core/analyzer-identity.ts | 47 +- .../src/core/ingestion/export-detection.ts | 30 +- .../languages/csharp/arity-metadata.ts | 6 +- .../languages/dart/arity-metadata.ts | 4 +- .../languages/java/arity-metadata.ts | 6 +- .../languages/kotlin/arity-metadata.ts | 6 +- .../ingestion/languages/php/arity-metadata.ts | 6 +- .../languages/python/arity-metadata.ts | 4 +- .../languages/swift/arity-metadata.ts | 11 +- .../languages/typescript/arity-metadata.ts | 16 +- .../method-extractors/configs/csharp.ts | 3 +- .../method-extractors/configs/dart.ts | 7 +- .../method-extractors/configs/jvm.ts | 5 +- .../method-extractors/configs/php.ts | 3 +- .../method-extractors/configs/python.ts | 3 +- .../method-extractors/configs/swift.ts | 7 +- .../configs/typescript-javascript.ts | 3 +- .../method-extractors/declaration-facts.ts | 61 ++ gitnexus/src/core/ingestion/method-types.ts | 4 +- .../src/core/ingestion/utils/method-props.ts | 4 +- .../core/ingestion/workers/parse-worker.ts | 20 +- gitnexus/test/integration/parsing.test.ts | 48 +- .../unit/analyzer-identity-guard-io.test.ts | 189 +++++ .../unit/declaration-parameter-types.test.ts | 86 ++ .../unit/method-declaration-facts.test.ts | 296 +++++++ .../unit/method-declaration-scaling.test.ts | 70 ++ gitnexus/test/unit/method-props.test.ts | 4 +- 36 files changed, 2676 insertions(+), 68 deletions(-) create mode 100644 gitnexus/bench/method-declaration-facts/README.md create mode 100644 gitnexus/bench/method-declaration-facts/RESULTS.md create mode 100644 gitnexus/bench/method-declaration-facts/baseline.json create mode 100644 gitnexus/bench/method-declaration-facts/measure.mjs create mode 100644 gitnexus/bench/method-declaration-facts/memory.mjs create mode 100644 gitnexus/bench/method-declaration-facts/scaling.mjs create mode 100644 gitnexus/bench/method-declaration-facts/trace-parameters.mjs create mode 100644 gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts create mode 100644 gitnexus/test/unit/analyzer-identity-guard-io.test.ts create mode 100644 gitnexus/test/unit/declaration-parameter-types.test.ts create mode 100644 gitnexus/test/unit/method-declaration-facts.test.ts create mode 100644 gitnexus/test/unit/method-declaration-scaling.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1da1c4662..c9e0a8d41 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -691,7 +691,8 @@ jobs: benchmarks: name: benchmarks (GITNEXUS_BENCH) runs-on: ubuntu-latest - timeout-minutes: 25 + # The callable size matrix runs 63 full analyses and 18 heap probes. + timeout-minutes: 30 steps: # persist-credentials: false — this job only runs npm + vitest benchmarks # and never pushes; the default-persisted token in .git/config would be at @@ -710,6 +711,19 @@ jobs: node --import tsx bench/python-scope/import-target-fingerprint.mjs --check working-directory: gitnexus + - name: Callable declaration facts / time / memory baseline + if: ${{ !cancelled() }} + run: node --import tsx bench/method-declaration-facts/measure.mjs --check --output method-declaration-facts.json + working-directory: gitnexus + + - name: Upload callable declaration measurements + if: always() + uses: ./.github/actions/upload-test-artifact + with: + name: method-declaration-facts-benchmark + path: gitnexus/method-declaration-facts.json + retention-days: 5 + - name: Java wildcard-static route constant guards (#3110) if: ${{ !cancelled() }} # Build-free: named-import control vs wildcard materialization; diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 26bc36874..4635dbff8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -356,6 +356,17 @@ accesses, value uses and inheritance edges. Register a `ScopeResolver` in `SCOPE_RESOLVERS` to participate; the old per-language migration flag and parallel call-resolution DAG were removed in RING4-1 #942. +Method extraction still supplies graph properties and overload identity; scope +extraction builds the declaration metadata used by resolution. Their repeated +node-only signature callbacks share facts through `memoizeDeclarationFact` +(`ingestion/method-extractors/declaration-facts.ts`). Each callback owns a WeakMap +from the immutable parse tree to native node IDs, so separate traversal wrappers +reuse one result without retaining old trees. Parameter facts are read-only; +providers can map syntax wrappers to the same declaration identity. Each consumer +keeps its own arity/type normalization. Owner-dependent flags and file-dependent +receiver rules remain separate. The cache stays inside parser isolates; serialized +graph/ParsedFile payloads and the persistent parse-cache format remain unchanged. + A provider can explicitly restrict its edge mode. COBOL currently uses `callable-flow-only`: its established processor retains structural CALLS/IMPORTS, and scope resolution contributes procedure-pointer calls. Registration alone is diff --git a/gitnexus/bench/method-declaration-facts/README.md b/gitnexus/bench/method-declaration-facts/README.md new file mode 100644 index 000000000..16bf844ea --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/README.md @@ -0,0 +1,155 @@ +# Shared callable declaration facts + +## CI measurements + +The existing **benchmarks (GITNEXUS_BENCH)** job in `ci-tests.yml` runs: + +```sh +node --import tsx bench/method-declaration-facts/measure.mjs --check --output method-declaration-facts.json +``` + +`--check` compares the current build with committed `baseline.json`; no second +checkout is required. It checks all eight providers' semantic fingerprints and +shared parameter/type-array counts, cold/warm/changed graph and cache counts at +each workload size, and tree collection. Three fresh-process +samples report median wall/CPU time, endpoint heap, peak RSS, and retained heap, +with percentage differences from the recorded measurements. Timing and heap/RSS +are machine-dependent references, not absolute CI failure thresholds. + +Like the C# pipeline benchmark, this check increases input size and compares cost +growth with input growth. It uses the stricter **1.5** normalized growth budget +already used by the Python scope benchmark: + +```text +normalized growth = (large cost / small cost) / (large input / small input) +``` + +| Workload | Sizes | Gated costs | +| --- | --- | --- | +| Java + TypeScript, spread across files | 8 / 32 / 128 files per language, 40 methods per class | Cold/warm/changed wall time, CPU, endpoint heap, sampled peak RSS and process maximum RSS | +| Java + TypeScript, concentrated in one class per language | 320 / 1,280 / 5,120 methods per class | Same full-indexing measurements | +| Retained declaration metadata, separately for Java and TypeScript | 2,500 / 10,000 / 40,000 declarations | Extraction time, extraction retained heap, emitted retained heap | + +Both indexing shapes contain 640 / 2,560 / 10,240 generated methods, plus fixed +call and edit probes. The harness verifies the emitted method and parsed/cache-hit +file counts, so silently dropping work cannot improve a scaling result. Size order +alternates between repetitions. Every adjacent pair **and the full size span** must +stay within the budget, using per-size medians from the same run. Missing, zero, +negative or non-finite measurements fail. `--record` also enforces these guards. +Unit tests demonstrate that quadratic time or memory growth fails the check. + +The artifact includes raw samples, median costs and normalized ratios. These +bounded workloads guard against scaling regressions; they do not prove linear +complexity for every repository. Fixed startup costs can mask growth at small +sizes. Released heap deltas remain diagnostic because they can be zero or negative +after collection. + +CI uploads raw samples and comparisons as `method-declaration-facts-benchmark`, +using the same five-day retention as the other benchmark artifacts. The separate +all-language [scope-resolution benchmark](../scope-resolution/README.md) remains +unchanged. + +After reviewing an intentional fixture, semantic, or allocation change, run the +same command with `--record` instead of `--check`, then inspect `baseline.json`. +CI never records or accepts a new baseline automatically. Output files must not +already exist. The two-checkout commands below remain available for diagnostics. + +## Detailed local measurements + +Build the pre-change revision and candidate with `npm run build` in each package, +then run from the candidate's `gitnexus/` directory: + +```sh +node bench/method-declaration-facts/measure.mjs \ + --baseline-root /path/to/baseline/gitnexus \ + --samples 3 --output /tmp/method-declaration-facts.json +``` + +The output file must not exist; existing files and symlinks are rejected before +measurements start. Without `--output`, the harness creates a private temporary +directory and prints the result path. Results are written after all pairs finish. + +Use `--candidate-root` to measure a built candidate snapshot elsewhere. Put both +runtimes and their dependencies on the same filesystem. The source corpus always +comes from the checkout containing the script, and its content hash must match +between revisions. Each pair alternates revision order and uses fresh processes. + +The harness measures three forced analysis runs per process: a cold index, warm +parse-cache replay, and a changed method signature. It uses one real parse worker, +disables FTS and embeddings, and materializes graph edges for comparison. It checks +all structural nodes and edges, excluding Community/Process nodes and their membership edges, +between revisions; warm replay must match cold output, and the edit must change +the graph. Cache hit/reparse counts accompany every measurement. + +Two corpora exercise different costs: 64 generated Java/TypeScript class files +with 40 methods each, and the first 96 tracked ingestion provider sources plus +tracked same-arity, primary-constructor, record-component, and variadic fixtures. +Each includes a small file whose signature changes. These are bounded workloads, +not a claim about every repository or language distribution. + +Pass `--providers-only` for a separate nine-file parity corpus covering all eight +migrated providers plus the changed-file probe. It includes overload declarations, +a Java record, a C# primary constructor, a Kotlin companion, Python decoration, +PHP variadics, and Swift/Dart constructors. Every provider must emit its sample +callable and a resolved call to it. This mode uses the same repeated cold, warm, +and changed-file graph comparisons; all eight grammars must be installed. + +Counters separately run real scope and MethodInfo consumers for all eight +migrated languages. They report distinct parameter arrays and declaration-node +AST accesses, with semantic output hashes. These are instrumentation runs, outside +the timed analysis. A forced-GC probe reports collection of 100 discarded trees; +it does not add GC assertions to the unit suite. +Separate parameter-only timings call the callback twice for each of 1,000 +declarations in a fresh tree, across nine trials per provider. Parsing, forced GC, +and the counter instrumentation are outside those timed regions. + +Wall time includes analysis and persistence. CPU and RSS include worker threads. +RSS sampling uses 20 ms intervals; cumulative maximum RSS is the process high-water +mark, so later scenarios may inherit an earlier peak. GC duration covers only the +main isolate. Progress-phase times use callback intervals, not exclusive CPU +profiles. Compare distributions and report noise; reduced extraction counts alone +do not establish an end-to-end speedup. + +See [the recorded results](RESULTS.md) for the initial measurements and their limits. + +## Retained memory + +Run the same probe against both built revisions, alternating their order across +at least five fresh-process pairs for each language: + +```sh +node --expose-gc bench/method-declaration-facts/memory.mjs \ + --root /path/to/runtime/gitnexus --language java --declarations 10000 +node --expose-gc bench/method-declaration-facts/memory.mjs \ + --root /path/to/runtime/gitnexus --language typescript --declarations 10000 +``` + +The probe holds the parsed tree, runs scope arity and structure extraction, then +releases method records, the tree, and emitted metadata in stages. It reports +forced-GC JavaScript heap at each stage, a semantic digest, and whether the native +tree became collectible. `extractionRetainedBytes` subtracts the tree-only heap; +`emittedRetainedBytes` subtracts the initialized empty heap after releasing the +tree. `releasedDeltaBytes` includes remaining cache capacity and runtime warmup. +Compare those deltas rather than absolute process heap. + +This probe measures one isolate and a small declaration shape. Its RSS and +extraction timing are **not** end-to-end indexing measurements. Use `measure.mjs` +for complete cold/warm/changed indexing, including worker and native-memory RSS. +Keep both runtimes on the same filesystem with the same dependencies and give +every end-to-end child a fresh private `XDG_RUNTIME_DIR` so analyzer identity +caches start equally cold. The CI check includes this retained-heap probe in +the same `method-declaration-facts-benchmark` report as the indexing measurements. + +To count requests and distinct parameter arrays in real workers, use the optional +preload below from the candidate package. Repeat with the candidate runtime and a +different, nonexistent output file. The preload also runs inside parser workers; +group records by thread, tree, provider, and node, and count unique `(thread, array)` +pairs. These runs are instrumented: discard their timing and memory measurements. + +```sh +FACT_RUNTIME_ROOT=/path/to/baseline/gitnexus \ +FACT_COUNTER_LOG=/tmp/baseline-parameters.jsonl \ +NODE_OPTIONS='--import=/absolute/path/to/gitnexus/bench/method-declaration-facts/trace-parameters.mjs' \ +node --expose-gc bench/method-declaration-facts/measure.mjs \ + --child analysis --root /path/to/baseline/gitnexus --corpus repository +``` diff --git a/gitnexus/bench/method-declaration-facts/RESULTS.md b/gitnexus/bench/method-declaration-facts/RESULTS.md new file mode 100644 index 000000000..95ef6065f --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/RESULTS.md @@ -0,0 +1,93 @@ +# Measurement results — 2026-10-25 + +The measurements below are the original extraction-cache experiment, before +the later analyzer-identity and shared parameter-type optimizations. Ongoing CI +measurements use the existing [scope-resolution benchmark](../scope-resolution/README.md). +The later isolated experiments remain available in [GitHub Actions](https://github.com/abhigyanpatwari/GitNexus/actions/runs/40289711096). + +Baseline: `ee7eff97f7f4eb7d68f7090cba448f7cc8285db2`. Candidate: this refactor before commit. The SHA-256 of sorted changed production paths and file bytes, each separated by NUL, is `b6e82236a38c3a0cbd073f2ef162885613869a32c6da9c2d906ecc85a6ff5ea6`. Both revisions were built independently and used the same installed dependencies. + +Runtime: Node v22.23.3, Linux, 12th Gen Intel(R) Core(TM) i9-12900K, one parse worker. Runtimes, dependencies, and temporary corpora were on the same native filesystem. This was a shared development host, without CPU isolation. Full tests and graph indexing were started only after these measurements. + +## Extraction work + +For all eight providers, scope and structure consumers produced identical semantic hashes. Distinct parameter arrays fell from **2 to 1**. Separate uninstrumented trials called parameter extraction twice on each of 1,000 declarations in a fresh tree; parsing and forced GC were outside the timed region. The table reports medians of nine trials per provider. These small fixtures measure extraction cost, not overall indexing. + +| Provider | Baseline wall ms | Candidate wall ms | Baseline CPU ms | Candidate CPU ms | Root AST accesses before → after | +| ---------- | ---------------: | ----------------: | --------------: | ---------------: | -------------------------------: | +| java | 32.60 | 15.46 | 60.19 | 32.73 | 63 → 62 | +| typescript | 37.75 | 19.20 | 64.63 | 36.29 | 59 → 58 | +| kotlin | 47.40 | 28.13 | 75.01 | 49.54 | 105 → 101 | +| csharp | 60.80 | 30.37 | 92.36 | 52.97 | 181 → 180 | +| python | 89.85 | 44.75 | 152.75 | 84.62 | 67 → 57 | +| php | 27.56 | 17.37 | 43.57 | 27.17 | 63 → 62 | +| swift | 85.46 | 48.28 | 98.17 | 64.23 | 269 → 246 | +| dart | 77.63 | 34.35 | 102.06 | 55.27 | 50 → 66 | + +The Dart wrapper-to-inner-declaration lookup adds root-node accesses while avoiding the second parameter traversal. Root access counts do not count descendant reads, `tree`, or `id`; they are not total AST traversal counts. Scope-specific transformations still run independently. + +A separate real-worker trace of the repository corpus, aggregated across cold, +warm, and changed-file analysis, recorded **1,391 parameter requests** in both +revisions. They produced **1,391 distinct arrays before and 784 after** (43.6% +fewer). Of 784 declarations, 607 were requested twice and 177 arrow functions were +requested once. Those single requests still incur cache bookkeeping. The observed +migrated callbacks in this corpus were TypeScript; cold analysis parsed 98 of its +142 manifest files. Graph hashes still matched. These trace runs overlapped the +test suite, so their timing and memory values are excluded from the tables below. +The optional preload in [README.md](README.md) reproduces these counters. + +## End-to-end analysis + +Three paired trials per corpus alternated revision order. Each process ran cold analysis, warm parse-cache replay, and a changed-file analysis. Wall time includes persistence. All six pairs matched structural graph hashes and node/edge counts in every scenario. Cold/warm graphs matched; changed signatures changed the graph. Cold and changed runs used real parser workers; warm runs replayed cache entries without workers. + +| Corpus | Files | Content manifest SHA-256 | +| ------------ | ----: | ------------------------------------------------------------------ | +| declarations | 65 | `a8f8aa6206812f6bdcd24fce9d89dcf25d14eaf7b1c86ba49aea1ca64f729e40` | +| repository | 142 | `ccafcd91d22aaccd886e978717499148823ca804132d64956f7f1ca9e9df1277` | + +Wall times are median [minimum–maximum], milliseconds. + +| Corpus / scenario | Baseline | Candidate | Median change | +| ---------------------- | ------------------: | ------------------: | ------------: | +| declarations / cold | 6621 [6485–7814] | 7616 [6773–8238] | +15.0% | +| declarations / warm | 2939 [2754–3693] | 3027 [2828–3125] | +3.0% | +| declarations / changed | 3565 [3564–4160] | 3721 [3343–4708] | +4.4% | +| repository / cold | 13737 [11731–15283] | 16039 [10623–18116] | +16.8% | +| repository / warm | 6829 [5838–7794] | 7580 [5162–10662] | +11.0% | +| repository / changed | 7556 [6348–7656] | 8035 [5832–9154] | +6.3% | + +Additional medians, shown baseline → candidate. RSS is the sampled peak, CPU includes workers, and GC measures only the main isolate. Parsing is a progress-callback interval rather than an exclusive profile. + +| Corpus / scenario | CPU ms | RSS MiB | Main GC ms | Parsing interval ms | +| ---------------------- | ----------------: | ------------: | ------------: | ------------------: | +| declarations / cold | 7367.8 → 8149.3 | 530.5 → 558.7 | 131.4 → 170.2 | 3286.3 → 3211.9 | +| declarations / warm | 2931.8 → 2876.1 | 636.6 → 651.3 | 195.1 → 198.4 | 175.6 → 154.4 | +| declarations / changed | 3397.3 → 3522.6 | 708.9 → 713.0 | 121.3 → 120.9 | 1008.0 → 1060.6 | +| repository / cold | 17199.8 → 20591.9 | 759.5 → 753.4 | 229.3 → 306.4 | 6954.7 → 8205.3 | +| repository / warm | 7137.5 → 7288.9 | 883.5 → 873.4 | 248.3 → 268.6 | 299.7 → 315.3 | +| repository / changed | 7698.1 → 8652.0 | 951.2 → 954.2 | 253.2 → 261.4 | 1202.5 → 1242.8 | + +The final end-to-end medians were slower. An earlier complete six-pair run, before the narrow Python/Dart wrapper identity correction, showed cold median changes of +1.7% for declarations and −4.4% for repository sources. The final run also shifted warm-cache timings, where the extraction cache is not used. These observations and the wide overlapping ranges leave the overall indexing effect unresolved; they do **not** establish a speedup or rule out a regression. Repeat on an isolated host and larger representative corpora before claiming an end-to-end gain. + +The isolated measurement does establish lower repeated parameter-extraction cost. It does not account for workload-dependent cache bookkeeping across every declaration shape. Retention remained bounded in the forced-GC probe: **100/100 discarded trees were collected in both revisions**. Post-GC main heap was 6.87 MiB before and 6.88 MiB after. Warm reparse counts were zero; changed-file runs reparsed one file. The RSS sample and small lifetime probe cannot establish a general peak-memory improvement. + +Reproduce with the command and limitations in [README.md](README.md). The harness emits per-trial JSON, including all phase intervals, CPU, GC, sampled/cumulative RSS, cache counts, corpus manifests, graph hashes, and parameter-only timings. + +## Mixed-provider worker parity + +Three additional paired runs used `--providers-only`: eight provider files plus +the changed-file probe. Every run asserted that each provider emitted its sample +callable and a resolved call to it. The fixtures include Java records, C# primary +constructors, Kotlin companions, overloads, Python decorators, PHP variadics, and +Swift/Dart constructors. Swift's existing `Function` classification is preserved. + +All pairs matched **47 structural nodes and 46 edges** in every scenario. Cold +runs parsed nine files; warm runs parsed zero; changed runs parsed one. The cold +and warm graph hash was +`131f1043fb43b9635228595748bd1a15d853b10f86ed40902303612918b15169`; +the changed graph hash was +`93734f2c8ea9ba0d8841e269467dc49cd2289be83aa18b13a8cebe871a6106ab`. +The corpus manifest was +`0a75726a1e88b189f63675a9e12476ca3d928d1a25ed4abf9648efbd0c9b339f`. +These extra runs overlapped the full test suite and provide correctness evidence; +their timing and memory measurements are excluded from the performance tables. diff --git a/gitnexus/bench/method-declaration-facts/baseline.json b/gitnexus/bench/method-declaration-facts/baseline.json new file mode 100644 index 000000000..5f6e0f671 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/baseline.json @@ -0,0 +1,541 @@ +{ + "_note": "Reviewed semantic and allocation baseline. --check compares fingerprints, cache counts and shared-array counts and gates same-run normalized time/memory growth at 1.5. Absolute timing and heap/RSS medians are references. CI never records a baseline.", + "environment": { + "node": "v22.23.3", + "platform": "linux", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", + "workers": 1 + }, + "correctness": { + "counts": [ + { + "language": "java", + "facts": "f1acdfa027c072521b69ca5b9b694ebb3a528e45f5383edffe0b9d4adfbe70d0", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "typescript", + "facts": "51290686f9885e3ecc18c6de969b53a825d8677b3ffaeb82e752e1d996b3d56e", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "kotlin", + "facts": "32250ca227cb267e75efd4856184dd96e17ec08c23cdbdc61b8301b602efe071", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "csharp", + "facts": "e7158eb25c8b100ec6d835b32ca28df16246dbc32ac9a202e202e4bfc865ef30", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "python", + "facts": "78ba377afae0198395d638d5ff9aa3a7c44504666184003b099e2709fe544a4d", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "php", + "facts": "d23a4178521e676a4d267ff365dd6de11d9b5df8b0880ee380acadec5933158b", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "swift", + "facts": "39ef708f6d9c3515fa0c95ce79b719e2c3ef7638fa5a9c59a1254a8766568c02", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "dart", + "facts": "2379fc6fb6ce0ba092304790eebc1e36030e939c16bfe51f0aabca6112590f07", + "parameterArrays": 1, + "parameterTypeArrays": 1 + } + ], + "analysis": { + "declarations-8": { + "manifest": { + "files": 17, + "manifest": "353a6948d690eaba364c8d31df4937d1b969e855ccb7bed1ded1c95b6a528847" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 691, + "edges": 690, + "hash": "c2918b76eea971e5d925afd2e912c53ce372da6abfd38ba4adcccade22c07ef9" + }, + "reparsed": 17, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 691, + "edges": 690, + "hash": "c2918b76eea971e5d925afd2e912c53ce372da6abfd38ba4adcccade22c07ef9" + }, + "reparsed": 0, + "hits": 17 + }, + { + "scenario": "changed", + "facts": { + "nodes": 691, + "edges": 690, + "hash": "e220495c5c40fba7820c452b44c6ced2302981e60a7428eb4bb73ec628d7daa8" + }, + "reparsed": 1, + "hits": 16 + } + ] + }, + "declarations": { + "manifest": { + "files": 65, + "manifest": "a8f8aa6206812f6bdcd24fce9d89dcf25d14eaf7b1c86ba49aea1ca64f729e40" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 2755, + "edges": 2754, + "hash": "f9f0363efa933fcd0fdfbb64076e43bd7eb361f880251fda2de6dac1f716a0fa" + }, + "reparsed": 65, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 2755, + "edges": 2754, + "hash": "f9f0363efa933fcd0fdfbb64076e43bd7eb361f880251fda2de6dac1f716a0fa" + }, + "reparsed": 0, + "hits": 65 + }, + { + "scenario": "changed", + "facts": { + "nodes": 2755, + "edges": 2754, + "hash": "58969c80be830cff4ba91511587aec077106936e7abe80d95f77ad284be2fcd4" + }, + "reparsed": 1, + "hits": 64 + } + ] + }, + "declarations-128": { + "manifest": { + "files": 257, + "manifest": "b9625a8c2fe98276a16847c4d77f9e4c46cd696a61d555440d7840faf0107780" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 11011, + "edges": 11010, + "hash": "11fbfb9fdf42fb9ac78c11e359fb9d7a6aea448887a65650b9d54e039f3e73a2" + }, + "reparsed": 257, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 11011, + "edges": 11010, + "hash": "11fbfb9fdf42fb9ac78c11e359fb9d7a6aea448887a65650b9d54e039f3e73a2" + }, + "reparsed": 0, + "hits": 257 + }, + { + "scenario": "changed", + "facts": { + "nodes": 11011, + "edges": 11010, + "hash": "88a8baaa628aa037c02c83b6d912ef9ca206fd63c035a284dc55a9c3f1ed4903" + }, + "reparsed": 1, + "hits": 256 + } + ] + }, + "concentrated-320": { + "manifest": { + "files": 3, + "manifest": "b2bcb40fa14120ac4ada96e78569ffa2413250ea3009dbf39b7d4d4c1bb533da" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 649, + "edges": 648, + "hash": "4606ff07912fef5a01f2b057e6b9d9bc5df3cc3a40295fd558fca8d45df9527e" + }, + "reparsed": 3, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 649, + "edges": 648, + "hash": "4606ff07912fef5a01f2b057e6b9d9bc5df3cc3a40295fd558fca8d45df9527e" + }, + "reparsed": 0, + "hits": 3 + }, + { + "scenario": "changed", + "facts": { + "nodes": 649, + "edges": 648, + "hash": "562dc4337b5840e0a64c1b3f4d4f90cc1f2fd75a49ccb23a50b7c6adcbf3fd07" + }, + "reparsed": 1, + "hits": 2 + } + ] + }, + "concentrated-1280": { + "manifest": { + "files": 3, + "manifest": "59d63e1f0a4b92388b4b9c9008e90338d50839ebc7c8779c86169f1adac57a06" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 2569, + "edges": 2568, + "hash": "ad17f7f24a6948d140ccfdca0230f9562a3e44717a5931a29ae62af504899041" + }, + "reparsed": 3, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 2569, + "edges": 2568, + "hash": "ad17f7f24a6948d140ccfdca0230f9562a3e44717a5931a29ae62af504899041" + }, + "reparsed": 0, + "hits": 3 + }, + { + "scenario": "changed", + "facts": { + "nodes": 2569, + "edges": 2568, + "hash": "90625066c2e3ef78881ef820b573f5a1201cb5b50b96a3a5d9bb4a6572dbeb5a" + }, + "reparsed": 1, + "hits": 2 + } + ] + }, + "concentrated-5120": { + "manifest": { + "files": 3, + "manifest": "8b2fb11e9b2a4ba1c124c1efdb22e55db91b8595e1a519f55da75e94af0a5ff1" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 10249, + "edges": 10248, + "hash": "f1f7ef339c2ba2f15f1c38fc9c7caffc0d5d92a222059cc21352a17fb30a2879" + }, + "reparsed": 3, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 10249, + "edges": 10248, + "hash": "f1f7ef339c2ba2f15f1c38fc9c7caffc0d5d92a222059cc21352a17fb30a2879" + }, + "reparsed": 0, + "hits": 3 + }, + { + "scenario": "changed", + "facts": { + "nodes": 10249, + "edges": 10248, + "hash": "4b521b01aef30d967ea1ce113f8021a1f2c30319ef9102f303986c77390b4c27" + }, + "reparsed": 1, + "hits": 2 + } + ] + }, + "providers": { + "manifest": { + "files": 9, + "manifest": "0a75726a1e88b189f63675a9e12476ca3d928d1a25ed4abf9648efbd0c9b339f" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 47, + "edges": 46, + "hash": "131f1043fb43b9635228595748bd1a15d853b10f86ed40902303612918b15169" + }, + "reparsed": 9, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 47, + "edges": 46, + "hash": "131f1043fb43b9635228595748bd1a15d853b10f86ed40902303612918b15169" + }, + "reparsed": 0, + "hits": 9 + }, + { + "scenario": "changed", + "facts": { + "nodes": 47, + "edges": 46, + "hash": "93734f2c8ea9ba0d8841e269467dc49cd2289be83aa18b13a8cebe871a6106ab" + }, + "reparsed": 1, + "hits": 8 + } + ] + } + }, + "memory": { + "java-2500": { + "digest": "5d72a942d521e860c36563aceb57cc32d17857273a18d0d428b4266bd617e06f", + "declarations": 2500 + }, + "java": { + "digest": "44e5fc1e221e02a9644075284d18b3936d3afc36b1ff696341f7ab5a4a5b15b0", + "declarations": 10000 + }, + "java-40000": { + "digest": "bb0f0b9a3281108e1aa910478e8f9171f0e375e96bf527e7020f46bb5e04cba3", + "declarations": 40000 + }, + "typescript-2500": { + "digest": "14e9b13d31e8874849cfd64f2367716b09ed9d6577912f75318f358bdc8c741d", + "declarations": 2500 + }, + "typescript": { + "digest": "9e1ae4a2165a63a3f89aa97d474e799ffc18d09cb47578b5a112a7ed75c54e40", + "declarations": 10000 + }, + "typescript-40000": { + "digest": "632331d244854ab84e4974e4ea7ef669460e3acc5e97e933cb822ca0e39844c2", + "declarations": 40000 + } + }, + "collectedTrees": 100 + }, + "measured": { + "declarations-8/cold": { + "wallMs": 4598.0965830000005, + "cpuMs": 5028.881, + "heapUsedMiB": 69.43971252441406, + "sampledPeakRssMiB": 552.4375, + "cumulativeMaxRssMiB": 552.4375 + }, + "declarations-8/warm": { + "wallMs": 1469.5753610000002, + "cpuMs": 1310.647, + "heapUsedMiB": 100.86992645263672, + "sampledPeakRssMiB": 639.1484375, + "cumulativeMaxRssMiB": 639.1484375 + }, + "declarations-8/changed": { + "wallMs": 2224.902616000001, + "cpuMs": 2158.139, + "heapUsedMiB": 70.97301483154297, + "sampledPeakRssMiB": 691.4453125, + "cumulativeMaxRssMiB": 691.4453125 + }, + "declarations/cold": { + "wallMs": 6287.4714890000005, + "cpuMs": 7338.656, + "heapUsedMiB": 102.16807556152344, + "sampledPeakRssMiB": 568.0625, + "cumulativeMaxRssMiB": 574.22265625 + }, + "declarations/warm": { + "wallMs": 1898.1291850000007, + "cpuMs": 2115.823, + "heapUsedMiB": 107.16117858886719, + "sampledPeakRssMiB": 672.890625, + "cumulativeMaxRssMiB": 679.06640625 + }, + "declarations/changed": { + "wallMs": 2573.0706090000003, + "cpuMs": 2741.011, + "heapUsedMiB": 101.39332580566406, + "sampledPeakRssMiB": 741.0625, + "cumulativeMaxRssMiB": 747.2890625 + }, + "declarations-128/cold": { + "wallMs": 11950.934368, + "cpuMs": 15088.994, + "heapUsedMiB": 96.21513366699219, + "sampledPeakRssMiB": 642.6796875, + "cumulativeMaxRssMiB": 642.8671875 + }, + "declarations-128/warm": { + "wallMs": 3071.7164410000005, + "cpuMs": 3799.132, + "heapUsedMiB": 151.96864318847656, + "sampledPeakRssMiB": 752.5546875, + "cumulativeMaxRssMiB": 758.421875 + }, + "declarations-128/changed": { + "wallMs": 3785.085203999999, + "cpuMs": 4623.536, + "heapUsedMiB": 165.27832794189453, + "sampledPeakRssMiB": 825.6484375, + "cumulativeMaxRssMiB": 832.015625 + }, + "concentrated-320/cold": { + "wallMs": 4694.445621, + "cpuMs": 4989.052, + "heapUsedMiB": 67.74372100830078, + "sampledPeakRssMiB": 543.921875, + "cumulativeMaxRssMiB": 543.921875 + }, + "concentrated-320/warm": { + "wallMs": 1478.7918250000002, + "cpuMs": 1216.909, + "heapUsedMiB": 98.2383804321289, + "sampledPeakRssMiB": 634.32421875, + "cumulativeMaxRssMiB": 634.32421875 + }, + "concentrated-320/changed": { + "wallMs": 2217.1457689999997, + "cpuMs": 2067.268, + "heapUsedMiB": 69.39957427978516, + "sampledPeakRssMiB": 703.64453125, + "cumulativeMaxRssMiB": 703.64453125 + }, + "concentrated-1280/cold": { + "wallMs": 8088.226249, + "cpuMs": 9146.62, + "heapUsedMiB": 110.2262191772461, + "sampledPeakRssMiB": 601.5078125, + "cumulativeMaxRssMiB": 602.08203125 + }, + "concentrated-1280/warm": { + "wallMs": 1780.5142610000003, + "cpuMs": 1871.551, + "heapUsedMiB": 116.76529693603516, + "sampledPeakRssMiB": 714.1328125, + "cumulativeMaxRssMiB": 714.75390625 + }, + "concentrated-1280/changed": { + "wallMs": 2454.9453699999995, + "cpuMs": 2596.301, + "heapUsedMiB": 117.36157989501953, + "sampledPeakRssMiB": 772.80078125, + "cumulativeMaxRssMiB": 772.80078125 + }, + "concentrated-5120/cold": { + "wallMs": 19038.470589, + "cpuMs": 21988.894, + "heapUsedMiB": 87.05318450927734, + "sampledPeakRssMiB": 806.7421875, + "cumulativeMaxRssMiB": 807.34765625 + }, + "concentrated-5120/warm": { + "wallMs": 2599.4626009999993, + "cpuMs": 3164.342, + "heapUsedMiB": 196.42921447753906, + "sampledPeakRssMiB": 931.328125, + "cumulativeMaxRssMiB": 931.328125 + }, + "concentrated-5120/changed": { + "wallMs": 3300.2307550000005, + "cpuMs": 3837.783, + "heapUsedMiB": 198.03103637695312, + "sampledPeakRssMiB": 1009.9921875, + "cumulativeMaxRssMiB": 1009.9921875 + }, + "providers/cold": { + "wallMs": 5234.342168, + "cpuMs": 5416.471, + "heapUsedMiB": 64.36933135986328, + "sampledPeakRssMiB": 581.26953125, + "cumulativeMaxRssMiB": 587.6796875 + }, + "providers/warm": { + "wallMs": 1649.076336, + "cpuMs": 1537.857, + "heapUsedMiB": 80.01580810546875, + "sampledPeakRssMiB": 682.64453125, + "cumulativeMaxRssMiB": 688.83984375 + }, + "providers/changed": { + "wallMs": 2417.3744080000006, + "cpuMs": 2417.786, + "heapUsedMiB": 65.51468658447266, + "sampledPeakRssMiB": 764.375, + "cumulativeMaxRssMiB": 770.55859375 + }, + "java-2500/retained": { + "extractionMs": 97.82188899999998, + "extractionRetainedBytes": 3176480, + "emittedRetainedBytes": 1596952, + "releasedDeltaBytes": 332792 + }, + "java/retained": { + "extractionMs": 378.996999, + "extractionRetainedBytes": 12057136, + "emittedRetainedBytes": 5701312, + "releasedDeltaBytes": 660560 + }, + "java-40000/retained": { + "extractionMs": 1499.86337, + "extractionRetainedBytes": 47273448, + "emittedRetainedBytes": 21694640, + "releasedDeltaBytes": 1447192 + }, + "typescript-2500/retained": { + "extractionMs": 116.37295999999999, + "extractionRetainedBytes": 3198712, + "emittedRetainedBytes": 1619784, + "releasedDeltaBytes": 355624 + }, + "typescript/retained": { + "extractionMs": 472.583266, + "extractionRetainedBytes": 12017024, + "emittedRetainedBytes": 5662008, + "releasedDeltaBytes": 621256 + }, + "typescript-40000/retained": { + "extractionMs": 1857.2782389999998, + "extractionRetainedBytes": 47241792, + "emittedRetainedBytes": 21663784, + "releasedDeltaBytes": 1416336 + } + } +} diff --git a/gitnexus/bench/method-declaration-facts/measure.mjs b/gitnexus/bench/method-declaration-facts/measure.mjs new file mode 100644 index 000000000..efae55061 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/measure.mjs @@ -0,0 +1,799 @@ +#!/usr/bin/env node +// Build first, then use --check; paired diagnostics accept --baseline-root. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { cp, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { performance, PerformanceObserver } from 'node:perf_hooks'; +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { evaluateScaling } from './scaling.mjs'; + +const here = fileURLToPath(import.meta.url); +const candidate = path.resolve(path.dirname(here), '../..'); +const args = process.argv.slice(2); +const option = (name, fallback) => (args.includes(name) ? args[args.indexOf(name) + 1] : fallback); +const root = path.resolve(option('--root', candidate)); +const load = (relative) => import(pathToFileURL(path.join(root, 'dist', relative))); +const digest = (value) => createHash('sha256').update(value).digest('hex'); +const stable = (value) => + JSON.stringify(value, (_, v) => + v && !Array.isArray(v) && typeof v === 'object' + ? Object.fromEntries( + Object.keys(v) + .sort() + .map((key) => [key, v[key]]), + ) + : v, + ); + +const providerFixtures = { + 'J.java': + 'record J(int value) { int f(int x) { return x; } String f(String x) { return x; } int call() { return f(1); } int fromValue() { return f(value); } }', + 'T.ts': + 'export class T { f(x: number): number; f(x: string): string; f(x: number | string) { return x; } call() { return this.f(1); } }', + 'K.kt': + 'class K(val value: Int) {\n fun f(x: Int): Int { return x }\n fun call(): Int { return f(value) }\n companion object { fun build(): K { return K(1) } }\n}', + 'C.cs': + 'public class C(int value) { public int f(int x) => x; public string f(string x) => x; public int Call() => f(1); public int FromValue() => f(value); }', + 'P.py': + 'class P:\n @staticmethod\n def f(x: int) -> int:\n return x\n def call(self) -> int:\n return P.f(1)\n', + 'Ph.php': + 'f(1, "a"); } }', + 'S.swift': + 'class S { init() {}\n func f(_ x: Int, y: String = "") -> Int { return x }\n func call() -> Int { return f(1) }\n}', + 'D.dart': + "class D { D(int value) {} int f(int x, [String y = '']) => x; int call() => f(1); int get value => 1; }", +}; + +async function countFacts() { + const require = createRequire(path.join(root, 'package.json')); + const Parser = require('tree-sitter'); + const { getLanguageGrammar } = await load('core/tree-sitter/parser-loader.js'); + const { createMethodExtractor } = await load('core/ingestion/method-extractors/generic.js'); + const { buildMethodProps } = await load('core/ingestion/utils/method-props.js'); + const specs = [ + [ + 'java', + 'jvm', + 'java', + 'method_declaration', + 'class C { int f(int x, String y) { return x; } }', + ], + [ + 'typescript', + 'typescript-javascript', + 'typescript', + 'method_definition', + 'class C { f(x: number, y: string): number { return x; } }', + ], + [ + 'kotlin', + 'jvm', + 'kotlin', + 'function_declaration', + 'class C {\n fun f(x: Int, y: String): Int { return x }\n}', + ], + [ + 'csharp', + 'csharp', + 'csharp', + 'method_declaration', + 'class C { int f(int x, string y) { return x; } }', + ], + [ + 'python', + 'python', + 'python', + 'function_definition', + 'class C:\n def f(self, x: int, y: str) -> int:\n return x\n', + ], + [ + 'php', + 'php', + 'php', + 'method_declaration', + ' Int { return x } }', + ], + [ + 'dart', + 'dart', + 'dart', + 'method_signature', + 'class C { int f(int x, String y) { return x; } }', + ], + ]; + const result = []; + for (const [language, module, prefix, nodeType, source] of specs) { + const grammar = getLanguageGrammar(language); + if (!grammar) { + result.push({ language, skipped: 'grammar unavailable' }); + continue; + } + const config = (await load(`core/ingestion/method-extractors/configs/${module}.js`))[ + `${prefix}MethodConfig` + ]; + const arityModule = await load(`core/ingestion/languages/${language}/arity-metadata.js`); + const arity = Object.values(arityModule).find((value) => typeof value === 'function'); + const parser = new Parser(); + parser.setLanguage(grammar); + const node = parser.parse(source).rootNode.descendantsOfType(nodeType)[0]; + let nativeReads = 0; + const counted = new Proxy(node, { + get(target, property) { + if (property !== 'tree' && property !== 'id') nativeReads++; + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const parameterResults = new Set(); + const original = config.extractParameters; + config.extractParameters = (...values) => { + const parameters = original(...values); + parameterResults.add(parameters); + return parameters; + }; + try { + const scope = arity(counted); + const method = createMethodExtractor(config).extractFromNode(counted, { + language, + filePath: 'fixture', + }); + const graph = buildMethodProps(method); + result.push({ + language, + nativeReads, + parameterArrays: parameterResults.size, + parameterTypeArrays: new Set([scope.parameterTypes, graph.parameterTypes]).size, + facts: digest(stable({ scope, method })), + }); + } finally { + config.extractParameters = original; + } + const timingSource = + language === 'php' + ? ' { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar('java')); + return Array.from({ length: 100 }, () => { + const tree = parser.parse('class C { int f(int x) { return x; } }'); + const node = tree.rootNode.descendantsOfType('method_declaration')[0]; + javaMethodConfig.extractParameters(node); + javaMethodConfig.extractParameters(node); + assert.equal(javaExportChecker(node.childForFieldName('name'), 'f'), false); + return new WeakRef(tree); + }); + })(); + // WeakRef targets survive the current job; collect only after yielding it. + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setImmediate(resolve)); + global.gc(); + } + return { + created: refs.length, + collected: refs.filter((ref) => ref.deref() === undefined).length, + heapAfterGcMiB: process.memoryUsage().heapUsed / 1048576, + }; +} + +async function makeCorpus(repo, corpus) { + await mkdir(repo, { recursive: true }); + await writeFile(path.join(repo, '.gitignore'), '.gitnexus/\n'); + if (corpus === 'providers') { + for (const [file, source] of Object.entries(providerFixtures)) { + await writeFile(path.join(repo, file), source + '\n'); + } + } else if (corpus === 'declarations') { + const files = Number(option('--files', '32')); + const methods = Number(option('--methods', '40')); + assert(Number.isSafeInteger(files) && files > 0); + assert(Number.isSafeInteger(methods) && methods > 0); + const ts = Array.from( + { length: methods }, + (_, i) => `m${i}(a: number, b: string, c: Array, d = 0): number { return a + d; }`, + ).join('\n'); + const java = Array.from( + { length: methods }, + (_, i) => `int m${i}(int a, String b, List c, int d) { return a + d; }`, + ).join('\n'); + for (let file = 0; file < files; file++) { + await writeFile( + path.join(repo, `T${file}.ts`), + `export class T${file} {\n${ts}\n call() { return this.m0(1, "", [], 0); }\n}\n`, + ); + await writeFile( + path.join(repo, `J${file}.java`), + `class J${file} {\n${java}\n int call() { return m0(1, "", null, 0); }\n}\n`, + ); + } + } else { + // The same tracked source snapshot feeds both revisions. + const paths = execFileSync( + 'git', + [ + 'ls-files', + 'src/core/ingestion/method-extractors', + 'src/core/ingestion/languages', + 'test/fixtures/lang-resolution', + ], + { cwd: candidate, encoding: 'utf8' }, + ) + .trim() + .split('\n'); + const sources = paths + .filter((p) => p.startsWith('src/') && /\.ts$/.test(p)) + .sort() + .slice(0, 96); + const fixtures = paths.filter((p) => + /(?:same-arity|primary-constructor|record-component|variadic)/.test(p), + ); + for (const file of [...sources, ...fixtures]) { + await mkdir(path.dirname(path.join(repo, file)), { recursive: true }); + await cp(path.join(candidate, file), path.join(repo, file)); + } + } + await writeFile( + path.join(repo, 'change.ts'), + 'export class Changed { method(value: number): number { return value; } }\n', + ); + const git = (...values) => execFileSync('git', values, { cwd: repo, stdio: 'pipe' }); + git('init', '-q'); + git('add', '-A'); + git( + '-c', + 'user.name=benchmark', + '-c', + 'user.email=benchmark@example.invalid', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'fixture', + ); + const files = git('ls-files').toString().trim().split('\n'); + const contents = await Promise.all( + files.map(async (f) => [f, digest(await readFile(path.join(repo, f)))]), + ); + return { files: files.length - 1, manifest: digest(stable(contents)) }; +} + +function graphFacts(graph) { + const nodes = []; + graph.forEachNode((node) => { + if (!['Community', 'Process'].includes(node.label)) nodes.push(stable(node)); + }); + const edges = [...graph.relationships] + .filter((edge) => !['MEMBER_OF', 'STEP_IN_PROCESS'].includes(edge.type)) + .map(stable); + return { + nodes: nodes.length, + edges: edges.length, + hash: digest(stable([nodes.sort(), edges.sort()])), + }; +} + +async function measureAnalysis(corpus) { + const temp = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-callable-facts-')); + Object.assign(process.env, { + GITNEXUS_HOME: path.join(temp, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_LBUG_BUFFER_POOL_SIZE: '134217728', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + GITNEXUS_MEMORY: 'off', + }); + const repo = path.join(temp, 'repo'); + try { + const manifest = await makeCorpus(repo, corpus); + const { runFullAnalysis } = await load('core/run-analyze.js'); + const runs = []; + for (const scenario of ['cold', 'warm', 'changed']) { + if (scenario === 'changed') + await writeFile( + path.join(repo, 'change.ts'), + 'export class Changed { method(value: string): string { return value; } }\n', + ); + global.gc?.(); + let peakRss = process.memoryUsage().rss; + let gcMs = 0; + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) gcMs += entry.duration; + }); + observer.observe({ entryTypes: ['gc'] }); + const sampler = setInterval(() => { + peakRss = Math.max(peakRss, process.memoryUsage().rss); + }, 20); + const phases = {}; + let previousPhase = 'startup', + previousTime = performance.now(); + const onProgress = (phase) => { + if (phase !== previousPhase) console.error(`${scenario}: ${phase}`); + const now = performance.now(); + phases[previousPhase] = (phases[previousPhase] ?? 0) + now - previousTime; + previousPhase = phase; + previousTime = now; + }; + const cpu = process.cpuUsage(), + start = performance.now(); + let analysis; + try { + analysis = await runFullAnalysis( + repo, + { + force: true, + skipAgentsMd: true, + skipSkills: true, + skipFts: true, + streamGraphEmit: false, + workerPoolSize: 1, + }, + { onProgress, onLog: (message) => console.error(message) }, + ); + } finally { + clearInterval(sampler); + } + const wallMs = performance.now() - start, + used = process.cpuUsage(cpu); + onProgress('done'); + await new Promise((resolve) => setImmediate(resolve)); + observer.disconnect(); + const pipeline = analysis.pipelineResult; + assert(pipeline, 'analysis must return its graph'); + if (scenario !== 'warm') + assert(pipeline.usedWorkerPool, 'cache misses must use real parser workers'); + if (scenario === 'warm') assert.equal(pipeline.reparsedFileCount, 0); + if (scenario === 'changed') assert(pipeline.reparsedFileCount > 0); + if (corpus === 'declarations') { + const files = Number(option('--files', '32')); + const methods = Number(option('--methods', '40')); + let declarations = 0; + pipeline.graph.forEachNode((node) => { + if (node.label === 'Method' && /^m\d+$/.test(node.properties.name)) declarations++; + }); + assert.equal(declarations, 2 * files * methods, 'Scaling workload lost declarations'); + assert.equal(pipeline.reparsedFileCount + pipeline.parseCacheHitFileCount, 2 * files + 1); + } + if (corpus === 'providers') { + if (scenario === 'cold') { + assert.equal(pipeline.reparsedFileCount, Object.keys(providerFixtures).length + 1); + } + const methods = new Set(); + pipeline.graph.forEachNode((node) => { + if (['Method', 'Function'].includes(node.label) && node.properties.name === 'f') { + methods.add(node.properties.filePath); + } + }); + assert.deepEqual([...methods].sort(), Object.keys(providerFixtures).sort()); + const calledFiles = new Set(); + for (const edge of pipeline.graph.iterRelationshipsByType('CALLS')) { + const target = pipeline.graph.getNode(edge.targetId); + if (target?.properties.name === 'f') calledFiles.add(target.properties.filePath); + } + assert.deepEqual([...calledFiles].sort(), Object.keys(providerFixtures).sort()); + } + const facts = graphFacts(pipeline.graph); + if (scenario === 'warm') + assert.equal(facts.hash, runs[0].facts.hash, 'warm replay changed graph facts'); + if (scenario === 'changed') + assert.notEqual(facts.hash, runs[0].facts.hash, 'edited signature retained stale facts'); + runs.push({ + scenario, + wallMs, + cpuMs: (used.user + used.system) / 1000, + phases, + gcMs, + sampledPeakRssMiB: Math.max(peakRss, process.memoryUsage().rss) / 1048576, + cumulativeMaxRssMiB: process.resourceUsage().maxRSS / 1024, + heapUsedMiB: process.memoryUsage().heapUsed / 1048576, + reparsed: pipeline.reparsedFileCount, + hits: pipeline.parseCacheHitFileCount, + facts, + }); + } + return { manifest, runs }; + } finally { + await rm(temp, { recursive: true, force: true }); + } +} + +async function child(revision, mode, corpus, extraArgs = []) { + const cache = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-callable-identity-')); + try { + const memory = mode === 'memory'; + const output = execFileSync( + process.execPath, + memory + ? [ + '--expose-gc', + fileURLToPath(new URL('./memory.mjs', import.meta.url)), + '--root', + revision, + '--language', + corpus, + ...extraArgs, + ] + : [ + '--expose-gc', + here, + '--child', + mode, + '--root', + revision, + '--corpus', + corpus ?? '', + ...extraArgs, + ], + { + encoding: 'utf8', + timeout: 600_000, + maxBuffer: 32 * 1024 * 1024, + env: { ...process.env, GITNEXUS_MEMORY: 'off', XDG_RUNTIME_DIR: cache }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + if (memory) return JSON.parse(output); + const line = output.split('\n').find((line) => line.startsWith('BENCH_RESULT=')); + assert(line, 'child did not return measurements'); + return JSON.parse(line.slice('BENCH_RESULT='.length)); + } finally { + await rm(cache, { recursive: true, force: true }); + } +} + +async function checkBenchmark() { + const record = args.includes('--record'); + assert(!(record && args.includes('--check')), '--check cannot record a baseline'); + assert( + !['--baseline-root', '--candidate-root', '--providers-only'].some((flag) => + args.includes(flag), + ), + 'CI modes use the fixed declaration and provider corpora', + ); + const baselinePath = new URL('./baseline.json', import.meta.url); + const baseline = record ? undefined : JSON.parse(await readFile(baselinePath, 'utf8')); + const samples = Number(option('--samples', '3')); + assert(Number.isInteger(samples) && samples >= 3, '--samples must be at least 3'); + const output = option('--output'); + assert( + !output || path.resolve(output) !== fileURLToPath(baselinePath), + 'Output must not overwrite baseline.json', + ); + const outputFile = output ? await open(output, 'wx', 0o600) : undefined; + const report = { + passed: false, + environment: { + node: process.version, + platform: process.platform, + cpu: os.cpus()[0]?.model, + workers: 1, + }, + samples, + counts: [], + lifetime: {}, + trials: {}, + memory: {}, + measured: {}, + scaling: {}, + }; + const median = (rows, key) => { + assert( + rows.every( + (row) => Number.isFinite(row[key]) && (key === 'releasedDeltaBytes' || row[key] > 0), + ), + `Invalid ${key} sample`, + ); + return rows.map((row) => row[key]).sort((a, b) => a - b)[Math.floor(rows.length / 2)]; + }; + const summarize = (rows, keys) => Object.fromEntries(keys.map((key) => [key, median(rows, key)])); + try { + report.counts = await child(root, 'counts'); + assert.equal(report.counts.length, 8); + assert( + report.counts.every((row) => !row.skipped), + 'All eight provider grammars are required', + ); + report.lifetime = await child(root, 'lifetime'); + assert.equal( + report.lifetime.collected, + report.lifetime.created, + 'declaration cache retained trees', + ); + const correctness = { + counts: report.counts.map(({ language, facts, parameterArrays, parameterTypeArrays }) => ({ + language, + facts, + parameterArrays, + parameterTypeArrays, + })), + analysis: {}, + memory: {}, + collectedTrees: report.lifetime.collected, + }; + const workloads = [ + ...[8, 32, 128].map((files) => ({ + name: files === 32 ? 'declarations' : `declarations-${files}`, + shape: 'spread', + files, + methods: 40, + corpus: 'declarations', + })), + ...[320, 1280, 5120].map((methods) => ({ + name: `concentrated-${methods}`, + shape: 'concentrated', + files: 1, + methods, + corpus: 'declarations', + })), + { name: 'providers', corpus: 'providers' }, + ]; + report.workloads = workloads; + for (const { name } of workloads) report.trials[name] = []; + for (let trial = 0; trial < samples; trial++) { + for (const { name, corpus, files, methods } of trial % 2 + ? [...workloads].reverse() + : workloads) { + console.log(`${name} ${trial + 1}/${samples}`); + const extraArgs = + corpus === 'declarations' ? ['--files', String(files), '--methods', String(methods)] : []; + report.trials[name].push(await child(root, 'analysis', corpus, extraArgs)); + } + } + const analysisMetrics = [ + 'wallMs', + 'cpuMs', + 'heapUsedMiB', + 'sampledPeakRssMiB', + 'cumulativeMaxRssMiB', + ]; + for (const { name } of workloads) { + const trials = report.trials[name]; + const facts = (trial) => ({ + manifest: trial.manifest, + runs: trial.runs.map(({ scenario, facts, reparsed, hits }) => ({ + scenario, + facts, + reparsed, + hits, + })), + }); + correctness.analysis[name] = facts(trials[0]); + for (const trial of trials) { + assert.deepEqual( + facts(trial), + correctness.analysis[name], + `${name}: nondeterministic graph or cache replay`, + ); + assert.equal(trial.runs[2].reparsed, 1); + } + for (const scenario of ['cold', 'warm', 'changed']) { + report.measured[`${name}/${scenario}`] = summarize( + trials.map((trial) => trial.runs.find((run) => run.scenario === scenario)), + analysisMetrics, + ); + } + } + for (const shape of ['spread', 'concentrated']) { + for (const scenario of ['cold', 'warm', 'changed']) { + report.scaling[`${shape}/${scenario}`] = evaluateScaling( + workloads + .filter((row) => row.shape === shape) + .map(({ name, files, methods }) => ({ + size: 2 * files * methods, + ...report.measured[`${name}/${scenario}`], + })), + analysisMetrics, + ); + } + } + const memorySizes = [2500, 10000, 40000]; + const memoryMetrics = ['extractionMs', 'extractionRetainedBytes', 'emittedRetainedBytes']; + for (const language of ['java', 'typescript']) { + const nameFor = (size) => (size === 10000 ? language : `${language}-${size}`); + for (const size of memorySizes) report.memory[nameFor(size)] = []; + for (let trial = 0; trial < samples; trial++) { + for (const size of trial % 2 ? [...memorySizes].reverse() : memorySizes) { + console.log(`${language} retained ${size} ${trial + 1}/${samples}`); + report.memory[nameFor(size)].push( + await child(root, 'memory', language, ['--declarations', String(size)]), + ); + } + } + for (const size of memorySizes) { + const name = nameFor(size); + const trials = report.memory[name]; + assert( + trials.every((trial) => trial.treeCollected), + `${language}: heap probe retained trees`, + ); + assert.equal( + new Set(trials.map((trial) => trial.digest)).size, + 1, + `${language}: nondeterministic metadata`, + ); + assert( + trials.every((trial) => trial.declarations === size), + `${name}: wrong declaration count`, + ); + correctness.memory[name] = { + digest: trials[0].digest, + declarations: trials[0].declarations, + }; + report.measured[`${name}/retained`] = summarize(trials, [ + ...memoryMetrics, + 'releasedDeltaBytes', + ]); + } + report.scaling[`${language}/retained`] = evaluateScaling( + memorySizes.map((size) => ({ size, ...report.measured[`${nameFor(size)}/retained`] })), + memoryMetrics, + ); + } + report.correctness = correctness; + console.log(JSON.stringify({ scaling: report.scaling }, null, 2)); + assert( + Object.values(report.scaling).every((row) => row.passed), + 'Callable time or memory growth exceeds the 1.5 normalized scaling budget', + ); + if (baseline) + assert.deepEqual( + correctness, + baseline.correctness, + 'callable declaration facts or allocation counts differ from baseline.json', + ); + report.baselineMeasured = baseline?.measured; + report.changesPercent = baseline + ? Object.fromEntries( + Object.entries(report.measured).map(([name, metrics]) => [ + name, + Object.fromEntries( + Object.entries(metrics).map(([key, value]) => [ + key, + 100 * (value / baseline.measured[name][key] - 1), + ]), + ), + ]), + ) + : {}; + console.log( + JSON.stringify({ measured: report.measured, changesPercent: report.changesPercent }, null, 2), + ); + if (record) + await writeFile( + baselinePath, + JSON.stringify( + { + _note: + 'Reviewed semantic and allocation baseline. --check compares fingerprints, cache counts and shared-array counts and gates same-run normalized time/memory growth at 1.5. Absolute timing and heap/RSS medians are references. CI never records a baseline.', + environment: report.environment, + correctness, + measured: report.measured, + }, + null, + 2, + ) + '\n', + ); + report.passed = true; + console.log(`[method-declaration-facts ${record ? '--record' : '--check'}] PASS`); + } finally { + if (outputFile) { + await outputFile.writeFile(JSON.stringify(report, null, 2) + '\n'); + await outputFile.close(); + } + } +} + +if (args.includes('--child')) { + const mode = option('--child'); + const result = + mode === 'counts' + ? await countFacts() + : mode === 'lifetime' + ? await retainedTrees() + : await measureAnalysis(option('--corpus')); + console.log('BENCH_RESULT=' + JSON.stringify(result)); +} else if (args.includes('--check') || args.includes('--record')) { + await checkBenchmark(); +} else { + const baseline = option('--baseline-root'); + const candidateRuntime = path.resolve(option('--candidate-root', candidate)); + assert(baseline, 'Supply --baseline-root pointing at the built pre-change package'); + const samples = Number(option('--samples', '3')); + assert(Number.isInteger(samples) && samples >= 3, '--samples must be at least 3'); + const outputPath = + option('--output') ?? + path.join(await mkdtemp(path.join(os.tmpdir(), 'gitnexus-callable-results-')), 'results.json'); + const outputFile = await open(outputPath, 'wx', 0o600); + try { + const results = { + measuredAt: new Date().toISOString(), + environment: { + node: process.version, + platform: process.platform, + cpu: os.cpus()[0]?.model, + workers: 1, + }, + counts: {}, + lifetime: {}, + samples: [], + }; + for (const [name, revision] of [ + ['baseline', baseline], + ['candidate', candidateRuntime], + ]) { + results.counts[name] = await child(revision, 'counts'); + results.lifetime[name] = await child(revision, 'lifetime'); + } + assert.deepEqual( + results.counts.baseline.map((r) => [r.language, r.facts]), + results.counts.candidate.map((r) => [r.language, r.facts]), + ); + for (const corpus of args.includes('--providers-only') + ? ['providers'] + : ['declarations', 'repository']) { + for (let sample = 0; sample < samples; sample++) { + const pair = {}; + for (const name of sample % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) { + console.log(`${corpus} pair ${sample + 1}/${samples}: ${name}`); + pair[name] = await child( + name === 'baseline' ? baseline : candidateRuntime, + 'analysis', + corpus, + ); + } + assert.deepEqual(pair.baseline.manifest, pair.candidate.manifest); + assert.deepEqual( + pair.baseline.runs.map((r) => r.facts), + pair.candidate.runs.map((r) => r.facts), + 'before/after graph facts differ', + ); + results.samples.push({ corpus, sample, ...pair }); + } + } + await outputFile.writeFile(JSON.stringify(results, null, 2) + '\n'); + console.log(`Measurements saved to ${outputPath}`); + } finally { + await outputFile.close(); + } +} diff --git a/gitnexus/bench/method-declaration-facts/memory.mjs b/gitnexus/bench/method-declaration-facts/memory.mjs new file mode 100644 index 000000000..c766b2058 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/memory.mjs @@ -0,0 +1,105 @@ +// Isolate retained JS heap from whole-process RSS. Run with --expose-gc. +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const args = process.argv.slice(2); +const option = (name, fallback) => (args.includes(name) ? args[args.indexOf(name) + 1] : fallback); +const root = path.resolve(option('--root', '.')); +const language = option('--language', 'java'); +const count = Number(option('--declarations', '10000')); +assert(global.gc, 'Run node with --expose-gc'); +assert(['java', 'typescript'].includes(language)); +assert(Number.isSafeInteger(count) && count > 0); +const load = (name) => import(pathToFileURL(path.join(root, 'dist', name))); +const Parser = createRequire(path.join(root, 'package.json'))('tree-sitter'); +const { getLanguageGrammar } = await load('core/tree-sitter/parser-loader.js'); +const { createMethodExtractor } = await load('core/ingestion/method-extractors/generic.js'); +const { buildMethodProps } = await load('core/ingestion/utils/method-props.js'); +const java = language === 'java'; +const configModule = await load( + `core/ingestion/method-extractors/configs/${java ? 'jvm' : 'typescript-javascript'}.js`, +); +const config = configModule[`${language}MethodConfig`]; +const arityModule = await load(`core/ingestion/languages/${language}/arity-metadata.js`); +const arity = arityModule[java ? 'computeJavaArityMetadata' : 'computeTsArityMetadata']; +const extractor = createMethodExtractor(config); +const parser = new Parser(); +parser.setLanguage(getLanguageGrammar(language)); +const source = `class C {\n${Array.from({ length: count }, (_, i) => + java + ? `int f${i}(int x, String y) { return x; }` + : `f${i}(x: number, y: string): number { return x; }`, +).join('\n')}\n}`; +const nodeType = java ? 'method_declaration' : 'method_definition'; +const context = { filePath: `fixture.${java ? 'java' : 'ts'}`, language }; + +async function heap() { + // WeakRef targets and pending native finalizers need a new event-loop turn. + for (let i = 0; i < 4; i++) { + await new Promise((resolve) => setImmediate(resolve)); + global.gc(); + } + return process.memoryUsage().heapUsed; +} + +// Warm module/class initialization separately from the measured tree. +for (let i = 0; i < 20; i++) { + const warm = parser.parse( + java + ? 'class W { int f(int x, String y) { return x; } }' + : 'class W { f(x: number, y: string): number { return x; } }', + ); + const node = warm.rootNode.descendantsOfType(nodeType)[0]; + arity(node); + buildMethodProps(extractor.extractFromNode(node, context)); +} +const emptyHeap = await heap(); +let tree = parser.parse(source); +assert(!tree.rootNode.hasError); +let nodes = tree.rootNode.descendantsOfType(nodeType); +assert.equal(nodes.length, count); +const treeHeap = await heap(); +function extract(declarations) { + const methods = []; + const emitted = []; + const start = performance.now(); + for (const node of declarations) { + const scope = arity(node); + const method = extractor.extractFromNode(node, context); + assert(method); + methods.push(method); + emitted.push({ scope, graph: buildMethodProps(method) }); + } + return { methods, emitted, extractionMs: performance.now() - start }; +} +let { methods, emitted, extractionMs } = extract(nodes); +const liveHeap = await heap(); +assert.equal(methods.length, count); // Keep the method records live through the heap sample. +const digest = createHash('sha256').update(JSON.stringify(emitted)).digest('hex'); +methods = null; +const emittedWithTreeHeap = await heap(); +const weakTree = new WeakRef(tree); +nodes = null; +tree = null; +const emittedHeap = await heap(); +const treeCollected = weakTree.deref() === undefined; +emitted = null; +const releasedHeap = await heap(); +console.log( + JSON.stringify({ + language, + declarations: count, + digest, + extractionMs, + treeCollected, + heapBytes: { emptyHeap, treeHeap, liveHeap, emittedWithTreeHeap, emittedHeap, releasedHeap }, + extractionRetainedBytes: liveHeap - treeHeap, + emittedRetainedBytes: emittedHeap - emptyHeap, + releasedDeltaBytes: releasedHeap - emptyHeap, + processMaxRssMiB: process.resourceUsage().maxRSS / 1024, + note: 'Fresh process; forced-GC JS heap, not native or worker heap. Extraction time excludes parse/GC. Peak RSS is for this probe only. Compare repeated same-runtime runs.', + }), +); diff --git a/gitnexus/bench/method-declaration-facts/scaling.mjs b/gitnexus/bench/method-declaration-facts/scaling.mjs new file mode 100644 index 000000000..6afbf77fc --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/scaling.mjs @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; + +export function evaluateScaling(rows, metrics) { + const budget = 1.5; + assert(rows.length >= 2 && metrics.length > 0, 'Scaling needs sizes and metrics'); + for (const [index, row] of rows.entries()) { + assert(Number.isFinite(row.size) && row.size > 0, 'Invalid workload size'); + if (index) assert(row.size > rows[index - 1].size, 'Sizes must increase'); + for (const metric of metrics) + assert(Number.isFinite(row[metric]) && row[metric] > 0, `Invalid ${metric}`); + } + const pairs = rows.slice(1).map((row, index) => [rows[index], row]); + if (rows.length > 2) pairs.push([rows[0], rows.at(-1)]); + const comparisons = pairs.map(([small, large]) => { + const inputRatio = large.size / small.size; + const ratios = Object.fromEntries( + metrics.map((metric) => [metric, large[metric] / small[metric] / inputRatio]), + ); + return { + from: small.size, + to: large.size, + inputRatio, + ratios, + passed: Object.values(ratios).every((ratio) => ratio <= budget), + }; + }); + return { budget, comparisons, passed: comparisons.every((row) => row.passed) }; +} diff --git a/gitnexus/bench/method-declaration-facts/trace-parameters.mjs b/gitnexus/bench/method-declaration-facts/trace-parameters.mjs new file mode 100644 index 000000000..2d62abb07 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/trace-parameters.mjs @@ -0,0 +1,47 @@ +// Optional instrumentation preload; discard timings from traced runs. +import assert from 'node:assert/strict'; +import { appendFileSync, writeFileSync } from 'node:fs'; +import { threadId } from 'node:worker_threads'; +import { pathToFileURL } from 'node:url'; +const root = process.env.FACT_RUNTIME_ROOT; +const target = process.env.FACT_COUNTER_LOG; +assert(root && target, 'Set FACT_RUNTIME_ROOT and FACT_COUNTER_LOG for this preload'); +if (threadId === 0) writeFileSync(target, '', { flag: 'wx' }); +const trees = new WeakMap(), + arrays = new WeakMap(); +let treeSerial = 0, + arraySerial = 0; +for (const [module, prefixes] of [ + ['jvm', ['java', 'kotlin']], + ['typescript-javascript', ['typescript', 'javascript']], + ['csharp', ['csharp']], + ['python', ['python']], + ['php', ['php']], + ['swift', ['swift']], + ['dart', ['dart']], +]) { + const exports = await import( + pathToFileURL(root + '/dist/core/ingestion/method-extractors/configs/' + module + '.js') + ); + for (const prefix of prefixes) { + const config = exports[prefix + 'MethodConfig']; + const original = config.extractParameters; + config.extractParameters = function (node, ...args) { + const result = original(node, ...args); + if (node.tree && !trees.has(node.tree)) trees.set(node.tree, ++treeSerial); + if (!arrays.has(result)) arrays.set(result, ++arraySerial); + appendFileSync( + target, + JSON.stringify({ + thread: threadId, + provider: prefix, + tree: trees.get(node.tree), + node: node.id, + kind: node.type, + array: arrays.get(result), + }) + '\n', + ); + return result; + }; + } +} diff --git a/gitnexus/src/core/analyzer-identity.ts b/gitnexus/src/core/analyzer-identity.ts index 5782a5724..9bd3902a1 100644 --- a/gitnexus/src/core/analyzer-identity.ts +++ b/gitnexus/src/core/analyzer-identity.ts @@ -2241,8 +2241,13 @@ function loadIdentityCache( } const CACHE_GUARD_PROBE_SCRIPT = String.raw` -const fs = require('node:fs/promises'); +const fs = require('node:fs'); +const { performance } = require('node:perf_hooks'); const crypto = require('node:crypto'); +const syncFs = { + lstat: fs.lstatSync, stat: fs.statSync, + readlink: fs.readlinkSync, readdir: fs.readdirSync, +}; const state = (value) => ({ dev: String(value.dev), ino: String(value.ino), mode: String(value.mode), nlink: String(value.nlink), size: String(value.size), @@ -2272,30 +2277,30 @@ const inventoryDigest = (entries) => { for (const entry of normalized) frame(hash, [entry.name, entry.kind]); return 'sha256:' + hash.digest('hex'); }; -const probe = async (request) => { +const probe = async (request, io) => { try { - const link = await fs.lstat(request.absolutePath, { bigint: true }); + const link = await io.lstat(request.absolutePath, { bigint: true }); const type = link.isDirectory() ? 'directory' : link.isFile() ? 'file' : link.isSymbolicLink() ? 'symlink' : 'other'; if (request.mode === 'link') return { type, state: state(link), - ...(type === 'symlink' ? { symlinkTarget: await fs.readlink(request.absolutePath) } : {}), + ...(type === 'symlink' ? { symlinkTarget: await io.readlink(request.absolutePath) } : {}), }; if (request.mode === 'directory-inventory') { if (!link.isDirectory() || link.isSymbolicLink()) return null; const entriesDigest = inventoryDigest( - await fs.readdir(request.absolutePath, { withFileTypes: true }), + await io.readdir(request.absolutePath, { withFileTypes: true }), ); - const after = await fs.lstat(request.absolutePath, { bigint: true }); + const after = await io.lstat(request.absolutePath, { bigint: true }); if (JSON.stringify(state(link)) !== JSON.stringify(state(after))) return null; return { type: 'directory-inventory', state: state(after), entriesDigest }; } - const target = link.isFile() ? link : await fs.stat(request.absolutePath, { bigint: true }); + const target = link.isFile() ? link : await io.stat(request.absolutePath, { bigint: true }); if (!target.isFile()) return null; const result = { type: 'readable-file', state: { link: state(link), target: state(target) } }; - if (link.isSymbolicLink()) result.state.symlinkTarget = await fs.readlink(request.absolutePath); + if (link.isSymbolicLink()) result.state.symlinkTarget = await io.readlink(request.absolutePath); return result; } catch { return null; } }; @@ -2305,9 +2310,29 @@ process.stdin.on('data', (chunk) => { input += chunk; }); process.stdin.on('end', async () => { try { const requests = JSON.parse(input); - const results = []; - for (let offset = 0; offset < requests.length; offset += 512) { - results.push(...await Promise.all(requests.slice(offset, offset + 512).map(probe))); + const results = new Array(requests.length); + // Local stats are cheaper without thread-pool dispatch. Sample across the + // install tree so slow dependency mounts still use concurrent async I/O. + const sampleCount = Math.min(16, requests.length); + const started = performance.now(); + for (let sample = 0; sample < sampleCount; sample += 1) { + const index = Math.floor(sample * requests.length / sampleCount); + results[index] = await probe(requests[index], syncFs); + if (performance.now() - started >= 8) break; + } + const useSync = performance.now() - started < 8; + if (useSync) { + for (let index = 0; index < requests.length; index += 1) { + if (results[index] === undefined) results[index] = await probe(requests[index], syncFs); + } + } else { + for (let offset = 0; offset < requests.length; offset += 512) { + await Promise.all(requests.slice(offset, offset + 512).map(async (request, index) => { + if (results[offset + index] === undefined) { + results[offset + index] = await probe(request, fs.promises); + } + })); + } } process.stdout.write(JSON.stringify(results)); } catch { process.exitCode = 1; } diff --git a/gitnexus/src/core/ingestion/export-detection.ts b/gitnexus/src/core/ingestion/export-detection.ts index 143358c6f..c436e5d23 100644 --- a/gitnexus/src/core/ingestion/export-detection.ts +++ b/gitnexus/src/core/ingestion/export-detection.ts @@ -41,25 +41,33 @@ export const tsExportChecker: ExportChecker = (node, _name) => { /** Python: public if no leading underscore (convention). */ export const pythonExportChecker: ExportChecker = (_node, name) => !name.startsWith('_'); +// Parser trees are immutable during extraction; weak keys do not retain discarded trees. +const javaPublicModifierCache = new WeakMap(); + /** Java: check for 'public' modifier — modifiers are siblings of the name node, not parents. */ export const javaExportChecker: ExportChecker = (node, _name) => { let current: SyntaxNode | null = node; while (current) { - if (current.parent) { - const parent = current.parent; - for (let i = 0; i < parent.childCount; i++) { - const child = parent.child(i); - if (child?.type === 'modifiers' && child.text?.includes('public')) { - return true; + const parent: SyntaxNode | null = current.parent; + if (parent) { + let isPublic = javaPublicModifierCache.get(parent); + if (isPublic === undefined) { + isPublic = false; + for (let i = 0; i < parent.childCount; i++) { + const child = parent.child(i); + if (child?.type === 'modifiers' && child.text?.includes('public')) { + isPublic = true; + break; + } } - } - if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') { - if (parent.text?.trimStart().startsWith('public')) { - return true; + if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') { + isPublic ||= parent.text?.trimStart().startsWith('public') ?? false; } + javaPublicModifierCache.set(parent, isPublic); } + if (isPublic) return true; } - current = current.parent; + current = parent; } return false; }; diff --git a/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts index 92b377352..20a63b375 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts @@ -19,6 +19,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { csharpMethodConfig } from '../../method-extractors/configs/csharp.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface CsharpArityMetadata { readonly parameterCount: number | undefined; @@ -31,13 +32,12 @@ export function computeCsharpArityMetadata(fnNode: SyntaxNode): CsharpArityMetad let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + let types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) hasVariadic = true; else if (p.isOptional) optionalCount++; - if (p.type !== null) types.push(p.type); } - if (hasVariadic) types.push('params'); + if (hasVariadic) types = [...types, 'params']; const total = params.length; // `params int[] args` declares one formal param but accepts any arg diff --git a/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts index 28ab666c7..6e569f6a8 100644 --- a/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts @@ -12,6 +12,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { dartMethodConfig } from '../../method-extractors/configs/dart.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface DartArityMetadata { parameterCount: number | undefined; @@ -29,10 +30,9 @@ export function computeDartArityMetadata(fnNode: SyntaxNode): DartArityMetadata const params = dartMethodConfig.extractParameters?.(fnNode) ?? []; let optionalCount = 0; - const types: string[] = []; + const types = declarationParameterTypes(params); for (const p of params) { if (p.isOptional) optionalCount++; - if (p.type !== null) types.push(p.type); } const total = params.length; diff --git a/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts index 47cccbff9..75a92a48a 100644 --- a/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts @@ -12,6 +12,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { javaMethodConfig } from '../../method-extractors/configs/jvm.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; export interface JavaArityMetadata { readonly parameterCount: number | undefined; @@ -23,12 +24,11 @@ export function computeJavaArityMetadata(fnNode: SyntaxNode): JavaArityMetadata const params = javaMethodConfig.extractParameters?.(fnNode) ?? []; let hasVariadic = false; - const types: string[] = []; + let types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) hasVariadic = true; - if (p.type !== null) types.push(p.type); } - if (hasVariadic) types.push('varargs'); + if (hasVariadic) types = [...types, 'varargs']; const total = params.length; // For varargs methods, `parameterCount` (max) is unknown — any number of diff --git a/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts index 5cdf76cf6..6f54ef673 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts @@ -1,5 +1,6 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { kotlinMethodConfig } from '../../method-extractors/configs/jvm.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; export interface KotlinArityMetadata { readonly parameterCount: number | undefined; @@ -10,12 +11,11 @@ export interface KotlinArityMetadata { export function computeKotlinArityMetadata(fnNode: SyntaxNode): KotlinArityMetadata { const params = kotlinMethodConfig.extractParameters?.(fnNode) ?? []; let hasVararg = false; - const parameterTypes: string[] = []; + let parameterTypes = declarationParameterTypes(params); for (const param of params) { if (param.isVariadic) hasVararg = true; - if (param.type !== null) parameterTypes.push(param.type); } - if (hasVararg) parameterTypes.push('vararg'); + if (hasVararg) parameterTypes = [...parameterTypes, 'vararg']; const required = params.filter((p) => !p.isOptional && !p.isVariadic).length; return { diff --git a/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts index 40662bd61..95506884b 100644 --- a/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts @@ -21,6 +21,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { phpMethodConfig } from '../../method-extractors/configs/php.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface PhpArityMetadata { readonly parameterCount: number | undefined; @@ -33,7 +34,7 @@ export function computePhpArityMetadata(fnNode: SyntaxNode): PhpArityMetadata { let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + let types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) { @@ -41,7 +42,6 @@ export function computePhpArityMetadata(fnNode: SyntaxNode): PhpArityMetadata { } else if (p.isOptional) { optionalCount++; } - if (p.type !== null) types.push(p.type); } // PHP variadic marker convention: append the literal '...' string to // `parameterTypes`. This is intentionally DIFFERENT from C#, which uses @@ -54,7 +54,7 @@ export function computePhpArityMetadata(fnNode: SyntaxNode): PhpArityMetadata { // arity logic lives in `phpArityCompatibility` (arity.ts) and now // also in `phpEmitUnresolvedReceiverEdges` (scope-resolver.ts), both // of which check `'...'`. Finding 9 of PR #1497 adversarial review. - if (hasVariadic) types.push('...'); + if (hasVariadic) types = [...types, '...']; const total = params.length; // Variadic methods accept any arg count ≥ required — leave `parameterCount` diff --git a/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts index 6d5e38af2..5168a537f 100644 --- a/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts @@ -18,6 +18,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { pythonMethodConfig } from '../../method-extractors/configs/python.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface PythonArityMetadata { readonly parameterCount: number | undefined; @@ -31,11 +32,10 @@ export function computePythonArityMetadata(fnNode: SyntaxNode): PythonArityMetad let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + const types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) hasVariadic = true; else if (p.isOptional) optionalCount++; - if (p.type !== null) types.push(p.type); } const total = params.length; diff --git a/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts index da3efaa61..527b3b7a0 100644 --- a/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts @@ -18,6 +18,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { swiftMethodConfig } from '../../method-extractors/configs/swift.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface SwiftArityMetadata { readonly parameterCount: number | undefined; @@ -30,13 +31,17 @@ export function computeSwiftArityMetadata(fnNode: SyntaxNode): SwiftArityMetadat let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + const sharedTypes = declarationParameterTypes(params); + // Scope signatures keep unknown positional slots; graph metadata omits them. + let types = + sharedTypes.length === params.length + ? sharedTypes + : params.map((parameter) => parameter.type ?? ''); for (const p of params) { if (p.isVariadic) hasVariadic = true; else if (p.isOptional) optionalCount++; - types.push(p.type ?? ''); } - if (hasVariadic) types.push('variadic'); + if (hasVariadic) types = [...types, 'variadic']; const total = params.length; const parameterCount = hasVariadic ? undefined : total; diff --git a/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts index 97ac7a0c9..3340efb1c 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts @@ -52,6 +52,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { typescriptMethodConfig } from '../../method-extractors/configs/typescript-javascript.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface TsArityMetadata { readonly parameterCount: number | undefined; @@ -64,14 +65,21 @@ export function computeTsArityMetadata(fnNode: SyntaxNode): TsArityMetadata { let hasRest = false; let optionalCount = 0; - const types: string[] = []; + const sharedTypes = declarationParameterTypes(params); + let normalized: string[] | undefined; + let index = 0; for (const p of params) { if (p.isVariadic) hasRest = true; else if (p.isOptional) optionalCount++; const t = p.type !== null && p.type !== undefined ? stripGenericsAndArraySuffix(p.type) : ''; - types.push(t); + if (t !== sharedTypes[index]) { + normalized ??= sharedTypes.slice(); + normalized[index] = t; + } + index++; } - if (hasRest) types.push('params'); + const types = normalized ?? sharedTypes; + const markedTypes = hasRest ? [...types, 'params'] : types; const total = params.length; const parameterCount = hasRest ? undefined : total; @@ -82,7 +90,7 @@ export function computeTsArityMetadata(fnNode: SyntaxNode): TsArityMetadata { // registry without aiding narrowing — callers treat absence as // "types unknown". const hasAnyType = types.some((t) => t !== '' && t !== 'params'); - const parameterTypes = hasAnyType || hasRest ? (types.length > 0 ? types : undefined) : undefined; + const parameterTypes = hasAnyType || hasRest ? markedTypes : undefined; return { parameterCount, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts index c6550050f..b3a29b255 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts @@ -16,6 +16,7 @@ import { collectModifierTexts, } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; // --------------------------------------------------------------------------- @@ -234,7 +235,7 @@ export const csharpMethodConfig: MethodExtractionConfig = { return undefined; }, - extractParameters: extractCSharpParameters, + extractParameters: memoizeDeclarationFact(extractCSharpParameters), extractVisibility(node) { // Detect compound C# visibilities: protected internal, private protected diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts index af71c24bd..6afad2fc2 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts @@ -9,6 +9,7 @@ import type { } from '../../method-types.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Dart helpers @@ -415,9 +416,9 @@ export const dartMethodConfig: MethodExtractionConfig = { methodNodeTypes: ['method_signature', 'declaration'], bodyNodeTypes: ['class_body', 'extension_body'], - extractName: extractDartName, - extractReturnType: extractDartReturnType, - extractParameters: extractDartParameters, + extractName: memoizeDeclarationFact(extractDartName, getInnerSignature), + extractReturnType: memoizeDeclarationFact(extractDartReturnType, getInnerSignature), + extractParameters: memoizeDeclarationFact(extractDartParameters, getInnerSignature), extractVisibility: extractDartVisibility, isStatic: isDartStatic, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts index 44d65f4b3..080996f52 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts @@ -13,6 +13,7 @@ import { } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Shared JVM helpers @@ -131,7 +132,7 @@ export const javaMethodConfig: MethodExtractionConfig = { extractReturnType: extractReturnTypeFromField, - extractParameters: extractJavaParameters, + extractParameters: memoizeDeclarationFact(extractJavaParameters), extractVisibility(node) { return findVisibility(node, JAVA_VIS, 'package', 'modifiers'); @@ -289,7 +290,7 @@ export const kotlinMethodConfig: MethodExtractionConfig = { extractReturnType: extractKotlinReturnType, - extractParameters: extractKotlinParameters, + extractParameters: memoizeDeclarationFact(extractKotlinParameters), extractVisibility(node) { return findVisibility(node, KOTLIN_VIS, 'public', 'modifiers'); diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/php.ts b/gitnexus/src/core/ingestion/method-extractors/configs/php.ts index 2d04a6bb3..69f3de29b 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/php.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/php.ts @@ -9,6 +9,7 @@ import type { } from '../../method-types.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // PHP helpers @@ -282,7 +283,7 @@ export const phpMethodConfig: MethodExtractionConfig = { extractReturnType: extractPhpReturnType, - extractParameters: extractPhpParameters, + extractParameters: memoizeDeclarationFact(extractPhpParameters), extractVisibility: findPhpVisibility, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/python.ts b/gitnexus/src/core/ingestion/method-extractors/configs/python.ts index 0540956b1..2a6ace071 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/python.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/python.ts @@ -15,6 +15,7 @@ import { } from '../../languages/python/receiver-binding.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Python helpers @@ -300,7 +301,7 @@ export const pythonMethodConfig: MethodExtractionConfig = { }, extractReturnType: extractPythonReturnType, - extractParameters: extractPythonParameters, + extractParameters: memoizeDeclarationFact(extractPythonParameters, unwrapDecorated), extractVisibility: extractPythonVisibility, isStatic(node) { diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts index 1ee630b6a..12d783ad9 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts @@ -10,6 +10,7 @@ import type { import { findVisibility, hasKeyword, hasModifier } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Swift helpers @@ -282,9 +283,9 @@ export const swiftMethodConfig: MethodExtractionConfig = { // dev-mode "body field type not in bodyNodeTypes" warning. bodyNodeTypes: ['class_body', 'protocol_body', 'enum_class_body'], - extractName: extractSwiftName, - extractReturnType: extractSwiftReturnType, - extractParameters: extractSwiftParameters, + extractName: memoizeDeclarationFact(extractSwiftName), + extractReturnType: memoizeDeclarationFact(extractSwiftReturnType), + extractParameters: memoizeDeclarationFact(extractSwiftParameters), extractVisibility(node) { return findVisibility(node, SWIFT_VIS, 'internal', 'modifiers'); diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts index b57ad110a..b80f68613 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts @@ -10,6 +10,7 @@ import type { import { hasKeyword } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // TS/JS helpers @@ -308,7 +309,7 @@ const shared: Omit = { }, extractReturnType: extractTsJsReturnType, - extractParameters: extractTsJsParameters, + extractParameters: memoizeDeclarationFact(extractTsJsParameters), extractVisibility: extractTsJsVisibility, isStatic(node) { diff --git a/gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts b/gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts new file mode 100644 index 000000000..393dd0ea7 --- /dev/null +++ b/gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts @@ -0,0 +1,61 @@ +import type { ParameterInfo } from '../method-types.js'; +import type { SyntaxNode } from '../utils/ast-helpers.js'; + +type DeclarationFact = string | undefined | readonly Readonly[]; + +const parameterTypes = new WeakMap[], readonly string[]>(); +const noTypes: readonly string[] = []; + +/** The common type projection of immutable declaration parameters. Consumers + * with positional unknowns, normalized types, or variadic markers copy only + * when their language policy differs. Neither keys nor values retain an AST. + */ +export function declarationParameterTypes( + parameters: readonly Readonly[], +): readonly string[] { + if (parameters.length === 0) return noTypes; + const cached = parameterTypes.get(parameters); + if (cached) return cached; + const types: string[] = []; + for (const parameter of parameters) { + if (parameter.type !== null) types.push(parameter.type); + } + if (types.length === 0) return noTypes; + parameterTypes.set(parameters, types); + return types; +} + +/** Share a pure, node-only fact across structure and scope extraction. + * Trees are immutable during ingestion. Native node IDs survive different JS + * wrappers; tree ownership keeps IDs from colliding across files or reparses. + * Values must contain no AST nodes. File/owner-dependent callbacks stay uncached. + * Providers may identify an equivalent declaration inside the same tree when + * one consumer sees a syntax wrapper and another sees its inner declaration. + */ +export function memoizeDeclarationFact( + extract: (node: SyntaxNode) => T, + declaration?: (node: SyntaxNode) => SyntaxNode | null, +): (node: SyntaxNode) => T { + const trees = new WeakMap>(); + return (node) => { + const tree = node.tree; + const id = node.id; + if (!tree || typeof id !== 'number') return extract(node); + let facts = trees.get(tree); + if (facts?.has(id)) return facts.get(id)!; + const declarationId = declaration?.(node)?.id ?? id; + if (declarationId !== id && facts?.has(declarationId)) { + const fact = facts.get(declarationId)!; + facts.set(id, fact); + return fact; + } + const fact = extract(node); + if (!facts) { + facts = new Map(); + trees.set(tree, facts); + } + facts.set(id, fact); + if (declarationId !== id) facts.set(declarationId, fact); + return fact; + }; +} diff --git a/gitnexus/src/core/ingestion/method-types.ts b/gitnexus/src/core/ingestion/method-types.ts index fb7c61f1f..3e9edf2e9 100644 --- a/gitnexus/src/core/ingestion/method-types.ts +++ b/gitnexus/src/core/ingestion/method-types.ts @@ -23,7 +23,7 @@ export interface MethodInfo { name: string; receiverType: string | null; returnType: string | null; - parameters: ParameterInfo[]; + parameters: readonly Readonly[]; visibility: MethodVisibility; isStatic: boolean; isAbstract: boolean; @@ -100,7 +100,7 @@ export interface MethodExtractionConfig { * name is the file stem, so `fn incr(counter: *Counter)` in `Counter.zig` * is a method only when the file is known. Same optional-trailing-argument * shape as `extractOwnerName`; every other config ignores it. */ - extractParameters: (node: SyntaxNode, filePath?: string) => ParameterInfo[]; + extractParameters: (node: SyntaxNode, filePath?: string) => readonly Readonly[]; extractVisibility: (node: SyntaxNode) => MethodVisibility; isStatic: (node: SyntaxNode, filePath?: string) => boolean; isAbstract: (node: SyntaxNode, ownerNode: SyntaxNode) => boolean; diff --git a/gitnexus/src/core/ingestion/utils/method-props.ts b/gitnexus/src/core/ingestion/utils/method-props.ts index 4ade74c8a..fcf913059 100644 --- a/gitnexus/src/core/ingestion/utils/method-props.ts +++ b/gitnexus/src/core/ingestion/utils/method-props.ts @@ -1,4 +1,5 @@ import type { MethodInfo } from '../method-types.js'; +import { declarationParameterTypes } from '../method-extractors/declaration-facts.js'; import { SupportedLanguages, type ParameterTypeClass } from 'gitnexus-shared'; /** Languages where class overload signatures are declaration-only contracts @@ -210,12 +211,11 @@ export function parameterShapeIdTag( /** Convert MethodInfo from methodExtractor into flat properties for a graph node. */ export function buildMethodProps(info: MethodInfo): Record { - const types: string[] = []; + const types = declarationParameterTypes(info.parameters); const typeClasses: ParameterTypeClass[] = []; let optionalCount = 0; let hasVariadic = false; for (const p of info.parameters) { - if (p.type !== null) types.push(p.type); if (p.typeClass !== undefined) typeClasses.push(p.typeClass); if (p.isOptional) optionalCount++; if (p.isVariadic) hasVariadic = true; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 5d999ed24..fb0a43b2b 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -871,6 +871,20 @@ function selectFieldInfo( // ============================================================================ const methodInfoCache = new Map>(); +const methodCollisionGroups = new WeakMap, Map>(); + +/** Method maps are complete before publication and immutable for this file. + * Reuse their overload groups in both definition and enclosing-callable IDs. + * Weak ownership releases the groups with the per-file method cache. + */ +function getMethodCollisionGroups(methods: Map): Map { + let groups = methodCollisionGroups.get(methods); + if (!groups) { + groups = buildCollisionGroups(methods); + methodCollisionGroups.set(methods, groups); + } + return groups; +} /** * Get (or extract and cache) method info for a class node. @@ -1183,7 +1197,7 @@ const findEnclosingFunctionId = ( ? undefined : info.parameters.length; if (methodMap && arity !== undefined) { - const g = buildCollisionGroups(methodMap); + const g = getMethodCollisionGroups(methodMap); encTypeTag = typeTagForId(methodMap, funcName, arity, info, encLang, g) + constTagForId(methodMap, funcName, arity, info, g); @@ -1251,7 +1265,7 @@ const findEnclosingFunctionId = ( ? undefined : info2.parameters.length; if (methodMap2 && arity2 !== undefined) { - const g2 = buildCollisionGroups(methodMap2); + const g2 = getMethodCollisionGroups(methodMap2); encTypeTag2 = typeTagForId(methodMap2, customResult.funcName, arity2, info2, encLang2, g2) + constTagForId(methodMap2, customResult.funcName, arity2, info2, g2); @@ -2861,7 +2875,7 @@ const processFileGroup = ( const parameterTypes = methodProps.parameterTypes as string[] | undefined; if (parameterTypes?.length) arityTag += `~${parameterTypes.join(',')}`; } else if (arityTag && defMethodMap && defMethodInfo) { - const groups = buildCollisionGroups(defMethodMap); + const groups = getMethodCollisionGroups(defMethodMap); arityTag += typeTagForId( defMethodMap, nodeName, diff --git a/gitnexus/test/integration/parsing.test.ts b/gitnexus/test/integration/parsing.test.ts index f7295b759..c1d55a345 100644 --- a/gitnexus/test/integration/parsing.test.ts +++ b/gitnexus/test/integration/parsing.test.ts @@ -6,7 +6,7 @@ * PHP export detection (#20), symbol ID with startLine (#19), * definition node range (#22). */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, assert } from 'vitest'; import fs from 'fs/promises'; import path from 'path'; import { loadParser, loadLanguage } from '../../src/core/tree-sitter/parser-loader.js'; @@ -93,6 +93,52 @@ describe('parsing', () => { }); }); + describe('java', () => { + it('keeps sibling visibility independent when a parent check is reused', async () => { + const parser = await loadParser(); + await loadLanguage(SupportedLanguages.Java); + const tree = parser.parse('class C { private void a() {} public void b() {} void c() {} }'); + const methods = tree.rootNode.descendantsOfType('method_declaration'); + for (const order of [methods, [...methods].reverse()]) { + expect( + order.map((method) => { + const name = method.childForFieldName('name'); + assert(name); + return isNodeExported(name, name.text, 'java'); + }), + ).toEqual([false, true, false]); + } + }); + + it('preserves public-ancestor behavior across repeated checks and separate trees', async () => { + const parser = await loadParser(); + await loadLanguage(SupportedLanguages.Java); + for (const modifier of ['', 'public ', '']) { + const tree = parser.parse( + `${modifier}class C { private void a() {} class Inner { void b() {} } }`, + ); + const methods = tree.rootNode.descendantsOfType('method_declaration'); + expect(methods).toHaveLength(2); + for (const method of methods) { + const name = method.childForFieldName('name'); + assert(name); + expect(isNodeExported(name, name.text, 'java')).toBe(modifier === 'public '); + expect(isNodeExported(name, name.text, 'java')).toBe(modifier === 'public '); + } + } + }); + + it.each(['method_declaration', 'constructor_declaration'])( + 'preserves the %s text fallback', + (type) => { + const name = mockNode('identifier', 'C'); + mockNode(type, ' public C() {}', undefined, [name]); + expect(isNodeExported(name, 'C', 'java')).toBe(true); + expect(isNodeExported(name, 'C', 'java')).toBe(true); + }, + ); + }); + // Go describe('go', () => { it('uppercase first letter is exported', () => { diff --git a/gitnexus/test/unit/analyzer-identity-guard-io.test.ts b/gitnexus/test/unit/analyzer-identity-guard-io.test.ts new file mode 100644 index 000000000..f89622f5e --- /dev/null +++ b/gitnexus/test/unit/analyzer-identity-guard-io.test.ts @@ -0,0 +1,189 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { mkdir, symlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; + +const captured = vi.hoisted(() => ({ script: '' })); +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + spawnSync: (...args: Parameters) => { + const argv = args[1]; + if (Array.isArray(argv) && argv.includes('--input-type=commonjs')) { + captured.script = argv[argv.indexOf('-e') + 1]; + // Let identity resolution fall back to the real direct validator. + return { status: 1, stdout: '', stderr: '' }; + } + return actual.spawnSync(...args); + }, + }; +}); + +import { + _clearAnalyzerIdentityProcessCacheForTests, + resolveAnalyzerRunnerIdentity, +} from '../../src/core/analyzer-identity.js'; + +type Request = { absolutePath: string; mode: 'link' | 'readable-file' | 'directory-inventory' }; +type Result = { + type: string; + state: unknown; + entriesDigest?: string; + symlinkTarget?: string; +} | null; + +// Exercise the actual subprocess script captured through the public resolver. +// A deterministic clock chooses the I/O route without timing-sensitive tests. +async function runProbe(requests: Request[], slow: boolean, mutateDirectory?: string) { + const { spawnSync } = + await vi.importActual('node:child_process'); + const instrumentation = ` + const probeFs = require('node:fs'); + let ticks = 0; + Object.defineProperty(require('node:perf_hooks').performance, 'now', { + value: () => ticks++ * ${slow ? 100 : 0}, + }); + const counts = { sync: 0, async: 0 }; + const mutationDirectory = ${JSON.stringify(mutateDirectory ?? null)}; + let mutated = false; + const afterRead = (file) => { + if (!mutated && file === mutationDirectory) { + mutated = true; + probeFs.writeFileSync(require('node:path').join(file, 'added'), 'changed'); + } + }; + for (const name of ['lstat', 'stat', 'readlink', 'readdir']) { + const originalSync = probeFs[name + 'Sync']; + probeFs[name + 'Sync'] = (...args) => { + counts.sync++; + const result = originalSync(...args); + if (name === 'readdir') afterRead(args[0]); + return result; + }; + const originalAsync = probeFs.promises[name]; + probeFs.promises[name] = async (...args) => { + counts.async++; + const result = await originalAsync(...args); + if (name === 'readdir') afterRead(args[0]); + return result; + }; + } + process.on('exit', () => process.stderr.write(JSON.stringify(counts))); + `; + const child = spawnSync( + process.execPath, + ['--input-type=commonjs', '-e', instrumentation + captured.script], + { + input: JSON.stringify(requests), + encoding: 'utf8', + timeout: 10_000, + }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + return { + results: JSON.parse(child.stdout) as Result[], + counts: JSON.parse(child.stderr) as { sync: number; async: number }, + }; +} + +describe('identity guard subprocess I/O', () => { + let fixture: TestDBHandle; + let file: string; + let directory: string; + + beforeAll(async () => { + fixture = await createTempDir(); + directory = path.join(fixture.dbPath, 'src'); + await mkdir(directory); + await writeFile( + path.join(fixture.dbPath, 'package.json'), + '{"name":"guard-io","version":"1.0.0"}', + ); + for (let index = 0; index < 140; index++) { + await writeFile(path.join(directory, `file-${index}.ts`), `export const value = ${index};`); + } + file = path.join(directory, 'file-0.ts'); + vi.stubEnv('GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS', '0'); + _clearAnalyzerIdentityProcessCacheForTests(); + resolveAnalyzerRunnerIdentity(pathToFileURL(file).href, { + cacheDirectory: path.join(fixture.dbPath, 'cache'), + }); + expect(captured.script).not.toBe(''); + }); + + afterAll(async () => { + vi.unstubAllEnvs(); + _clearAnalyzerIdentityProcessCacheForTests(); + await fixture?.cleanup(); + }); + + it('returns identical ordered snapshots and failures across batch boundaries on both routes', async () => { + const cases: Request[] = [ + { absolutePath: file, mode: 'link' }, + { absolutePath: directory, mode: 'directory-inventory' }, + { absolutePath: file, mode: 'readable-file' }, + { absolutePath: path.join(fixture.dbPath, 'missing'), mode: 'link' }, + { absolutePath: directory, mode: 'readable-file' }, + { absolutePath: file, mode: 'directory-inventory' }, + ]; + const requests = Array.from({ length: 529 }, (_, index) => cases[index % cases.length]); + const fast = await runProbe(requests, false); + const slow = await runProbe(requests, true); + expect(fast.counts.async).toBe(0); + expect(fast.counts.sync).toBeGreaterThan(requests.length); + // A slow first probe exhausts the sample budget; the rest stay concurrent. + expect(slow.counts.sync).toBe(1); + expect(slow.counts.async).toBeGreaterThan(512); + expect(slow.results).toEqual(fast.results); + expect(fast.results).toHaveLength(requests.length); + for (let index = 0; index < requests.length; index++) { + const result = fast.results[index]; + if (index % cases.length >= 3) expect(result).toBeNull(); + else + expect(result?.type).toBe( + ['file', 'directory-inventory', 'readable-file'][index % cases.length], + ); + } + expect(fast.results[1]?.entriesDigest).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + + it.each([false, true])('rejects an inventory changed during a probe (slow=%s)', async (slow) => { + const changing = path.join(fixture.dbPath, `changing-${slow}`); + await mkdir(changing); + const requests: Request[] = Array.from({ length: 529 }, () => ({ + absolutePath: file, + mode: 'link', + })); + // Index 1 is outside the distributed sample, exercising the selected route. + requests[1] = { absolutePath: changing, mode: 'directory-inventory' }; + const result = await runProbe(requests, slow, changing); + expect(result.results[1]).toBeNull(); + expect(result.results[0]?.type).toBe('file'); + expect(result.results[528]?.type).toBe('file'); + }); + + it.skipIf(process.platform === 'win32')( + 'preserves symlink targets and dangling-link failures on both routes', + async () => { + const link = path.join(fixture.dbPath, 'linked.ts'); + const dangling = path.join(fixture.dbPath, 'dangling.ts'); + await symlink(file, link); + await symlink(path.join(fixture.dbPath, 'absent.ts'), dangling); + const cases: Request[] = [ + { absolutePath: link, mode: 'link' }, + { absolutePath: link, mode: 'readable-file' }, + { absolutePath: dangling, mode: 'readable-file' }, + ]; + const requests = Array.from({ length: 140 }, (_, index) => cases[index % cases.length]); + const fast = await runProbe(requests, false); + const slow = await runProbe(requests, true); + expect(slow.results).toEqual(fast.results); + expect(fast.results[0]?.symlinkTarget).toBe(file); + expect(fast.results[1]?.state).toMatchObject({ symlinkTarget: file }); + expect(fast.results[2]).toBeNull(); + }, + ); +}); diff --git a/gitnexus/test/unit/declaration-parameter-types.test.ts b/gitnexus/test/unit/declaration-parameter-types.test.ts new file mode 100644 index 000000000..04bc08440 --- /dev/null +++ b/gitnexus/test/unit/declaration-parameter-types.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import TypeScript from 'tree-sitter-typescript'; +import { declarationParameterTypes } from '../../src/core/ingestion/method-extractors/declaration-facts.js'; +import { createMethodExtractor } from '../../src/core/ingestion/method-extractors/generic.js'; +import { javaMethodConfig } from '../../src/core/ingestion/method-extractors/configs/jvm.js'; +import { typescriptMethodConfig } from '../../src/core/ingestion/method-extractors/configs/typescript-javascript.js'; +import { computeJavaArityMetadata } from '../../src/core/ingestion/languages/java/arity-metadata.js'; +import { computeTsArityMetadata } from '../../src/core/ingestion/languages/typescript/arity-metadata.js'; +import { buildMethodProps } from '../../src/core/ingestion/utils/method-props.js'; + +describe('shared parameter type projection', () => { + const parameter = (type: string | null) => + Object.freeze({ name: 'p', type, isOptional: false, isVariadic: false }); + + it('shares a projection without changing immutable parameters or merging declarations', () => { + const parameters = Object.freeze([parameter('A'), parameter(null), parameter('B')]); + const types = declarationParameterTypes(parameters); + expect(types).toEqual(['A', 'B']); + expect(declarationParameterTypes(parameters)).toBe(types); + expect(declarationParameterTypes([...parameters])).not.toBe(types); + expect(parameters.map((p) => p.type)).toEqual(['A', null, 'B']); + expect(declarationParameterTypes([])).toEqual([]); + expect(declarationParameterTypes([parameter(null)])).toEqual([]); + }); + + for (const fixture of [ + { java: true, signature: 'int f(int x, String y)', scope: ['int', 'String'], shared: true }, + { + java: true, + signature: 'int f(int x, String... y)', + scope: ['int', 'String', 'varargs'], + shared: false, + }, + { + java: false, + signature: 'f(x: number, y: string): number', + scope: ['number', 'string'], + shared: true, + }, + { java: false, signature: 'f(x, y: string): number', scope: ['', 'string'], shared: false }, + { + java: false, + signature: 'f(x: string, y, z: number): number', + scope: ['string', '', 'number'], + shared: false, + }, + { + java: false, + signature: 'f(x: Box[], ...y: string[]): number', + scope: ['Box', 'string', 'params'], + shared: false, + }, + ]) { + for (const scopeFirst of [true, false]) { + it(`${fixture.signature}: preserves provider projection, scopeFirst=${scopeFirst}`, () => { + const parser = new Parser(); + parser.setLanguage(fixture.java ? Java : TypeScript.typescript); + const tree = parser.parse(`class C { ${fixture.signature} { return 0; } }`); + expect(tree.rootNode.hasError).toBe(false); + const node = tree.rootNode.descendantsOfType( + fixture.java ? 'method_declaration' : 'method_definition', + )[0]!; + const config = fixture.java ? javaMethodConfig : typescriptMethodConfig; + const arity = fixture.java ? computeJavaArityMetadata : computeTsArityMetadata; + const extractor = createMethodExtractor(config); + let scope = scopeFirst ? arity(node) : undefined; + const info = extractor.extractFromNode!(node, { + language: config.language, + filePath: 'fixture', + })!; + const graph = buildMethodProps(info); + scope ??= arity(node); + expect(scope.parameterTypes).toEqual(fixture.scope); + expect(graph.parameterTypes).toEqual( + info.parameters.flatMap((p) => (p.type === null ? [] : [p.type])), + ); + if (fixture.shared) expect(scope.parameterTypes).toBe(graph.parameterTypes); + else expect(scope.parameterTypes).not.toBe(graph.parameterTypes); + expect(arity(node)).toEqual(scope); + expect(buildMethodProps(info)).toEqual(graph); + }); + } + } +}); diff --git a/gitnexus/test/unit/method-declaration-facts.test.ts b/gitnexus/test/unit/method-declaration-facts.test.ts new file mode 100644 index 000000000..3b6576c20 --- /dev/null +++ b/gitnexus/test/unit/method-declaration-facts.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it, vi } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import TypeScript from 'tree-sitter-typescript'; +import { createMethodExtractor } from '../../src/core/ingestion/method-extractors/generic.js'; +import { + javaMethodConfig, + kotlinMethodConfig, +} from '../../src/core/ingestion/method-extractors/configs/jvm.js'; +import { + typescriptMethodConfig, + javascriptMethodConfig, +} from '../../src/core/ingestion/method-extractors/configs/typescript-javascript.js'; +import { csharpMethodConfig } from '../../src/core/ingestion/method-extractors/configs/csharp.js'; +import { pythonMethodConfig } from '../../src/core/ingestion/method-extractors/configs/python.js'; +import { phpMethodConfig } from '../../src/core/ingestion/method-extractors/configs/php.js'; +import { swiftMethodConfig } from '../../src/core/ingestion/method-extractors/configs/swift.js'; +import { dartMethodConfig } from '../../src/core/ingestion/method-extractors/configs/dart.js'; +import { computeJavaArityMetadata } from '../../src/core/ingestion/languages/java/arity-metadata.js'; +import { computeTsArityMetadata } from '../../src/core/ingestion/languages/typescript/arity-metadata.js'; +import { memoizeDeclarationFact } from '../../src/core/ingestion/method-extractors/declaration-facts.js'; +import { computeKotlinArityMetadata } from '../../src/core/ingestion/languages/kotlin/arity-metadata.js'; +import { computeCsharpArityMetadata } from '../../src/core/ingestion/languages/csharp/arity-metadata.js'; +import { computePythonArityMetadata } from '../../src/core/ingestion/languages/python/arity-metadata.js'; +import { computePhpArityMetadata } from '../../src/core/ingestion/languages/php/arity-metadata.js'; +import { computeSwiftArityMetadata } from '../../src/core/ingestion/languages/swift/arity-metadata.js'; +import { computeDartArityMetadata } from '../../src/core/ingestion/languages/dart/arity-metadata.js'; +import { + getLanguageGrammar, + isLanguageAvailable, +} from '../../src/core/tree-sitter/parser-loader.js'; +import { synthesizeSwiftSignatureBindings } from '../../src/core/ingestion/languages/swift/signature-bindings.js'; +import { synthesizeDartSignatureBindings } from '../../src/core/ingestion/languages/dart/signature-bindings.js'; + +describe('shared callable declaration facts', () => { + for (const fixture of [ + { + config: pythonMethodConfig, + code: 'class C:\n @staticmethod\n def f(x: int) -> int:\n return x\n', + owner: 'class_definition', + declaration: 'function_definition', + arity: computePythonArityMetadata, + }, + { + config: dartMethodConfig, + code: 'class C { C(int x) {} }', + owner: 'class_definition', + declaration: 'constructor_signature', + arity: computeDartArityMetadata, + }, + ]) { + for (const scopeFirst of [true, false]) { + it.skipIf(!isLanguageAvailable(fixture.config.language))( + `${fixture.config.language}: shares facts across declaration wrappers, scopeFirst=${scopeFirst}`, + () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(fixture.config.language)!); + const tree = parser.parse(fixture.code); + expect(tree.rootNode.hasError).toBe(false); + const owner = tree.rootNode.descendantsOfType(fixture.owner)[0]!; + const declaration = owner.descendantsOfType(fixture.declaration)[0]!; + const spy = vi.spyOn(fixture.config, 'extractParameters'); + try { + if (scopeFirst) fixture.arity(declaration); + const result = createMethodExtractor(fixture.config).extract(owner, { + language: fixture.config.language, + filePath: 'fixture', + })!; + if (!scopeFirst) fixture.arity(declaration); + expect(result.methods).toHaveLength(1); + expect(result.methods[0]!.parameters.map((p) => p.type)).toEqual(['int']); + expect(spy).toHaveBeenCalledTimes(2); + expect(spy.mock.results[0]!.value).toBe(spy.mock.results[1]!.value); + } finally { + spy.mockRestore(); + } + }, + ); + } + } + for (const fixture of [ + { + language: Java, + config: javaMethodConfig, + code: 'class C { int f(int x, String... rest) { return x; } }', + nodeType: 'method_declaration', + arity: computeJavaArityMetadata, + expectedTypes: ['int', 'String'], + }, + { + language: TypeScript.typescript, + config: typescriptMethodConfig, + code: 'class C { f(x: number, ...rest: string[]): number { return x; } }', + nodeType: 'method_definition', + arity: computeTsArityMetadata, + expectedTypes: ['number', 'string[]'], + }, + ]) { + for (const scopeFirst of [true, false]) { + it(`${fixture.config.language}: traverses parameters once, scopeFirst=${scopeFirst}`, () => { + const parser = new Parser(); + parser.setLanguage(fixture.language); + const tree = parser.parse(fixture.code); + const node = tree.rootNode.descendantsOfType(fixture.nodeType)[0]!; + const read = vi.spyOn(node, 'childForFieldName'); + const extractor = createMethodExtractor(fixture.config); + const structure = () => + extractor.extractFromNode!(node, { + filePath: 'fixture', + language: fixture.config.language, + }); + if (scopeFirst) fixture.arity(node); + const info = structure()!; + if (!scopeFirst) fixture.arity(node); + expect(info.name).toBe('f'); + expect(info.parameters.map((p) => p.rawType ?? p.type)).toEqual(fixture.expectedTypes); + expect(info.parameters.map((p) => p.isVariadic)).toEqual([false, true]); + expect(read.mock.calls.filter(([field]) => field === 'parameters')).toHaveLength(1); + read.mockRestore(); + }); + } + } + + for (const fixture of [ + { + config: kotlinMethodConfig, + code: 'class C {\n fun f(x: Int, y: String): Int { return x }\n}', + nodeType: 'function_declaration', + arity: computeKotlinArityMetadata, + types: ['Int', 'String'], + }, + { + config: csharpMethodConfig, + code: 'class C { public int F(int x, string y = "") { return x; } }', + nodeType: 'method_declaration', + arity: computeCsharpArityMetadata, + types: ['int', 'string'], + }, + { + config: pythonMethodConfig, + code: 'class C:\n def f(self, x: int, y: str = "") -> int:\n return x\n', + nodeType: 'function_definition', + arity: computePythonArityMetadata, + types: ['int', 'str'], + }, + { + config: phpMethodConfig, + code: ' Int { return x } }', + nodeType: 'function_declaration', + arity: computeSwiftArityMetadata, + types: ['Int', 'String'], + }, + { + config: dartMethodConfig, + code: "class C { int f(int x, [String y = '']) { return x; } }", + nodeType: 'method_signature', + arity: computeDartArityMetadata, + types: ['int', 'String'], + }, + ]) { + it.skipIf(!isLanguageAvailable(fixture.config.language))( + `${fixture.config.language}: scope and structure reuse parameter facts`, + () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(fixture.config.language)!); + const tree = parser.parse(fixture.code); + expect(tree.rootNode.hasError).toBe(false); + const node = tree.rootNode.descendantsOfType(fixture.nodeType)[0]!; + const spy = vi.spyOn(fixture.config, 'extractParameters'); + try { + const scope = fixture.arity(node); + const info = createMethodExtractor(fixture.config).extractFromNode!(node, { + language: fixture.config.language, + filePath: 'fixture', + })!; + expect(info.parameters.map((p) => p.type)).toEqual(fixture.types); + expect(scope.parameterTypes?.slice(0, 2)).toEqual(fixture.types); + expect(spy).toHaveBeenCalledTimes(2); + expect(info.parameters).toBe(spy.mock.results[0]!.value); + } finally { + spy.mockRestore(); + } + }, + ); + } + + it('shares the parameter callback through the JavaScript config copy', () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(javascriptMethodConfig.language)!); + const node = parser + .parse('class C { f(x, ...rest) {} }') + .rootNode.descendantsOfType('method_definition')[0]!; + expect(javascriptMethodConfig.extractParameters).toBe(typescriptMethodConfig.extractParameters); + expect(javascriptMethodConfig.extractParameters(node)).toBe( + typescriptMethodConfig.extractParameters(node), + ); + }); + + for (const fixture of [ + { + config: swiftMethodConfig, + code: 'class C { func f(_ x: Int) -> Int { return x } }', + nodeType: 'function_declaration', + bindings: synthesizeSwiftSignatureBindings, + }, + { + config: dartMethodConfig, + code: 'class C { int f(int x) { return x; } }', + nodeType: 'method_signature', + bindings: (node: Parser.SyntaxNode) => + synthesizeDartSignatureBindings(node, node.nextNamedSibling), + }, + ]) { + it.skipIf(!isLanguageAvailable(fixture.config.language))( + `${fixture.config.language}: signature bindings share name and return facts`, + () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(fixture.config.language)!); + const node = parser.parse(fixture.code).rootNode.descendantsOfType(fixture.nodeType)[0]!; + let reads = 0; + const counted = new Proxy(node, { + get(target, property) { + if (property !== 'tree' && property !== 'id') reads++; + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + expect(fixture.bindings(counted).length).toBeGreaterThan(0); + const before = reads; + expect(before).toBeGreaterThan(0); + expect(fixture.config.extractName(counted)).toBe('f'); + expect(fixture.config.extractReturnType(counted)).toMatch(/^[Ii]nt$/); + expect(reads).toBe(before); + }, + ); + } + + it('separates same-line declarations and reparsed trees', () => { + const parser = new Parser(); + parser.setLanguage(Java); + const first = parser.parse('class C { void f(int x) {} void f(String s) {} }'); + const second = parser.parse('class C { void f(long x) {} }'); + const [intNode, stringNode] = first.rootNode.descendantsOfType('method_declaration'); + const longNode = second.rootNode.descendantsOfType('method_declaration')[0]!; + expect(javaMethodConfig.extractParameters(intNode!)[0]!.type).toBe('int'); + expect(javaMethodConfig.extractParameters(stringNode!)[0]!.type).toBe('String'); + expect(javaMethodConfig.extractParameters(longNode)[0]!.type).toBe('long'); + }); + + it('reuses empty and absent facts across wrappers with the same native identity', () => { + const parser = new Parser(); + parser.setLanguage(Java); + const tree = parser.parse('class C { void f() {} }'); + const node = tree.rootNode.descendantsOfType('method_declaration')[0]!; + // A fresh traversal may already reuse a JS wrapper. This wrapper deliberately + // differs while preserving the actual native tree/node identity. + const wrapper = new Proxy(node, {}); + expect(wrapper).not.toBe(node); + const absent = vi.fn(() => undefined); + const empty = vi.fn(() => []); + for (const extract of [absent, empty]) { + const shared = memoizeDeclarationFact(extract); + const first = shared(node); + expect(shared(wrapper)).toBe(first); + expect(extract).toHaveBeenCalledTimes(1); + } + }); + + it('retries failures and bypasses sharing when tree identity is unavailable', () => { + const parser = new Parser(); + parser.setLanguage(Java); + const node = parser.parse('class C {}').rootNode; + const extract = vi + .fn<() => string>() + .mockImplementationOnce(() => { + throw new Error('incomplete'); + }) + .mockReturnValue('complete'); + const shared = memoizeDeclarationFact(extract); + expect(() => shared(node)).toThrow('incomplete'); + expect(shared(node)).toBe('complete'); + expect(shared(node)).toBe('complete'); + expect(extract).toHaveBeenCalledTimes(2); + const partial = {} as Parser.SyntaxNode; + shared(partial); + shared(partial); + expect(extract).toHaveBeenCalledTimes(4); + }); +}); diff --git a/gitnexus/test/unit/method-declaration-scaling.test.ts b/gitnexus/test/unit/method-declaration-scaling.test.ts new file mode 100644 index 000000000..0989cd3d2 --- /dev/null +++ b/gitnexus/test/unit/method-declaration-scaling.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +const { evaluateScaling } = await import( + new URL('../../bench/method-declaration-facts/scaling.mjs', import.meta.url).href +); + +const sizes = [640, 2560, 10240]; +const series = (cost: (size: number) => number) => + sizes.map((size) => ({ size, time: cost(size), heap: size * 1000 })); + +describe('callable benchmark scaling guard', () => { + it('accepts linear growth with fixed startup overhead', () => { + expect( + evaluateScaling( + series((n) => 500 + n), + ['time', 'heap'], + ).passed, + ).toBe(true); + }); + + it('rejects quadratic time despite linear memory', () => { + expect( + evaluateScaling( + series((n) => n * n), + ['time', 'heap'], + ).passed, + ).toBe(false); + }); + + it('rejects quadratic memory despite linear time', () => { + const rows = series((n) => n).map((row) => ({ ...row, heap: row.size ** 2 })); + expect(evaluateScaling(rows, ['time', 'heap']).passed).toBe(false); + }); + + it('checks the whole size range as well as adjacent sizes', () => { + // Each 4x step grows cost by 5.28x, but the 16x span grows it by 27.86x. + expect( + evaluateScaling( + series((n) => n ** 1.2), + ['time'], + ).passed, + ).toBe(false); + }); + + it('rejects a regression confined to the largest workload', () => { + const rows = series((n) => n); + rows[2].time *= 7; + expect(evaluateScaling(rows, ['time']).passed).toBe(false); + }); + + it.each([0, -1, NaN, Infinity, undefined])('rejects an invalid reading: %s', (time) => { + const rows = series((n) => n); + expect(() => evaluateScaling([{ ...rows[0], time }, ...rows.slice(1)], ['time'])).toThrow(); + }); + + it('rejects missing sizes or metrics', () => { + expect(() => evaluateScaling(series((n) => n).slice(0, 1), ['time'])).toThrow(); + expect(() => + evaluateScaling( + series((n) => n), + [], + ), + ).toThrow(); + }); + + it.each([0, -1, NaN, Infinity, 640])('rejects invalid or repeated sizes: %s', (size) => { + const rows = series((n) => n); + expect(() => evaluateScaling([rows[0], { ...rows[1], size }, rows[2]], ['time'])).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/method-props.test.ts b/gitnexus/test/unit/method-props.test.ts index cd1be05e0..468580ff0 100644 --- a/gitnexus/test/unit/method-props.test.ts +++ b/gitnexus/test/unit/method-props.test.ts @@ -119,7 +119,7 @@ describe('typeTagForId', () => { it('returns empty string for variadic method (arity undefined)', () => { const m = makeMethodInfo('log', [{ name: 'args', type: 'String' }]); - m.parameters[0].isVariadic = true; + m.parameters = m.parameters.map((parameter) => ({ ...parameter, isVariadic: true })); const map = buildMethodMap([m]); expect(typeTagForId(map, 'log', undefined, m)).toBe(''); @@ -204,7 +204,7 @@ describe('arityForIdFromInfo', () => { it('returns undefined for variadic methods', () => { const m = makeMethodInfo('log', [{ name: 'args', type: 'String' }]); - m.parameters[0].isVariadic = true; + m.parameters = m.parameters.map((parameter) => ({ ...parameter, isVariadic: true })); expect(arityForIdFromInfo(m)).toBeUndefined(); });