diff --git a/README.md b/README.md index af0d89549..f011447fd 100644 --- a/README.md +++ b/README.md @@ -533,6 +533,8 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | +| `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. | +| `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. | | `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | | `GITNEXUS_MCP_AUTH_TOKEN` | unset | Bearer token for the dedicated `gitnexus mcp --http` server, for a **directly reachable** `gitnexus serve` `/api/mcp` route, and for the `docker-server` / web proxy in front of one. A non-loopback dedicated MCP bind requires it; `serve` enables protocol-layer MCP auth when it is set. Behind a proxy, set the **same** value on both services: the proxy spends the edge `GITNEXUS_SERVE_AUTH_TOKEN`, then replaces `Authorization` with this token on `/api/mcp` only. | Dedicated MCP, a `serve` the client can reach directly, or a proxied deploy (Render Blueprint) where the backend runs protocol-layer MCP auth — configure it on the proxy too. | | `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 595924638..1fa987320 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -668,6 +668,8 @@ const ANALYZE_CLI_ENV_KEYS = [ 'GITNEXUS_EMBEDDING_SUB_BATCH_SIZE', 'GITNEXUS_EMBEDDING_DEVICE', 'GITNEXUS_ANALYZE_PROGRESS_ACTIVE', + 'GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS', + 'GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO', 'GITNEXUS_EMBEDDING_URL', 'GITNEXUS_EMBEDDING_MODEL', 'GITNEXUS_EMBEDDING_API_KEY', diff --git a/gitnexus/src/core/analyzer-identity.ts b/gitnexus/src/core/analyzer-identity.ts index 6bc80848f..628f27716 100644 --- a/gitnexus/src/core/analyzer-identity.ts +++ b/gitnexus/src/core/analyzer-identity.ts @@ -15,6 +15,7 @@ */ import { + accessSync, closeSync, constants as fsConstants, existsSync, @@ -38,6 +39,7 @@ import { spawnSync } from 'node:child_process'; import { isDeepStrictEqual } from 'node:util'; import os from 'node:os'; import path from 'node:path'; +import { parseTruthyEnv } from './ingestion/utils/env.js'; import { fileURLToPath } from 'node:url'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; @@ -2335,8 +2337,30 @@ function snapshotCacheGuardDirect(request: CacheGuardRequest): CacheGuardResult } } -function snapshotCacheGuards(requests: CacheGuardRequest[]): CacheGuardResult[] { +function installTreeUnwritable(packageRoot: string, buildRoot: string): boolean { + for (const dir of [packageRoot, buildRoot]) { + try { + accessSync(dir, fsConstants.W_OK); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'EACCES' || code === 'EROFS') return true; + } + } + return false; +} + +function snapshotCacheGuards( + requests: CacheGuardRequest[], + packageRoot: string, + buildRoot: string, +): CacheGuardResult[] { if (requests.length < 128) return requests.map(snapshotCacheGuardDirect); + if ( + parseTruthyEnv(process.env.GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS) || + installTreeUnwritable(packageRoot, buildRoot) + ) { + return requests.map(snapshotCacheGuardDirect); + } try { const probe = spawnSync( process.execPath, @@ -2468,7 +2492,7 @@ function validateIdentityCache( return { mode, absolutePath }; }); options.onCacheValidationPass?.({ guardCount: requests.length }); - const actual = snapshotCacheGuards(requests); + const actual = snapshotCacheGuards(requests, cache.packageRoot, cache.buildRoot); const mismatch = actual.findIndex( (result, index) => !isDeepStrictEqual(result, entries[index][1]), ); diff --git a/gitnexus/src/core/ingestion/model/field-registry.ts b/gitnexus/src/core/ingestion/model/field-registry.ts index c45fb9cb5..ce9df4982 100644 --- a/gitnexus/src/core/ingestion/model/field-registry.ts +++ b/gitnexus/src/core/ingestion/model/field-registry.ts @@ -2,9 +2,9 @@ * Field Registry * * Owner-scoped field/property index extracted from SymbolTable. - * Stores Property / Variable / Const / Static symbols keyed by - * `ownerNodeId\0fieldName` for O(1) lookup. Supports multiple defs - * under the same (owner, name) — e.g. legacy Property plus a + * Stores Property / Variable / Const / Static symbols in a nested + * `Map>` for O(1) lookup. Supports + * multiple defs under the same (owner, name) — e.g. legacy Property plus a * scope-resolution Variable reconciliation entry. */ @@ -49,13 +49,13 @@ export interface MutableFieldRegistry extends FieldRegistry { // --------------------------------------------------------------------------- export const createFieldRegistry = (): MutableFieldRegistry => { - const fieldByOwner = new Map(); + const fieldByOwner = new Map>(); const lookupAllByOwner = ( ownerNodeId: string, fieldName: string, ): readonly SymbolDefinition[] => { - return fieldByOwner.get(`${ownerNodeId}\0${fieldName}`) ?? EMPTY; + return fieldByOwner.get(ownerNodeId)?.get(fieldName) ?? EMPTY; }; const lookupFieldByOwner = ( @@ -67,12 +67,16 @@ export const createFieldRegistry = (): MutableFieldRegistry => { }; const register = (ownerNodeId: string, fieldName: string, def: SymbolDefinition): void => { - const key = `${ownerNodeId}\0${fieldName}`; - const existing = fieldByOwner.get(key); + let byName = fieldByOwner.get(ownerNodeId); + if (!byName) { + byName = new Map(); + fieldByOwner.set(ownerNodeId, byName); + } + const existing = byName.get(fieldName); if (existing) { existing.push(def); } else { - fieldByOwner.set(key, [def]); + byName.set(fieldName, [def]); } }; diff --git a/gitnexus/src/core/ingestion/model/method-registry.ts b/gitnexus/src/core/ingestion/model/method-registry.ts index 75f9834c2..f64a1d8c7 100644 --- a/gitnexus/src/core/ingestion/model/method-registry.ts +++ b/gitnexus/src/core/ingestion/model/method-registry.ts @@ -2,9 +2,9 @@ * Method Registry * * Owner-scoped method index extracted from SymbolTable. - * Stores Method/Constructor/Function-with-ownerId symbols keyed by - * `ownerNodeId\0methodName` for O(1) lookup. Supports overloads - * (array values) and arity-based filtering. + * Stores Method/Constructor/Function-with-ownerId symbols in a nested + * `Map>` for O(1) lookup. Supports + * overloads (array values) and arity-based filtering. */ import type { SymbolDefinition } from 'gitnexus-shared'; @@ -92,7 +92,7 @@ export interface MutableMethodRegistry extends MethodRegistry { // --------------------------------------------------------------------------- export const createMethodRegistry = (): MutableMethodRegistry => { - const methodByOwner = new Map(); + const methodByOwner = new Map>(); // Secondary flat-by-name index. Values are the SAME SymbolDefinition // references stored under `methodByOwner` — no copy, just a second key. // Populated in lockstep by `register()` and emptied by `clear()`. @@ -102,12 +102,15 @@ export const createMethodRegistry = (): MutableMethodRegistry => { // dedup fast-path. Monotonic: never unset except on `clear()`. let hasFunctionMethodsFlag = false; + const ownerDefs = (ownerNodeId: string, methodName: string): SymbolDefinition[] | undefined => + methodByOwner.get(ownerNodeId)?.get(methodName); + const lookupMethodByOwner = ( ownerNodeId: string, methodName: string, argCount?: number, ): SymbolDefinition | undefined => { - const defs = methodByOwner.get(`${ownerNodeId}\0${methodName}`); + const defs = ownerDefs(ownerNodeId, methodName); if (!defs || defs.length === 0) return undefined; // Arity narrowing: when an argCount is provided and there are multiple @@ -176,16 +179,20 @@ export const createMethodRegistry = (): MutableMethodRegistry => { ownerNodeId: string, methodName: string, ): readonly SymbolDefinition[] => { - return methodByOwner.get(`${ownerNodeId}\0${methodName}`) ?? EMPTY; + return ownerDefs(ownerNodeId, methodName) ?? EMPTY; }; const register = (ownerNodeId: string, methodName: string, def: SymbolDefinition): void => { - const key = `${ownerNodeId}\0${methodName}`; - const existing = methodByOwner.get(key); + let owned = methodByOwner.get(ownerNodeId); + if (!owned) { + owned = new Map(); + methodByOwner.set(ownerNodeId, owned); + } + const existing = owned.get(methodName); if (existing) { existing.push(def); } else { - methodByOwner.set(key, [def]); + owned.set(methodName, [def]); } const byName = methodsByName.get(methodName); if (byName) { diff --git a/gitnexus/src/core/ingestion/model/type-registry.ts b/gitnexus/src/core/ingestion/model/type-registry.ts index 4dc87e924..135e61dfd 100644 --- a/gitnexus/src/core/ingestion/model/type-registry.ts +++ b/gitnexus/src/core/ingestion/model/type-registry.ts @@ -70,7 +70,7 @@ export const createTypeRegistry = (): MutableTypeRegistry => { const classByName = new Map(); const classByQualifiedName = new Map(); const implByName = new Map(); - const nestedByOwner = new Map(); + const nestedByOwner = new Map>(); const lookupClassByName = (name: string): SymbolDefinition[] => { return classByName.get(name) ?? []; @@ -88,7 +88,7 @@ export const createTypeRegistry = (): MutableTypeRegistry => { ownerNodeId: string, simpleName: string, ): readonly SymbolDefinition[] => { - return nestedByOwner.get(`${ownerNodeId}\0${simpleName}`) ?? EMPTY; + return nestedByOwner.get(ownerNodeId)?.get(simpleName) ?? EMPTY; }; const registerClass = (name: string, qualifiedName: string, def: SymbolDefinition): void => { @@ -121,12 +121,16 @@ export const createTypeRegistry = (): MutableTypeRegistry => { simpleName: string, def: SymbolDefinition, ): void => { - const key = `${ownerNodeId}\0${simpleName}`; - const existing = nestedByOwner.get(key); + let byName = nestedByOwner.get(ownerNodeId); + if (!byName) { + byName = new Map(); + nestedByOwner.set(ownerNodeId, byName); + } + const existing = byName.get(simpleName); if (existing) { existing.push(def); } else { - nestedByOwner.set(key, [def]); + byName.set(simpleName, [def]); } }; diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts index d0f99de15..55becdcb6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts @@ -36,6 +36,32 @@ import { import { templateConstraintsIdTag } from '../../utils/template-arguments.js'; import { parameterShapeIdTag } from '../../utils/method-props.js'; import { definitionIdPosition } from '../utils/definition-id.js'; + +const defGraphIdMemoByLookup = new WeakMap>(); + +const isResolveDefGraphIdMemoEnabled = (): boolean => { + const raw = process.env.GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO; + if (raw === undefined || raw.trim() === '') return true; + const value = raw.trim().toLowerCase(); + return value !== '0' && value !== 'false' && value !== 'off' && value !== 'no'; +}; + +const defGraphIdMemoKey = ( + filePath: string, + def: { + nodeId?: string; + qualifiedName?: string; + type?: NodeLabel; + parameterTypes?: readonly string[]; + parameterTypeClasses?: readonly ParameterTypeClass[]; + parameterCount?: number; + templateArguments?: readonly string[]; + templateConstraints?: unknown; + namespacePrefix?: string; + }, +): string => + `${filePath}\0${def.nodeId ?? ''}\0${def.type ?? ''}\0${def.qualifiedName ?? ''}\0${def.parameterCount ?? ''}\0${(def.parameterTypes ?? []).join(',')}\0${(def.parameterTypeClasses ?? []).join(',')}\0${def.namespacePrefix ?? ''}\0${(def.templateArguments ?? []).join(',')}\0${templateConstraintsIdTag(def.templateConstraints)}`; + /** * Labels that may legitimately ANCHOR a CALLS/ACCESSES edge as the * source ("caller"). A Variable / Property can be the TARGET of an @@ -230,6 +256,38 @@ export function resolveDefGraphId( namespacePrefix?: string; }, nodeLookup: GraphNodeLookup, +): string | undefined { + if (!isResolveDefGraphIdMemoEnabled()) { + return resolveDefGraphIdUncached(filePath, def, nodeLookup); + } + const qn = def.qualifiedName; + if (qn === undefined || qn.length === 0) return undefined; + let bucket = defGraphIdMemoByLookup.get(nodeLookup); + if (bucket === undefined) { + bucket = new Map(); + defGraphIdMemoByLookup.set(nodeLookup, bucket); + } + const key = defGraphIdMemoKey(filePath, def); + if (bucket.has(key)) return bucket.get(key); + const resolved = resolveDefGraphIdUncached(filePath, def, nodeLookup); + bucket.set(key, resolved); + return resolved; +} + +function resolveDefGraphIdUncached( + filePath: string, + def: { + nodeId?: string; + qualifiedName?: string; + type?: NodeLabel; + parameterTypes?: readonly string[]; + parameterTypeClasses?: readonly ParameterTypeClass[]; + parameterCount?: number; + templateArguments?: readonly string[]; + templateConstraints?: unknown; + namespacePrefix?: string; + }, + nodeLookup: GraphNodeLookup, ): string | undefined { const qn = def.qualifiedName; if (qn === undefined || qn.length === 0) return undefined; diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts index 6a8e1ba83..4ce65c63a 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts @@ -82,6 +82,15 @@ export interface EmitCallableValueFlowInput { readonly isCallableValueTarget?: (def: SymbolDefinition) => boolean; readonly hasFileLocalCallableLinkage?: (def: SymbolDefinition) => boolean; readonly onWarn?: (warning: CallableValueFlowWarning) => void; + /** When set, skip a second `collectDeferredIndirectSites` walk. */ + readonly deferredIndirectSites?: ReadonlySet; + /** When set, skip a second `referenceSites` signature walk. */ + readonly callSignaturesBySite?: ReadonlyMap; +} + +export interface DeferredIndirectCollection { + readonly sites: ReadonlySet; + readonly callSignaturesBySite: ReadonlyMap; } /** Position key shared with the existing free/reference skip-set contract. */ @@ -100,7 +109,16 @@ export function collectDeferredIndirectSites( parsedFiles: readonly ParsedFile[], scopes?: ScopeResolutionIndexes, ): ReadonlySet { + return collectDeferredIndirectCollection(parsedFiles, scopes).sites; +} + +/** One `referenceSites` walk for deferred keys and call-signature evidence. */ +export function collectDeferredIndirectCollection( + parsedFiles: readonly ParsedFile[], + scopes?: ScopeResolutionIndexes, +): DeferredIndirectCollection { const out = new Set(); + const callSignaturesBySite = new Map(); const flowCells = new Set(); if (scopes !== undefined) { for (const parsed of parsedFiles) { @@ -113,11 +131,23 @@ export function collectDeferredIndirectSites( } } for (const parsed of parsedFiles) { - const canonical = new Set( - parsed.referenceSites - .filter((site) => site.kind === 'call') - .map((site) => callableFlowSiteKey(parsed.filePath, site.atRange)), - ); + const canonical = new Set(); + for (const site of parsed.referenceSites) { + if (site.kind !== 'call') continue; + const key = callableFlowSiteKey(parsed.filePath, site.atRange); + canonical.add(key); + const signature: CallableFlowExpectedSignature = { + ...(site.arity !== undefined ? { parameterCount: site.arity } : {}), + ...(site.argumentTypes !== undefined ? { parameterTypes: site.argumentTypes } : {}), + ...(site.argumentTypeClasses !== undefined + ? { parameterTypeClasses: site.argumentTypeClasses } + : {}), + }; + const previous = callSignaturesBySite.get(key); + if (previous === undefined || signatureEvidence(signature) > signatureEvidence(previous)) { + callSignaturesBySite.set(key, signature); + } + } for (const site of parsed.callableFlowSites ?? []) { if (site.kind !== 'invoke') continue; const key = callableFlowSiteKey(parsed.filePath, site.callSite); @@ -132,7 +162,7 @@ export function collectDeferredIndirectSites( } } } - return out; + return { sites: out, callSignaturesBySite }; } function flowCellOperand(site: CallableFlowSite): CallableFlowOperand | undefined { @@ -156,7 +186,8 @@ function flowCellOperand(site: CallableFlowSite): CallableFlowOperand | undefine export function emitCallableValueFlow(input: EmitCallableValueFlowInput): CallableValueFlowResult { const facts: FileFact[] = []; const invokes: FileInvoke[] = []; - const canonicalInvokeKeys = collectDeferredIndirectSites(input.parsedFiles, input.scopes); + const canonicalInvokeKeys = + input.deferredIndirectSites ?? collectDeferredIndirectSites(input.parsedFiles, input.scopes); let unmatchedInvokes = 0; for (const parsed of input.parsedFiles) { for (const site of parsed.callableFlowSites ?? []) { @@ -485,7 +516,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab targetIndexes, aliasesByTargetId, ); - const callSignaturesBySite = indexCallSignatures(input.parsedFiles); + const callSignaturesBySite = input.callSignaturesBySite ?? indexCallSignatures(input.parsedFiles); const dynamicCallees = new Map>(); const dynamicOverflow = new Set(); const dynamicTargetHistory = new Map>(); diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index 689780d40..f39aa4dd0 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -90,7 +90,7 @@ import { import { emitImportEdges } from '../graph-bridge/imports-to-edges.js'; import { callableFlowSiteKey, - collectDeferredIndirectSites, + collectDeferredIndirectCollection, emitCallableValueFlow, } from '../passes/callable-value-flow.js'; import type { ScopeResolver, UndecidedSatisfaction } from '../contract/scope-resolver.js'; @@ -988,7 +988,8 @@ export function runScopeResolution( // ── Phase 4: emit graph edges (LOAD-BEARING ORDER — see I1) ──────────── input.onProgress?.('linking symbols', files.length, files.length); const handledSites = new Set(preEmittedInheritanceSites); - const deferredIndirectSites = collectDeferredIndirectSites(emitParsedFiles, indexes); + const deferredIndirectCollection = collectDeferredIndirectCollection(emitParsedFiles, indexes); + const deferredIndirectSites = deferredIndirectCollection.sites; const callableArgumentSites = new Set(); if (input.pdg !== true && deferredIndirectSites.size > 0) { for (const parsed of emitParsedFiles) { @@ -1237,6 +1238,8 @@ export function runScopeResolution( collapseByCallerTarget: provider.collapseMemberCallsByCallerTarget === true, isCallableValueTarget: provider.isCallableValueTarget, hasFileLocalCallableLinkage: provider.hasFileLocalCallableLinkage, + deferredIndirectSites, + callSignaturesBySite: deferredIndirectCollection.callSignaturesBySite, onWarn: (warning) => logger.warn( warning, diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index e5043c3fe..23955682f 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -177,14 +177,22 @@ export const isBinaryContent = (content: string): boolean => { return nonPrintable / end > 0.1; }; +interface PreparedFileContent { + readonly content: string; + readonly lines: string[]; + readonly isBinary: boolean; +} + +const EMPTY_PREPARED: PreparedFileContent = { content: '', lines: [''], isBinary: false }; + /** * LRU content cache — avoids re-reading the same source file for every * symbol defined in it. Sized generously so most files stay cached during - * the single-pass node iteration. + * the single-pass node iteration. Insertion order on the Map is LRU + * (delete+set on touch). */ class FileContentCache { - private cache = new Map(); - private accessOrder: string[] = []; + private cache = new Map(); private maxSize: number; private repoPath: string; @@ -193,36 +201,36 @@ class FileContentCache { this.maxSize = maxSize; } - async get(relativePath: string): Promise { - if (!relativePath) return ''; + async get(relativePath: string): Promise { + if (!relativePath) return EMPTY_PREPARED; const cached = this.cache.get(relativePath); if (cached !== undefined) { - // Move to end of accessOrder (LRU promotion) - const idx = this.accessOrder.indexOf(relativePath); - if (idx !== -1) { - this.accessOrder.splice(idx, 1); - this.accessOrder.push(relativePath); - } + this.cache.delete(relativePath); + this.cache.set(relativePath, cached); return cached; } try { const fullPath = path.join(this.repoPath, relativePath); const content = await fs.readFile(fullPath, 'utf-8'); - this.set(relativePath, content); - return content; + const prepared: PreparedFileContent = { + content, + lines: content.split('\n'), + isBinary: isBinaryContent(content), + }; + this.set(relativePath, prepared); + return prepared; } catch { - this.set(relativePath, ''); - return ''; + this.set(relativePath, EMPTY_PREPARED); + return EMPTY_PREPARED; } } - private set(key: string, value: string) { - if (this.cache.size >= this.maxSize) { - const oldest = this.accessOrder.shift(); - if (oldest) this.cache.delete(oldest); + private set(key: string, value: PreparedFileContent) { + if (this.cache.size >= this.maxSize && !this.cache.has(key)) { + const oldest = this.cache.keys().next().value; + if (oldest !== undefined) this.cache.delete(oldest); } this.cache.set(key, value); - this.accessOrder.push(key); } } @@ -277,10 +285,11 @@ const EXACT_SYMBOL_CONTENT_LABELS = SYMBOL_NODE_LABELS; const extractContent = async (node: GraphNode, contentCache: FileContentCache): Promise => { const filePath = node.properties.filePath; - const content = await contentCache.get(filePath); + const prepared = await contentCache.get(filePath); + const content = prepared.content; if (!content) return ''; if (node.label === 'Folder') return ''; - if (isBinaryContent(content)) return '[Binary file - content not stored]'; + if (prepared.isBinary) return '[Binary file - content not stored]'; // File content is stored in full — intentionally NOT length-capped here, so // text past the old 10KB cutoff stays FTS-searchable (#2317). It is already @@ -295,7 +304,7 @@ const extractContent = async (node: GraphNode, contentCache: FileContentCache): const endLine = node.properties.endLine; if (startLine === undefined || endLine === undefined) return ''; - const lines = content.split('\n'); + const lines = prepared.lines; const exactSymbolContent = EXACT_SYMBOL_CONTENT_LABELS.has(node.label); const start = Math.max(0, exactSymbolContent ? startLine : startLine - 2); const end = Math.min(lines.length - 1, exactSymbolContent ? endLine : endLine + 2); diff --git a/gitnexus/test/unit/analyzer-identity-in-process-guards.test.ts b/gitnexus/test/unit/analyzer-identity-in-process-guards.test.ts new file mode 100644 index 000000000..7595fe24a --- /dev/null +++ b/gitnexus/test/unit/analyzer-identity-in-process-guards.test.ts @@ -0,0 +1,263 @@ +/** + * #3092 item 6 — ≥128 identity cache guards stay in a subprocess unless + * `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` is truthy or packageRoot / + * buildRoot fail `W_OK` with EACCES/EROFS. Persist/cache write failure is not + * that signal. Mutation still fail-closes on both paths. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { writeFileSync } from 'node:fs'; + +const spawnCtx = vi.hoisted(() => ({ + spawnSync: vi.fn(), +})); + +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal(); + spawnCtx.spawnSync.mockImplementation(((...args: Parameters) => + actual.spawnSync(...args)) as typeof actual.spawnSync); + return { + ...actual, + spawnSync: ((...args: Parameters) => + spawnCtx.spawnSync(...args)) as typeof actual.spawnSync, + }; +}); + +const fsCtx = vi.hoisted(() => ({ + accessSync: vi.fn(), + unwritableExact: new Set(), + wOkProbes: [] as string[], +})); + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + fsCtx.accessSync.mockImplementation((( + p: Parameters[0], + mode?: number, + ) => { + const pathStr = String(p); + if (mode === actual.constants.W_OK) fsCtx.wOkProbes.push(pathStr); + if (mode === actual.constants.W_OK && fsCtx.unwritableExact.has(pathStr)) { + const err = new Error('EACCES') as NodeJS.ErrnoException; + err.code = 'EACCES'; + throw err; + } + return actual.accessSync(p, mode); + }) as typeof actual.accessSync); + return { + ...actual, + accessSync: ((...args: Parameters) => + fsCtx.accessSync(...args)) as typeof actual.accessSync, + }; +}); + +import { + _clearAnalyzerIdentityProcessCacheForTests, + resolveAnalyzerRunnerIdentity, +} from '../../src/core/analyzer-identity.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const ENV_KEY = 'GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS'; + +const isCacheGuardSpawn = (call: unknown[]): boolean => { + const argv = call[1]; + return Array.isArray(argv) && argv.includes('--input-type=commonjs') && argv.includes('-e'); +}; + +const cacheGuardSpawnCount = (): number => + spawnCtx.spawnSync.mock.calls.filter((call) => isCacheGuardSpawn(call as unknown[])).length; + +async function seedWideBuildTree(root: string): Promise<{ + modulePath: string; + sourceRoot: string; + packageRoot: string; + mutatedPath: string; +}> { + const packageRoot = root; + const sourceRoot = path.join(root, 'src'); + const modulePath = path.join(sourceRoot, 'core', 'analyzer.ts'); + await mkdir(path.dirname(modulePath), { recursive: true }); + await writeFile( + path.join(root, 'package.json'), + '{"name":"fixture-analyzer","version":"9.8.7"}\n', + ); + await writeFile(modulePath, 'export const analyzer = 1;\n'); + for (let i = 0; i < 140; i += 1) { + await writeFile(path.join(sourceRoot, `wide-${i}.ts`), `export const n${i} = ${i};\n`); + } + return { + modulePath, + sourceRoot, + packageRoot, + mutatedPath: path.join(sourceRoot, 'wide-0.ts'), + }; +} + +describe('analyzer identity in-process cache guards (#3092)', () => { + let previousEnv: string | undefined; + + beforeEach(() => { + previousEnv = process.env[ENV_KEY]; + delete process.env[ENV_KEY]; + spawnCtx.spawnSync.mockClear(); + fsCtx.unwritableExact.clear(); + fsCtx.wOkProbes.length = 0; + _clearAnalyzerIdentityProcessCacheForTests(); + }); + + afterEach(() => { + if (previousEnv === undefined) delete process.env[ENV_KEY]; + else process.env[ENV_KEY] = previousEnv; + fsCtx.unwritableExact.clear(); + _clearAnalyzerIdentityProcessCacheForTests(); + }); + + it('uses spawnSync for ≥128 guards on a writable tree when env is unset', async () => { + const fixture = await createTempDir(); + try { + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + let guardCount = 0; + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { + cacheDirectory, + onCacheValidationPass: ({ guardCount: n }) => { + guardCount = n; + }, + }); + expect(guardCount).toBeGreaterThanOrEqual(128); + spawnCtx.spawnSync.mockClear(); + _clearAnalyzerIdentityProcessCacheForTests(); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + expect(cacheGuardSpawnCount()).toBeGreaterThan(0); + } finally { + await fixture.cleanup(); + } + }); + + it.each(['1', 'true', 'yes'])('skips spawn when env is %j', async (value) => { + const fixture = await createTempDir(); + try { + process.env[ENV_KEY] = value; + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + spawnCtx.spawnSync.mockClear(); + _clearAnalyzerIdentityProcessCacheForTests(); + const warm = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { + cacheDirectory, + }); + expect(cacheGuardSpawnCount()).toBe(0); + expect(warm.schemaVersion).toBe(4); + } finally { + await fixture.cleanup(); + } + }); + + it.each(['0', 'false', 'off', ''])('still spawns when env is %j', async (value) => { + const fixture = await createTempDir(); + try { + process.env[ENV_KEY] = value; + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + spawnCtx.spawnSync.mockClear(); + _clearAnalyzerIdentityProcessCacheForTests(); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + expect(cacheGuardSpawnCount()).toBeGreaterThan(0); + } finally { + await fixture.cleanup(); + } + }); + + it('uses in-process snapshots when packageRoot W_OK fails with EACCES', async () => { + const fixture = await createTempDir(); + try { + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + fsCtx.unwritableExact.add(tree.packageRoot); + spawnCtx.spawnSync.mockClear(); + _clearAnalyzerIdentityProcessCacheForTests(); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + expect(cacheGuardSpawnCount()).toBe(0); + } finally { + await fixture.cleanup(); + } + }); + + it('does not treat persist-cache W_OK failure as an unwritable install', async () => { + const fixture = await createTempDir(); + try { + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + fsCtx.unwritableExact.add(cacheDirectory); + spawnCtx.spawnSync.mockClear(); + fsCtx.wOkProbes.length = 0; + _clearAnalyzerIdentityProcessCacheForTests(); + resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory }); + expect(fsCtx.wOkProbes).toEqual(expect.arrayContaining([tree.packageRoot, tree.sourceRoot])); + expect(fsCtx.wOkProbes).not.toContain(cacheDirectory); + expect(cacheGuardSpawnCount()).toBeGreaterThan(0); + } finally { + await fixture.cleanup(); + } + }); + + it('fail-closes on mutation with the default spawn path', async () => { + const fixture = await createTempDir(); + try { + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + const first = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { + cacheDirectory, + }); + spawnCtx.spawnSync.mockClear(); + _clearAnalyzerIdentityProcessCacheForTests(); + let mutated = false; + const second = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { + cacheDirectory, + onCacheValidationPass: () => { + if (!mutated) { + mutated = true; + writeFileSync(tree.mutatedPath, 'export const n0 = 999;\n'); + } + }, + }); + expect(cacheGuardSpawnCount()).toBeGreaterThan(0); + expect(second.build.digest).not.toBe(first.build.digest); + } finally { + await fixture.cleanup(); + } + }); + + it('fail-closes on mutation when in-process guards are opted in', async () => { + const fixture = await createTempDir(); + try { + process.env[ENV_KEY] = '1'; + const tree = await seedWideBuildTree(fixture.dbPath); + const cacheDirectory = path.join(fixture.dbPath, 'identity-cache'); + const first = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { + cacheDirectory, + }); + spawnCtx.spawnSync.mockClear(); + _clearAnalyzerIdentityProcessCacheForTests(); + let mutated = false; + const second = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { + cacheDirectory, + onCacheValidationPass: () => { + if (!mutated) { + mutated = true; + writeFileSync(tree.mutatedPath, 'export const n0 = 1000;\n'); + } + }, + }); + expect(cacheGuardSpawnCount()).toBe(0); + expect(second.build.digest).not.toBe(first.build.digest); + } finally { + await fixture.cleanup(); + } + }); +}); diff --git a/gitnexus/test/unit/model/field-registry.test.ts b/gitnexus/test/unit/model/field-registry.test.ts index 8b2789896..eb1ebf09b 100644 --- a/gitnexus/test/unit/model/field-registry.test.ts +++ b/gitnexus/test/unit/model/field-registry.test.ts @@ -72,4 +72,10 @@ describe('FieldRegistry', () => { expect(reg.lookupFieldByOwner('class:User', 'name')?.nodeId).toBe('prop:second'); }); + + it('returns the same frozen empty array on miss', () => { + const reg = createFieldRegistry(); + expect(reg.lookupAllByOwner('class:Nope', 'x')).toBe(reg.lookupAllByOwner('class:Nope', 'y')); + expect(Object.isFrozen(reg.lookupAllByOwner('class:Nope', 'x'))).toBe(true); + }); }); diff --git a/gitnexus/test/unit/model/method-registry.test.ts b/gitnexus/test/unit/model/method-registry.test.ts index bdb534202..384f42049 100644 --- a/gitnexus/test/unit/model/method-registry.test.ts +++ b/gitnexus/test/unit/model/method-registry.test.ts @@ -372,3 +372,12 @@ describe('hasFunctionMethods flag', () => { expect(reg.hasFunctionMethods).toBe(false); }); }); + +describe('MethodRegistry — EMPTY identity', () => { + it('returns the same frozen empty array on miss', () => { + const reg = createMethodRegistry(); + expect(reg.lookupMethodByName('missing')).toBe(reg.lookupMethodByName('other')); + expect(reg.lookupAllByOwner('class:Nope', 'x')).toBe(reg.lookupAllByOwner('class:Nope', 'y')); + expect(Object.isFrozen(reg.lookupAllByOwner('class:Nope', 'x'))).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/model/type-registry.test.ts b/gitnexus/test/unit/model/type-registry.test.ts index 9eaf61c4e..fd02b6c73 100644 --- a/gitnexus/test/unit/model/type-registry.test.ts +++ b/gitnexus/test/unit/model/type-registry.test.ts @@ -144,3 +144,22 @@ describe('TypeRegistry — clear()', () => { expect(reg.lookupClassByName('User')[0].nodeId).toBe('class:second'); }); }); + +describe('TypeRegistry — nested owner EMPTY vs class miss', () => { + it('lookupAllByOwner miss returns the same frozen EMPTY', () => { + const reg = createTypeRegistry(); + expect(reg.lookupAllByOwner('class:Outer', 'Inner')).toBe( + reg.lookupAllByOwner('class:Outer', 'Other'), + ); + expect(Object.isFrozen(reg.lookupAllByOwner('class:Outer', 'Inner'))).toBe(true); + }); + + it('class/impl miss returns a fresh empty array, not frozen EMPTY', () => { + const reg = createTypeRegistry(); + const a = reg.lookupClassByName('Nope'); + const b = reg.lookupClassByName('Nope'); + expect(a).toEqual([]); + expect(a).not.toBe(b); + expect(Object.isFrozen(a)).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts b/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts index 3a22dac63..9c0ffcea1 100644 --- a/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts +++ b/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts @@ -17,7 +17,10 @@ import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import type { ScopeResolutionIndexes } from '../../../src/core/ingestion/model/scope-resolution-indexes.js'; import { buildGraphNodeLookup } from '../../../src/core/ingestion/scope-resolution/graph-bridge/node-lookup.js'; import { createCalleeIdAccumulator } from '../../../src/core/ingestion/scope-resolution/graph-bridge/callee-id-sink.js'; -import { emitCallableValueFlow } from '../../../src/core/ingestion/scope-resolution/passes/callable-value-flow.js'; +import { + emitCallableValueFlow, + collectDeferredIndirectCollection, +} from '../../../src/core/ingestion/scope-resolution/passes/callable-value-flow.js'; const FILE = 'chain.ts'; const MODULE = 'scope:module' as ScopeId; @@ -200,3 +203,28 @@ describe('callable-value-flow dependency worklist', () => { ).toEqual(['target']); }); }); + +describe('collectDeferredIndirectCollection', () => { + it('fills call signatures from the first referenceSites walk', () => { + const parsed: ParsedFile = { + filePath: FILE, + moduleScope: MODULE, + scopes: [], + parsedImports: [], + localDefs: [], + referenceSites: [ + { + name: 'target', + kind: 'call', + callForm: 'free', + atRange: range(1), + inScope: MODULE, + arity: 0, + }, + ], + callableFlowSites: [], + }; + const collected = collectDeferredIndirectCollection([parsed]); + expect(collected.callSignaturesBySite.get(`${FILE}:1:0`)).toEqual({ parameterCount: 0 }); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts b/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts index bb5900cd3..976247da4 100644 --- a/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts +++ b/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts @@ -1,5 +1,5 @@ import type { NodeLabel } from 'gitnexus-shared'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { createSemanticModel } from '../../../src/core/ingestion/model/semantic-model.js'; @@ -196,3 +196,89 @@ describe('parse-result graph insertion determinism', () => { ).toBe(record.id); }); }); + +function countingLookup(inner: ReturnType): { + lookup: ReturnType; + gets: () => number; +} { + let n = 0; + const lookup = new Proxy(inner, { + get(target, prop, receiver) { + if (prop === 'get') { + return (key: string) => { + n += 1; + return target.get(key); + }; + } + const value = Reflect.get(target, prop, receiver) as unknown; + return typeof value === 'function' + ? (value as (...args: never[]) => unknown).bind(target) + : value; + }, + }); + return { lookup, gets: () => n }; +} + +describe('resolveDefGraphId memo', () => { + const MEMO_ENV = 'GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO'; + let previousMemoEnv: string | undefined; + + beforeEach(() => { + previousMemoEnv = process.env[MEMO_ENV]; + delete process.env[MEMO_ENV]; + }); + + afterEach(() => { + if (previousMemoEnv === undefined) delete process.env[MEMO_ENV]; + else process.env[MEMO_ENV] = previousMemoEnv; + }); + + it('returns the same id on a repeated lookup and does not leak across rebuilt lookups', () => { + const method = { + id: `Method:${FILE}:Service.save#1`, + label: 'Method' as const, + name: 'save', + qualifiedName: 'Service.save', + startLine: 10, + }; + const countedA = countingLookup(buildLookup([method])); + const countedB = countingLookup(buildLookup([method])); + const def = { + type: 'Method' as const, + qualifiedName: 'Service.save', + nodeId: 'def:src/service.ts#11:0:Method:Service.save', + }; + const first = resolveDefGraphId(FILE, def, countedA.lookup); + const getsAfterFirst = countedA.gets(); + const second = resolveDefGraphId(FILE, def, countedA.lookup); + expect(first).toBe(method.id); + expect(second).toBe(first); + expect(getsAfterFirst).toBeGreaterThan(0); + expect(countedA.gets()).toBe(getsAfterFirst); + const otherLookup = resolveDefGraphId(FILE, def, countedB.lookup); + expect(otherLookup).toBe(method.id); + expect(countedB.gets()).toBeGreaterThan(0); + expect(countedA.lookup).not.toBe(countedB.lookup); + }); + + it('walks the lookup again when the memo env opt-out is set', () => { + process.env[MEMO_ENV] = '0'; + const method = { + id: `Method:${FILE}:Service.save#1`, + label: 'Method' as const, + name: 'save', + qualifiedName: 'Service.save', + startLine: 10, + }; + const counted = countingLookup(buildLookup([method])); + const def = { + type: 'Method' as const, + qualifiedName: 'Service.save', + nodeId: 'def:src/service.ts#11:0:Method:Service.save', + }; + expect(resolveDefGraphId(FILE, def, counted.lookup)).toBe(method.id); + const getsAfterFirst = counted.gets(); + expect(resolveDefGraphId(FILE, def, counted.lookup)).toBe(method.id); + expect(counted.gets()).toBeGreaterThan(getsAfterFirst); + }); +});