From 4f16bd8023c069d5d4656c0c7893171ccea4be91 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Sat, 29 Aug 2026 08:38:20 +0100 Subject: [PATCH 1/3] fix(impact): report scope extraction omissions (#3071) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(impact): surface scope extraction omissions * fix(impact): preserve complete index fixtures * fix(impact): preserve scope completeness evidence * test(analyze): model successful scope extraction in harnesses --------- Co-authored-by: Gergő Magyar --- GUARDRAILS.md | 6 ++ RUNBOOK.md | 11 ++ gitnexus/src/core/index-freshness.ts | 19 +++- .../src/core/ingestion/parsing-processor.ts | 7 ++ .../ingestion/pipeline-phases/parse-impl.ts | 13 +++ .../core/ingestion/pipeline-phases/parse.ts | 4 + gitnexus/src/core/ingestion/pipeline.ts | 6 +- .../core/ingestion/scope-extractor-bridge.ts | 12 ++- .../scope-resolution/pipeline/phase.ts | 22 +++- .../scope-resolution/pipeline/run.ts | 13 ++- .../scope-extraction-failures.ts | 49 +++++++++ .../core/ingestion/workers/parse-worker.ts | 13 ++- .../core/ingestion/workers/result-merge.ts | 5 +- gitnexus/src/core/run-analyze.ts | 8 ++ gitnexus/src/mcp/local/local-backend.ts | 36 +++++++ gitnexus/src/mcp/tools.ts | 10 +- gitnexus/src/storage/parse-cache.ts | 12 +-- gitnexus/src/storage/repo-meta.ts | 18 +++- gitnexus/src/types/pipeline.ts | 4 + .../convex-impact-epistemic-e2e.test.ts | 8 +- .../impact-epistemic-lower-bound.test.ts | 102 +++++++++++++++++- .../impact-scope-omission-persistence.test.ts | 75 +++++++++++++ .../impact-undecided-satisfaction.test.ts | 1 + .../skip-optional-pipeline.test.ts | 1 + .../test/unit/incremental-parse-cache.test.ts | 28 ++++- .../index-freshness-graph-collapse.test.ts | 50 ++++++++- gitnexus/test/unit/list-status-branch.test.ts | 2 + ...mpl-warm-cache-parsedfile-coverage.test.ts | 25 ++++- .../unit/preprocess-source-parity.test.ts | 15 +++ gitnexus/test/unit/repo-manager.test.ts | 24 +++++ gitnexus/test/unit/resources.test.ts | 1 + gitnexus/test/unit/result-merge.test.ts | 31 ++++++ .../test/unit/run-analyze-fts-repair.test.ts | 4 + .../unit/scope-extraction-failures.test.ts | 54 ++++++++++ .../scope-resolution-phase-failures.test.ts | 94 ++++++++++++++++ .../scope-resolution/run-progress.test.ts | 47 ++++++++ gitnexus/vitest.config.ts | 4 + 37 files changed, 803 insertions(+), 31 deletions(-) create mode 100644 gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts create mode 100644 gitnexus/test/integration/impact-scope-omission-persistence.test.ts create mode 100644 gitnexus/test/unit/scope-extraction-failures.test.ts create mode 100644 gitnexus/test/unit/scope-resolution-phase-failures.test.ts diff --git a/GUARDRAILS.md b/GUARDRAILS.md index e157ade1e..72e9c1e59 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -52,6 +52,12 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. - **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal. +### Scope extraction is incomplete + +- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["scope-extraction-failed"]` when files were omitted, or `incompleteReasons: ["scope-extraction-unverified"]` when the index predates the completeness receipt or its metadata is unreadable. `impact`/`context` reports the same uncertainty as `epistemic: "lower-bound"`; confirmed omissions set `causes.scopeExtractionFiles > 0`. +- **Do:** Re-run `npx gitnexus analyze` (`--force` for a full graph rebuild). If the reason persists, inspect the scope-extraction warnings and treat impact counts as floors until the affected source is supported or corrected. +- **Why:** Parsing continued, but scope captures for the reported file count could not be produced even after the main-thread fallback. Calls, inheritance, imports, or accesses originating there may therefore be absent from the graph. + ### Analyze reports INCOMPLETE with a collapsed graph write - **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["graph-write-collapsed"]`; the analyze summary printed `Repository indexed INCOMPLETELY` naming an expected and a persisted relationship count, and the CLI exited non-zero. diff --git a/RUNBOOK.md b/RUNBOOK.md index 0f5c8b7bb..d16ccd52d 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -46,6 +46,17 @@ npx gitnexus status npx gitnexus list ``` +**Scope extraction incomplete:** `npx gitnexus status` reports +`incompleteReasons: ["scope-extraction-failed"]` when one or more files still +lack scope captures after the worker and fallback passes. `impact` and `context` +then report a lower bound with `causes.scopeExtractionFiles` set to the affected +file count. Re-run `npx gitnexus analyze --force`; if the reason remains, inspect +the scope-extraction warnings for the unsupported or malformed source file. +Every pre-existing index remains unverified until it is analyzed once by a +version that writes the completeness receipt. An older index or unreadable completeness record reports +`incompleteReasons: ["scope-extraction-unverified"]`; re-analyze it before treating +empty impact results as exact. + --- ## Embeddings diff --git a/gitnexus/src/core/index-freshness.ts b/gitnexus/src/core/index-freshness.ts index ea577f834..52e62e8d8 100644 --- a/gitnexus/src/core/index-freshness.ts +++ b/gitnexus/src/core/index-freshness.ts @@ -1,11 +1,14 @@ import { checkpointKind } from './embedding-checkpoint.js'; import type { RepoMeta } from '../storage/repo-manager.js'; +import { scopeExtractionFailureTotal } from './ingestion/scope-resolution/scope-extraction-failures.js'; export const INDEX_INCOMPLETE_REASONS = [ 'incremental-in-progress', 'embedding-checkpoint-pending', 'embedding-count-unverified', 'graph-write-collapsed', + 'scope-extraction-unverified', + 'scope-extraction-failed', ] as const; export type IndexIncompleteReason = (typeof INDEX_INCOMPLETE_REASONS)[number]; @@ -130,7 +133,14 @@ export function detectGraphWriteCollapse( /** Stable machine-readable reasons an index cannot be certified complete. */ export function getIndexIncompleteReasons( meta: - | Pick + | Pick< + RepoMeta, + | 'incrementalInProgress' + | 'embeddingCheckpoint' + | 'graphWriteCollapsed' + | 'scopeExtractionFailures' + | 'scopeExtractionReceipt' + > | null | undefined, ): IndexIncompleteReason[] { @@ -142,6 +152,13 @@ export function getIndexIncompleteReasons( // answers from a graph missing most of its edges, which is indistinguishable // from a codebase that genuinely has no such relationships. if (meta?.graphWriteCollapsed) reasons.push('graph-write-collapsed'); + if (meta?.scopeExtractionReceipt !== 1) { + reasons.push('scope-extraction-unverified'); + } else { + const total = scopeExtractionFailureTotal(meta.scopeExtractionFailures); + if (total === undefined) reasons.push('scope-extraction-unverified'); + else if (total > 0) reasons.push('scope-extraction-failed'); + } if (meta?.embeddingCheckpoint) { // The three checkpoint kinds are not one operator-facing state. GUARDRAILS // and the runbook document `embedding-checkpoint-pending` as "N node(s) diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index c91df2953..e4df7dae5 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -56,6 +56,8 @@ export interface WorkerExtractedData { * finalize-orchestrator. */ parsedFiles: ParsedFile[]; + /** Scope-extraction omissions represented by this worker/cache result. */ + scopeExtractionFailures: string[]; } type ParsedGraphNode = ParseWorkerResult['nodes'][number]; @@ -126,6 +128,7 @@ export const mergeChunkResults = ( const allORMQueries: ExtractedORMQuery[] = []; const fileScopeBindingsByFile: FileScopeBindings[] = []; const allParsedFiles: ParsedFile[] = []; + const scopeExtractionFailures: string[] = []; for (const result of chunkResults) { // Worker jobs and input files are already merged in stable start-index/path @@ -178,6 +181,9 @@ export const mergeChunkResults = ( if (result.fileScopeBindings) for (const item of result.fileScopeBindings) fileScopeBindingsByFile.push(item); if (result.parsedFiles) for (const item of result.parsedFiles) allParsedFiles.push(item); + for (const filePath of result.scopeExtractionFailures ?? []) { + scopeExtractionFailures.push(filePath); + } } return { @@ -195,6 +201,7 @@ export const mergeChunkResults = ( springTypes: allSpringTypes, fileScopeBindings: fileScopeBindingsByFile, parsedFiles: allParsedFiles, + scopeExtractionFailures, }; }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 0f4ba1b2e..cd4b8edf4 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -482,6 +482,9 @@ export async function runChunkedParseAndResolve( * cache analyze run can skip the dominant `extractParsedFile` cost * (otherwise ~58s on a 1000-file repo). */ parsedFiles: import('gitnexus-shared').ParsedFile[]; + scopeExtractionFailures: string[]; + /** Files excluded because their non-standalone language parser was unavailable. */ + unavailableScopeLanguageFiles: number; }> { const model = createSemanticModel(); const symbolTable = model.symbols; @@ -514,6 +517,10 @@ export async function runChunkedParseAndResolve( ); } } + const unavailableScopeLanguageFiles = [...skippedByLang.values()].reduce( + (total, count) => total + count, + 0, + ); // Sort parseableScanned alphabetically for stable chunk membership // across runs (Finding 4). Without this, filesystem-scan order can @@ -743,6 +750,7 @@ export async function runChunkedParseAndResolve( // the second-half of the parse-cache speedup since scope-resolution's // re-parse otherwise dominates the warm-cache wall-clock time. const allParsedFiles: import('gitnexus-shared').ParsedFile[] = []; + const scopeExtractionFailures = new Set(); // Incremental parse cache (Option B): chunk-level content-addressed. // When the chunk's (filePath, content-hash) signature matches a prior @@ -844,6 +852,9 @@ export async function runChunkedParseAndResolve( chunkStartMs: number | null, ): Promise => { if (chunkWorkerData) { + for (const filePath of chunkWorkerData.scopeExtractionFailures) { + scopeExtractionFailures.add(filePath); + } if (chunkWorkerData.parsedFiles?.length) { if (parsedFileStorePath) { await persistParsedFileChunk( @@ -1616,5 +1627,7 @@ export async function runChunkedParseAndResolve( // cache: when the file's ParsedFile is here, scope-resolution skips its own // `extractParsedFile` call. parsedFiles: allParsedFiles, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), + unavailableScopeLanguageFiles, }; } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 2484baa01..38bd4601b 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -80,6 +80,10 @@ export interface ParseOutput { * costing ~58s on a 1000-file repo). */ readonly parsedFiles: readonly ParsedFile[]; + /** Files whose scope extraction failed while legacy parsing continued. */ + readonly scopeExtractionFailures: readonly string[]; + /** Files omitted because their non-standalone language parser was unavailable. */ + readonly unavailableScopeLanguageFiles: number; } export const parsePhase: PipelinePhase = { diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 34ecfc111..69858ff28 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -370,14 +370,16 @@ export const runPipelineFromRepo = async ( } // Extract final results for the PipelineResult contract - const { totalFiles, usedWorkerPool } = getPhaseOutput<{ + const { totalFiles, usedWorkerPool, unavailableScopeLanguageFiles } = getPhaseOutput<{ totalFiles: number; usedWorkerPool: boolean; + unavailableScopeLanguageFiles: number; }>(results, 'parse'); let communityResult: CommunitiesOutput['communityResult'] | undefined; let processResult: ProcessesOutput['processResult'] | undefined; const scopeResolutionOutput = getPhaseOutput(results, 'scopeResolution'); + const scopeExtractionFailures = scopeResolutionOutput.scopeExtractionFailures; const resolutionOutcomes = scopeResolutionOutput.resolutionOutcomes; const undecidedSatisfaction = scopeResolutionOutput.undecidedSatisfaction; // Streamed PDG-emit manifest (#2202): present only when streaming was on. @@ -424,6 +426,8 @@ export const runPipelineFromRepo = async ( resolutionOutcomes, undecidedSatisfaction, usedWorkerPool, + scopeExtractionFailures, + unavailableScopeLanguageFiles, pdgEmitManifest, propertyInference, }; diff --git a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts index b87fa3b19..17f639941 100644 --- a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts +++ b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts @@ -64,7 +64,17 @@ export function extractParsedFile( const message = `scope extraction failed for ${filePath}: ${ err instanceof Error ? err.message : String(err) }`; - if (onWarn !== undefined) onWarn(message); + if (onWarn !== undefined) { + try { + onWarn(message); + } catch (warnErr) { + logger.warn( + `scope extraction warning callback failed for ${filePath}: ${ + warnErr instanceof Error ? warnErr.message : String(warnErr) + }`, + ); + } + } logger.warn(message); return undefined; } diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index 0ca8b2bea..8218a9656 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -50,6 +50,7 @@ import { buildPropertyNameIndex } from '../passes/unique-name-properties.js'; import { PdgEmitSink, type PdgEmitManifest } from '../../../lbug/pdg-emit-sink.js'; import { resolveNativeSafeStorageDir } from '../../../lbug/lbug-config.js'; import type { ScopeResolver } from '../contract/scope-resolver.js'; +import { reconcileScopeExtractionFailures } from '../scope-extraction-failures.js'; import { logger } from '../../../logger.js'; export interface ScopeResolutionOutput { @@ -61,6 +62,8 @@ export interface ScopeResolutionOutput { readonly importsEmitted: number; /** Reference (CALLS / ACCESSES / INHERITS / USES) edges emitted. */ readonly referenceEdgesEmitted: number; + /** Files still missing scope captures after the main-thread fallback. */ + readonly scopeExtractionFailures: readonly string[]; /** Additive stream of resolver diagnostics; does not affect graph edges. */ readonly resolutionOutcomes: readonly ResolutionOutcome[]; /** @@ -125,6 +128,7 @@ const NOOP_OUTPUT: ScopeResolutionOutput = Object.freeze({ filesProcessed: 0, importsEmitted: 0, referenceEdgesEmitted: 0, + scopeExtractionFailures: [], resolutionOutcomes: [], // Deliberately absent, not `[]`: nothing ran, so nothing was decided either. perLanguage: new Map(), @@ -174,6 +178,7 @@ export const scopeResolutionPhase: PipelinePhase = { const { scannedFiles } = getPhaseOutput(deps, 'structure'); const parseOutput = getPhaseOutput(deps, 'parse'); const { model, parsedFiles: workerParsedFiles } = parseOutput; + const scopeExtractionFailures = new Set(parseOutput.scopeExtractionFailures); // SemanticModel populated during `parse`: scope-resolution consumes // TypeRegistry / MethodRegistry / SymbolTable lookups instead of // rebuilding parallel indexes. See ARCHITECTURE.md § "Semantic-model @@ -538,6 +543,14 @@ export const scopeResolutionPhase: PipelinePhase = { provider, ); + // Worker warnings are provisional: scope-resolution retries missing + // ParsedFiles on the main thread. Persist only final omissions. + reconcileScopeExtractionFailures( + scopeExtractionFailures, + files.map((file) => file.path), + stats.scopeExtractionFailedPaths, + ); + // Release file contents and pre-extracted entries after each language // to reduce memory pressure. For large codebases (16K+ PHP files), // holding all source code simultaneously with scope trees causes OOM. @@ -651,13 +664,20 @@ export const scopeResolutionPhase: PipelinePhase = { // Even when no language ran, surface a finalized manifest (its CSVs are on // disk) so loadGraphToLbug COPYs them rather than orphaning them — empty in // the no-files case, harmless. - if (!anyRan) return pdgEmitManifest ? { ...NOOP_OUTPUT, pdgEmitManifest } : NOOP_OUTPUT; + if (!anyRan) { + return { + ...NOOP_OUTPUT, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), + ...(pdgEmitManifest ? { pdgEmitManifest } : {}), + }; + } return { ran: true, filesProcessed: totalFiles, importsEmitted: totalImports, referenceEdgesEmitted: totalRefs, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), resolutionOutcomes, undecidedSatisfaction, perLanguage, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index fcc456d66..689780d40 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -464,6 +464,8 @@ interface RunScopeResolutionInput { interface RunScopeResolutionStats { readonly filesProcessed: number; readonly filesSkipped: number; + /** Files still missing a ParsedFile after the main-thread fallback. */ + readonly scopeExtractionFailedPaths: readonly string[]; readonly importsEmitted: number; readonly resolve: ResolveStats; readonly referenceEdgesEmitted: number; @@ -564,6 +566,7 @@ export function runScopeResolution( // ── Phase 1: extract each file → ParsedFile ──────────────────────────── const parsedFiles: ParsedFile[] = []; + const scopeExtractionFailedPaths: string[] = []; let filesSkipped = 0; const treeCache = input.treeCache; const preExtracted = input.preExtractedParsedFiles; @@ -587,15 +590,20 @@ export function runScopeResolution( } if (parsed === undefined) { const cachedTree = treeCache?.get(file.path); + let extractionWarned = false; parsed = extractParsedFile( provider.languageProvider, file.content, file.path, - onWarn, + (warning) => { + extractionWarned = true; + onWarn(warning); + }, cachedTree, ); if (parsed === undefined) { filesSkipped++; + if (extractionWarned) scopeExtractionFailedPaths.push(file.path); continue; } } @@ -643,6 +651,7 @@ export function runScopeResolution( return { filesProcessed: parsedFiles.length, filesSkipped, + scopeExtractionFailedPaths, importsEmitted: 0, resolve: { sitesProcessed: 0, referencesEmitted: 0, unresolved: 0 }, referenceEdgesEmitted: 0, @@ -680,6 +689,7 @@ export function runScopeResolution( return { filesProcessed: 0, filesSkipped, + scopeExtractionFailedPaths, importsEmitted: 0, resolve: { sitesProcessed: 0, referencesEmitted: 0, unresolved: 0 }, referenceEdgesEmitted: 0, @@ -1644,6 +1654,7 @@ export function runScopeResolution( return { filesProcessed: parsedFiles.length, filesSkipped, + scopeExtractionFailedPaths, importsEmitted, resolve: resolveStats, referenceEdgesEmitted: diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts b/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts new file mode 100644 index 000000000..ce3f6de57 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts @@ -0,0 +1,49 @@ +export interface ScopeExtractionFailureSummary { + /** Exact number of unique files whose scope extraction failed. */ + readonly total: number; + /** Deterministic sample of repo-relative paths for diagnostics. */ + readonly paths: readonly string[]; + /** True when `paths` is a capped sample rather than the full set. */ + readonly truncated?: boolean; +} + +export const SCOPE_EXTRACTION_FAILURE_PATH_LIMIT = 25; + +/** Read a persisted summary without trusting its runtime JSON shape. */ +export function scopeExtractionFailureTotal(summary: unknown): number | undefined { + if (summary === undefined) return 0; + if (typeof summary !== 'object' || summary === null) return undefined; + const total = (summary as { total?: unknown }).total; + if (total === 0) return 0; + return typeof total === 'number' && Number.isInteger(total) && total > 0 ? total : undefined; +} + +/** Replace provisional worker failures with the final fallback outcome. */ +export function reconcileScopeExtractionFailures( + failures: Set, + attemptedPaths: readonly string[], + failedPaths: readonly string[], +): void { + const stillFailed = new Set(failedPaths); + for (const filePath of attemptedPaths) { + if (stillFailed.has(filePath)) failures.add(filePath); + else failures.delete(filePath); + } +} + +export function summarizeScopeExtractionFailures( + paths: readonly string[] = [], + limit: number = SCOPE_EXTRACTION_FAILURE_PATH_LIMIT, +): ScopeExtractionFailureSummary | undefined { + const unique = [ + ...new Set(paths.filter((path): path is string => typeof path === 'string' && path.length > 0)), + ].sort(); + if (unique.length === 0) return undefined; + const boundedLimit = + Number.isInteger(limit) && limit >= 0 ? limit : SCOPE_EXTRACTION_FAILURE_PATH_LIMIT; + return { + total: unique.length, + paths: unique.slice(0, boundedLimit), + ...(unique.length > boundedLimit ? { truncated: true } : {}), + }; +} diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 946d060a6..29c94e8ee 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -499,6 +499,12 @@ export interface ParseWorkerResult { * finalize-orchestrator. */ parsedFiles: ParsedFile[]; + /** + * Repo-relative paths whose scope-capture/extraction step threw. Optional for + * parse-cache compatibility; unlike transient worker telemetry this must be + * replayed on a cache hit so the persisted index cannot claim completeness. + */ + scopeExtractionFailures?: string[]; skippedLanguages: Record; /** * Files whose parse output carried a value the structured-clone algorithm @@ -1587,14 +1593,19 @@ const processFileGroup = ( // see parsedfile-store.ts). parse-impl flushes `result.parsedFiles` to disk // per chunk and does NOT retain them in main-thread heap, so this no longer // costs ~1× the semantic model in RAM during parse. + let scopeExtractionFailed = false; const parsedFile = extractParsedFile( provider, parseContent, file.path, - reportWarning, + (message) => { + scopeExtractionFailed = true; + reportWarning(message); + }, tree, scopeSourceKind, ); + if (scopeExtractionFailed) (result.scopeExtractionFailures ??= []).push(file.path); if (parsedFile !== undefined) { // Capture-time side-channel (#1983): `extractParsedFile` just ran the // provider's `emitScopeCaptures`, which (for C++ ADL/namespace marks, diff --git a/gitnexus/src/core/ingestion/workers/result-merge.ts b/gitnexus/src/core/ingestion/workers/result-merge.ts index 014c9fb8a..fc6a24ac6 100644 --- a/gitnexus/src/core/ingestion/workers/result-merge.ts +++ b/gitnexus/src/core/ingestion/workers/result-merge.ts @@ -58,11 +58,14 @@ export const mergeResult = (target: ParseWorkerResult, src: ParseWorkerResult): appendAll(target.constructorBindings, src.constructorBindings); appendAll(target.fileScopeBindings, src.fileScopeBindings); appendAll(target.parsedFiles, src.parsedFiles); + if (src.scopeExtractionFailures && src.scopeExtractionFailures.length > 0) { + appendAll((target.scopeExtractionFailures ??= []), src.scopeExtractionFailures); + } for (const [lang, count] of Object.entries(src.skippedLanguages)) { target.skippedLanguages[lang] = (target.skippedLanguages[lang] || 0) + count; } if (src.skippedPaths && src.skippedPaths.length > 0) { - (target.skippedPaths ??= []).push(...src.skippedPaths); + appendAll((target.skippedPaths ??= []), src.skippedPaths); } target.fileCount += src.fileCount; }; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 3e8739e4c..f3889d444 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -22,6 +22,7 @@ import { summarizeUnresolvedReceivers, } from './ingestion/scope-resolution/unresolved-receivers.js'; import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/undecided-satisfaction.js'; +import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js'; import type { KnowledgeGraph } from './graph/types.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { @@ -3548,6 +3549,13 @@ async function runFullAnalysisInner( // Git-only: non-git repos never take the incremental path. schemaFingerprint: hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, unresolvedReceiverMembers: summarizeUnresolvedReceivers(resolutionOutcomes), + scopeExtractionFailures: summarizeScopeExtractionFailures( + pipelineResult.scopeExtractionFailures, + ), + // A receipt certifies that every scope-capable source file was inspected. + // Optional grammars may be unavailable by design; omitting the receipt in + // that case makes readers report an unverified lower bound. + scopeExtractionReceipt: pipelineResult.unavailableScopeLanguageFiles === 0 ? 1 : undefined, // Carried forward ONLY when this run could not measure — `saveMeta` writes // a fresh object, so omitting the key deletes a prior record and turns a // hedged answer back into a confident one. A run that DID measure always diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 1679a3b80..cf197c732 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -124,6 +124,7 @@ import { } from '../../core/ingestion/scope-resolution/unresolved-receivers.js'; import type { UnresolvedReceiverSummary } from '../../core/ingestion/scope-resolution/unresolved-receivers.js'; import type { UndecidedSatisfactionSummary } from '../../core/ingestion/scope-resolution/undecided-satisfaction.js'; +import { scopeExtractionFailureTotal } from '../../core/ingestion/scope-resolution/scope-extraction-failures.js'; import { lookupCount } from '../../core/ingestion/scope-resolution/summary-maps.js'; import { DEFERRED_IMPORT_REASON_SUFFIX, @@ -654,6 +655,8 @@ export interface CodebaseContext { * number of SENTENCES, which has no relation to how much is missing. */ export interface EpistemicCauses { + /** Files whose scope-extraction output is absent from this index. */ + readonly scopeExtractionFiles: number; /** * Call SITES dropped at index time because the receiver's type could not be * established. Unit: call sites, taken from the index's @@ -717,6 +720,7 @@ function epistemicFrom(dropped: { external: number; undecided: number; dispatch: number; + scopeExtraction: number; }): { epistemic: 'exact' | 'lower-bound'; boundaries?: string[]; @@ -730,6 +734,7 @@ function epistemicFrom(dropped: { ? { epistemic: 'exact', causes: { + scopeExtractionFiles: dropped.scopeExtraction, receiverTyping: 0, dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, @@ -745,6 +750,7 @@ function epistemicFrom(dropped: { // prose saying `2 call sites` — a consumer branching on the number // would read a different magnitude than the human reading the text. causes: { + scopeExtractionFiles: dropped.scopeExtraction, receiverTyping: dropped.sites, dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, @@ -753,6 +759,29 @@ function epistemicFrom(dropped: { }; } +function scopeExtractionBoundaries( + summary: unknown, + receipt: unknown, +): { notes: string[]; files: number } { + const unknown = { + notes: [ + 'Scope-extraction completeness was not recorded for this index, so actual impact may be higher.', + ], + files: 0, + }; + if (receipt !== 1) return unknown; + const total = scopeExtractionFailureTotal(summary); + if (total === undefined) return unknown; + if (total === 0) return { notes: [], files: 0 }; + return { + notes: [ + `Scope extraction failed for ${total} ${total === 1 ? 'file' : 'files'} while this index was built. ` + + `Scope-resolution edges from ${total === 1 ? 'that file are' : 'those files are'} absent, so actual impact may be higher.`, + ], + files: total, + }; +} + /** * Boundary notes for call sites the analyzer dropped because it could not type * their receiver, when the queried symbol's name is among them (#2744). @@ -6823,6 +6852,10 @@ export class LocalBackend { meta = undefined; } const receiverDrops = unresolvedReceiverBoundaries(meta?.unresolvedReceiverMembers, symName); + const scopeExtractionDrops = scopeExtractionBoundaries( + meta?.scopeExtractionFailures, + meta?.scopeExtractionReceipt, + ); // #2873 — satisfaction checks the analyzer never completed. Read on the // same footing as the receiver drops, and BEFORE the heritage probe for the // same reason: this cause leaves no edge for that probe to find, so a @@ -6860,6 +6893,7 @@ export class LocalBackend { ...receiverDrops, notes: [ ...receiverDrops.notes, + ...scopeExtractionDrops.notes, ...undecidedDrops.notes, ...(convexDispatch === undefined ? [] : [convexDispatch.boundary]), ], @@ -6868,6 +6902,7 @@ export class LocalBackend { // count of omitted symbols. Keep the magnitude at zero rather than // inventing one from the presence of a note. dispatch: 0, + scopeExtraction: scopeExtractionDrops.files, }; try { // Discover the interface / abstract supertypes on the target's boundary. @@ -6954,6 +6989,7 @@ export class LocalBackend { epistemic: 'lower-bound', boundaries: [...droppedBoundaries.notes, ...boundaries], causes: { + scopeExtractionFiles: droppedBoundaries.scopeExtraction, receiverTyping: droppedBoundaries.sites, dispatchBoundary: droppedBoundaries.dispatch + dispatchBoundarySymbols, externalBoundary: droppedBoundaries.external, diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index e7e453a1b..be793cbeb 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -287,13 +287,14 @@ NOTE: ACCESSES edges (field read/write tracking) are included in context results COMPLETENESS OF incoming: alongside symbol/incoming/outgoing the result carries the same epistemic envelope impact() returns: - epistemic: 'exact' | 'lower-bound' — 'lower-bound' means callers exist that this view provably does not list. - boundaries: string[] — one plain-language sentence per reason. Prose for humans; branch on causes instead. -- causes: { receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: + - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — RESOLVER GAP: the analyzer dropped that many call sites on this name because it could not type the receiver, so they are missing from incoming. Do not read an absent caller as proof none exists. - causes.externalBoundary (unit: call sites) > 0 — the calls left the indexed program (System.out.println, fetch(...)). NOT a defect: no in-graph node could have been reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary static analysis cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not decide whether a type satisfies an interface, so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Usually fixable by making the missing dependency available to analysis. -REQUIRES RE-INDEX: causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. +REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. GROUP MODE: set "repo" to "@" to run context in each member repo (aggregated list), or "@/" for one member. If you use "@" only, the member defaults to the lexicographically first key in group.yaml "repos". @@ -482,14 +483,15 @@ Output includes: - byDepth: affected symbols grouped by traversal depth (paginated by limit/offset; omitted when summaryOnly:true — use byDepthCounts for totals per depth, pagination object when truncated). Each item includes a processes:[{id,label,processType,step}] field listing the execution flows that symbol participates in. Empty when the symbol has no process membership. Can ALSO be empty when partial:true is set — either the process-aggregation pass hit its cap before detecting affected processes, or per-symbol enrichment was capped on a very large page. When partial:true, do NOT treat processes:[] as proof of no participation; cross-check the top-level affected_processes list. - epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the walk provably missed callers, so the count is a floor. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). - boundaries: string[] — one plain-language sentence per reason the count is short. Prose for humans; branch on causes instead. -- causes: { receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: + - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — the RESOLVER GAP signal: the analyzer dropped that many call sites because it could not establish the receiver's type (unresolved constructor, factory, chained expression). Those callers are absent from byDepth. Treat the result as incomplete: grep the symbol name before deleting or renaming. - causes.externalBoundary (unit: call sites) > 0 — those calls left the indexed program (System.out.println, fetch(...), os.environ.*). NOT a defect and NOT a reason the count is short: there is no in-graph node any edge could have reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary a static walk cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not DECIDE whether a type satisfies an interface (a type in a required signature named a package it could not resolve), so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Distinct from every cause above, which count decided facts that could not be attributed; this one counts questions never answered. It is the only cause that shortens a result WITHOUT leaving a trace in the graph, so an unhedged zero on a symbol reached only through such an interface would otherwise read as 'nobody calls this'. Usually fixable: it most often means a dependency is missing from the analyzed tree. -REQUIRES RE-INDEX: causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. +REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. Depth groups: - d=1: WILL BREAK (direct callers/importers) diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 8cc639900..636b73486 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -626,11 +626,11 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // the re-check line below is for, and why it says AT MERGE rather than // when you pick the number. // -// 77 is free at this merge: origin/main is 76, and the open PRs touching this -// constant are #2840 (a stale 71) and #1616 (a stale 2). Scan with the contents -// API at each PR head, not `gh pr diff` — that exits non-zero on an -// inaccessible fork and prints nothing, so a grep over its output skips the PR -// silently. #2840 was missed exactly that way this round. +// 78 was claimed concurrently by #3060 while this branch was in review. Both +// branches keep the same package version, so sharing 78 would replay +// incompatible worker output without a textual merge conflict. This branch +// therefore takes 79, the next free value above origin/main and every open PR +// found by the contents-API scan at their exact head SHAs. // // WHY THIS IS STILL A HAND-PICKED NUMBER, when `SCHEMA_FINGERPRINT` next door // is a derived sha256 that cannot collide. The derivation exists and already @@ -650,7 +650,7 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // `route-extractors/` and `workers/` module content — would close the missing- // bump axis without invalidating on unrelated churn, and is the real follow-up. // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING. -const SCHEMA_BUMP = 77; +const SCHEMA_BUMP = 79; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 16bd5ede9..3cda63545 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -21,7 +21,7 @@ * `isMissingFilesystemError`) so every existing import site keeps working * unchanged. * - * Imports `node:fs`/`node:path` and two type-only shapes. Keep it that way: a + * Imports `node:fs`/`node:path` and a few type-only summary shapes. Keep it that way: a * value import here would land in every consumer of `storage/`. */ @@ -29,6 +29,7 @@ import fs from 'fs/promises'; import path from 'path'; import type { UnresolvedReceiverSummary } from '../core/ingestion/scope-resolution/unresolved-receivers.js'; import type { UndecidedSatisfactionSummary } from '../core/ingestion/scope-resolution/undecided-satisfaction.js'; +import type { ScopeExtractionFailureSummary } from '../core/ingestion/scope-resolution/scope-extraction-failures.js'; /** The `.gitnexus` directory name, relative to a repo root. */ export const GITNEXUS_DIR = '.gitnexus'; @@ -245,6 +246,21 @@ export interface RepoMeta { * this adds no runtime dependency from storage/ on core/. */ unresolvedReceiverMembers?: UnresolvedReceiverSummary; + /** + * Files omitted from scope-resolution because their provider capture or + * extraction step threw. The rest of each file may still be present in the + * graph, so this is an index-completeness signal rather than a parse failure. + * Absent means the successful run recorded no such omission; older indexes + * also read as absent until re-analyzed. + */ + scopeExtractionFailures?: ScopeExtractionFailureSummary; + /** + * Completeness receipt for scope extraction in the successful run represented + * by this metadata. A missing or different value means completeness is + * unknown (legacy, malformed, or unreadable metadata), not that zero files + * were omitted. + */ + scopeExtractionReceipt?: 1; /** * Interfaces whose structural-satisfaction check this run could not COMPLETE * (#2873) — not interfaces found to have no implementors. diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index 015696ed6..950cd3f31 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -40,6 +40,10 @@ export interface PipelineResult { * affordance so regression suites can prove the pool engaged. */ usedWorkerPool: boolean; + /** Files omitted from scope-resolution while the rest of analysis continued. */ + scopeExtractionFailures: readonly string[]; + /** Files scope resolution could not inspect because their parser was unavailable. */ + unavailableScopeLanguageFiles: number; /** * Streamed PDG-emit COPY manifest (#2202). Present only when streaming/chunked * PDG emit was active (full rebuild + `--pdg` + enabled): the BasicBlock node diff --git a/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts b/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts index 74e1f1073..ae81e9570 100644 --- a/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts +++ b/gitnexus/test/integration/convex-impact-epistemic-e2e.test.ts @@ -14,7 +14,7 @@ import { pruneAndSaveDurableParsedFileStore, } from '../../src/storage/parsedfile-store.js'; import { LocalBackend } from '../../src/mcp/local/local-backend.js'; -import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { listRegisteredRepos, saveMeta, type RepoMeta } from '../../src/storage/repo-manager.js'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({ @@ -196,6 +196,12 @@ export const javascriptQuery = query({ handler: async () => null }); const adapter = await import('../../src/core/lbug/lbug-adapter.js'); await adapter.loadGraphToLbug(replay.graph, repoDir, storageDir); + await saveMeta(storageDir, { + repoPath: repoDir, + lastCommit: 'convex-e2e', + indexedAt: new Date(0).toISOString(), + scopeExtractionReceipt: 1, + } satisfies RepoMeta); }, poolAdapter: true, afterSetup: async (handle) => { diff --git a/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts b/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts index 6ca1d6dde..2ef3e6c78 100644 --- a/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts +++ b/gitnexus/test/integration/impact-epistemic-lower-bound.test.ts @@ -16,15 +16,16 @@ * container, so impact("EmailLogger", upstream) finds no direct caller — but * must flag that the true blast radius is higher. */ -import { it, expect, beforeAll, vi } from 'vitest'; +import { it, expect, beforeAll, beforeEach, vi } from 'vitest'; import { LocalBackend } from '../../src/mcp/local/local-backend.js'; -import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { listRegisteredRepos, loadMeta } from '../../src/storage/repo-manager.js'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; vi.mock('../../src/storage/repo-manager.js', () => ({ listRegisteredRepos: vi.fn().mockResolvedValue([]), cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), findSiblingClones: vi.fn().mockResolvedValue([]), + loadMeta: vi.fn().mockResolvedValue({ scopeExtractionReceipt: 1 }), })); const SEED = [ @@ -66,6 +67,11 @@ withTestLbugDB( beforeAll(() => { backend = (handle as any)._backend; }); + beforeEach(() => { + vi.mocked(loadMeta).mockResolvedValue({ + scopeExtractionReceipt: 1, + } as Awaited>); + }); it('flags impact() on a concrete impl behind an interface as lower-bound', async () => { const result = await backend.callTool('impact', { @@ -116,6 +122,78 @@ withTestLbugDB( expect(result.impactedCount).toBeGreaterThanOrEqual(1); }); + it('marks impact as a lower bound when scope extraction omitted files', async () => { + vi.mocked(loadMeta).mockResolvedValueOnce({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 2, + paths: ['src/broken-a.ts', 'src/broken-b.ts'], + }, + } as Awaited>); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain('Scope extraction failed for 2 files'); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 2 }); + }); + + it('never renders repository-controlled failure paths in boundary prose', async () => { + vi.mocked(loadMeta).mockResolvedValueOnce({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 1, + paths: ['src/`break`\n\u001b[31m\u202e\u200binject.ts'], + }, + } as Awaited>); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + const prose = result.boundaries.join(' '); + + expect(prose).toContain('Scope extraction failed for 1 file'); + expect(prose).not.toMatch(/[\n\u001b\u202e\u200b`]/u); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 1 }); + }); + + it.each([ + ['missing receipt', {}], + ['missing metadata', null], + ['malformed summary', { scopeExtractionReceipt: 1, scopeExtractionFailures: 'invalid' }], + ])('treats %s as an unknown lower bound', async (_label, metadata) => { + vi.mocked(loadMeta).mockResolvedValueOnce(metadata as Awaited>); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain( + 'Scope-extraction completeness was not recorded', + ); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 0 }); + }); + + it('treats a metadata read failure as an unknown lower bound', async () => { + vi.mocked(loadMeta).mockRejectedValueOnce(new Error('metadata unavailable')); + + const result = await backend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain( + 'Scope-extraction completeness was not recorded', + ); + }); + it.each([ ['listOrders', 'query'], ['createOrder', 'mutation'], @@ -159,6 +237,26 @@ withTestLbugDB( expect(result.epistemic).toBe('lower-bound'); }); + it('context() reports persisted scope extraction omissions as a lower bound', async () => { + vi.mocked(loadMeta).mockResolvedValueOnce({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 2, + paths: ['src/broken-a.ts', 'src/broken-b.ts'], + }, + } as Awaited>); + + const result = await backend.callTool('context', { + name: 'formatDate', + file_path: 'src/util.ts', + }); + + expect(result.status).toBe('found'); + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain('Scope extraction failed for 2 files'); + expect(result.causes).toMatchObject({ scopeExtractionFiles: 2 }); + }); + it('context() on a leaf interface itself is lower-bound (#1858 review F3)', async () => { // Logger is a leaf interface — it implements/extends nothing, so the only // boundary signal is computeEpistemicBoundary's symType==='Interface' diff --git a/gitnexus/test/integration/impact-scope-omission-persistence.test.ts b/gitnexus/test/integration/impact-scope-omission-persistence.test.ts new file mode 100644 index 000000000..83661475b --- /dev/null +++ b/gitnexus/test/integration/impact-scope-omission-persistence.test.ts @@ -0,0 +1,75 @@ +import { beforeEach, expect, it, vi } from 'vitest'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { listRegisteredRepos, saveMeta } from '../../src/storage/repo-manager.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; + +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, listRegisteredRepos: vi.fn() }; +}); + +const SEED = [ + `CREATE (:Function {id: 'Function:src/util.ts:formatDate', name: 'formatDate', filePath: 'src/util.ts', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, +]; + +withTestLbugDB( + 'impact-scope-omission-persistence', + (handle) => { + beforeEach(() => { + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date(0).toISOString(), + lastCommit: 'abc123', + stats: { files: 1, nodes: 1, communities: 0, processes: 0 }, + }, + ]); + }); + + it('persists omissions for reopened readers and clears them after a clean run', async () => { + const baseMeta = { + repoPath: '/test/repo', + lastCommit: 'abc123', + indexedAt: new Date(0).toISOString(), + scopeExtractionReceipt: 1 as const, + }; + await saveMeta(handle.tmpHandle.dbPath, { + ...baseMeta, + scopeExtractionFailures: { total: 1, paths: ['src/broken.ts'] }, + }); + + const incompleteBackend = new LocalBackend(); + await incompleteBackend.init(); + const impact = await incompleteBackend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + const context = await incompleteBackend.callTool('context', { + name: 'formatDate', + file_path: 'src/util.ts', + }); + expect(impact).toMatchObject({ + epistemic: 'lower-bound', + causes: { scopeExtractionFiles: 1 }, + }); + expect(context).toMatchObject({ + status: 'found', + epistemic: 'lower-bound', + causes: { scopeExtractionFiles: 1 }, + }); + + await saveMeta(handle.tmpHandle.dbPath, baseMeta); + const cleanBackend = new LocalBackend(); + await cleanBackend.init(); + const cleanImpact = await cleanBackend.callTool('impact', { + target: 'formatDate', + direction: 'upstream', + }); + expect(cleanImpact).toMatchObject({ epistemic: 'exact' }); + expect(cleanImpact).not.toHaveProperty('boundaries'); + }); + }, + { seed: SEED, poolAdapter: true }, +); diff --git a/gitnexus/test/integration/impact-undecided-satisfaction.test.ts b/gitnexus/test/integration/impact-undecided-satisfaction.test.ts index ce79b4a01..f40a407c1 100644 --- a/gitnexus/test/integration/impact-undecided-satisfaction.test.ts +++ b/gitnexus/test/integration/impact-undecided-satisfaction.test.ts @@ -108,6 +108,7 @@ withTestLbugDB( // uses, and it is atomic and dual-writes the legacy mirror. Writing the // file directly would pin a shape no real analyze can produce. await saveMeta(path.dirname(h.dbPath), { + scopeExtractionReceipt: 1, undecidedInterfaceSatisfaction: { counts: { CtxStore: 2 }, totalInterfaces: 1, diff --git a/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts b/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts index d60fca427..26663fd8d 100644 --- a/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts +++ b/gitnexus/test/integration/optional-grammars/skip-optional-pipeline.test.ts @@ -79,6 +79,7 @@ describe('optional-grammar pipeline exclusion (#2091/#2093)', () => { it('skips the Swift file at the parse phase (non-vacuity: Swift was present)', () => { expect(messages.some((m) => /Skipping 1 swift file\(s\)/.test(m))).toBe(true); + expect(result.unavailableScopeLanguageFiles).toBe(1); }); it('routes the opt-out message, not the missing-binding "npm rebuild" hint', () => { diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index d6500c702..b476f9743 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -240,14 +240,16 @@ describe('PARSE_CACHE_VERSION', () => { // collided, because each re-checked once and neither re-checked after the // other moved — which is why the rule is re-applied AT MERGE, not when the // number is picked. - it('pins SCHEMA_BUMP to 77 so concurrent bumps cannot silently collide (#2766)', () => { - expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(77); + it('pins SCHEMA_BUMP to 79 so concurrent bumps cannot silently collide (#2766, #3015)', () => { + expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(79); // The PREVIOUS version must fail the reuse gate, not merely differ from the // current one — a hardcoded number outside the conflict hunk rebases cleanly // while being wrong, which is exactly how the 37/38 exact clashes landed. // Every nearby historical or in-flight value is rejected, including 69, // which carried the route-table payload before this merge. - for (const taken of [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76]) { + for (const taken of [ + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + ]) { expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).not.toBe(taken); } }); @@ -634,12 +636,14 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { referenceSites: [], }, ], + scopeExtractionFailures: ['a.c'], }); const slim = slimParseWorkerResultsForCache([raw])[0]; expect(slim.calls).toEqual([]); expect(slim.assignments).toEqual([]); expect(slim.constructorBindings).toEqual([]); expect(slim.parsedFiles).toEqual([]); + expect(slim.scopeExtractionFailures).toEqual(['a.c']); expect(slim.fileCount).toBe(raw.fileCount); }); @@ -660,6 +664,24 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { expect(slim.nodes).toHaveLength(1); }); + it('round-trips scope extraction failures through a persisted cache shard', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const key = 'e'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[key, [minimalResult({ scopeExtractionFailures: ['src/broken.ts'] })]]]), + usedKeys: new Set([key]), + }); + + const loaded = await loadParseCache(dir); + const replayed = await loadParseCacheChunk(loaded, key); + expect(replayed?.[0]?.scopeExtractionFailures).toEqual(['src/broken.ts']); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('persistParseCacheChunk writes to disk without retaining in-memory entries', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { diff --git a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts index f92a8e375..d2c32a280 100644 --- a/gitnexus/test/unit/index-freshness-graph-collapse.test.ts +++ b/gitnexus/test/unit/index-freshness-graph-collapse.test.ts @@ -208,27 +208,69 @@ describe('graph-write-collapsed incomplete reason (B2)', () => { it('reports a collapsed write as incomplete rather than fresh', () => { expect( - getIndexIncompleteReasons({ graphWriteCollapsed: { expected: 23009, persisted: 2170 } }), + getIndexIncompleteReasons({ + graphWriteCollapsed: { expected: 23009, persisted: 2170 }, + scopeExtractionReceipt: 1, + }), ).toEqual(['graph-write-collapsed']); }); it('treats a missing relation table (zero persisted) the same way', () => { expect( - getIndexIncompleteReasons({ graphWriteCollapsed: { expected: 23009, persisted: 0 } }), + getIndexIncompleteReasons({ + graphWriteCollapsed: { expected: 23009, persisted: 0 }, + scopeExtractionReceipt: 1, + }), ).toEqual(['graph-write-collapsed']); }); it('says nothing on a healthy run', () => { - expect(getIndexIncompleteReasons({})).toEqual([]); - expect(getIndexIncompleteReasons(null)).toEqual([]); + expect(getIndexIncompleteReasons({ scopeExtractionReceipt: 1 })).toEqual([]); + }); + + it('marks missing metadata or receipt as scope-extraction-unverified', () => { + expect(INDEX_INCOMPLETE_REASONS).toContain('scope-extraction-unverified'); + expect(getIndexIncompleteReasons({})).toEqual(['scope-extraction-unverified']); + expect(getIndexIncompleteReasons(null)).toEqual(['scope-extraction-unverified']); }); it('reports alongside other reasons rather than masking them', () => { const reasons = getIndexIncompleteReasons({ incrementalInProgress: { startedAt: 1, toWriteCount: 0 }, graphWriteCollapsed: { expected: 500, persisted: 10 }, + scopeExtractionReceipt: 1, }); expect(reasons).toContain('incremental-in-progress'); expect(reasons).toContain('graph-write-collapsed'); }); }); + +describe('scope-extraction-failed incomplete reason (#3015)', () => { + it('is stable and reports a partial scope index as incomplete', () => { + expect(INDEX_INCOMPLETE_REASONS).toContain('scope-extraction-failed'); + expect( + getIndexIncompleteReasons({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { total: 2, paths: ['src/a.ts', 'src/b.ts'] }, + }), + ).toContain('scope-extraction-failed'); + }); + + it('does not report a malformed zero-count record as incomplete', () => { + expect( + getIndexIncompleteReasons({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { total: 0, paths: [] }, + }), + ).toEqual([]); + }); + + it('marks malformed summaries as unverified even when the receipt is present', () => { + expect( + getIndexIncompleteReasons({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { total: Number.NaN, paths: [] }, + }), + ).toEqual(['scope-extraction-unverified']); + }); +}); diff --git a/gitnexus/test/unit/list-status-branch.test.ts b/gitnexus/test/unit/list-status-branch.test.ts index 2e528cc8e..f472397f8 100644 --- a/gitnexus/test/unit/list-status-branch.test.ts +++ b/gitnexus/test/unit/list-status-branch.test.ts @@ -137,6 +137,7 @@ describe('status branch rendering (#2106)', () => { indexedAt: '2026-06-10T12:00:00.000Z', branch: 'main', runnerIdentity, + scopeExtractionReceipt: 1 as const, }, }; @@ -287,6 +288,7 @@ describe('status branch rendering (#2106)', () => { indexedAt: '2026-06-10T14:00:00.000Z', branch: 'feature/z', runnerIdentity, + scopeExtractionReceipt: 1, }); await statusCommand(); diff --git a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts index 36e9140ce..77ac0a48c 100644 --- a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts +++ b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts @@ -194,6 +194,7 @@ const reset = () => ({ routes: [], fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], routerIncludes: [], routerImports: [], toolDefs: [], ormQueries: [], constructorBindings: [], fileScopeBindings: [], parsedFiles: [], skippedLanguages: {}, fileCount: 0, + scopeExtractionFailures: [], }); let accumulated = reset(); parentPort.on('message', (msg) => { @@ -209,6 +210,7 @@ parentPort.on('message', (msg) => { accumulated.parsedFiles.push({ filePath, moduleScope: '', scopes: [], parsedImports: [], localDefs: [], referenceSites: [], }); + if (filePath.includes('broken')) accumulated.scopeExtractionFailures.push(filePath); accumulated.fileCount++; } parentPort.postMessage({ type: 'progress', filesProcessed: accumulated.fileCount }); @@ -293,9 +295,9 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { cache: ReturnType, files: { path: string; size: number }[], chunkByteBudget?: number, - ): Promise => { + ): Promise>> => { const rels = files.map((f) => f.path); - await runChunkedParseAndResolve( + return runChunkedParseAndResolve( createKnowledgeGraph(), files, rels, @@ -335,7 +337,7 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { const f = writeFile('src/degrade.ts', 'export function degrade() { return 1; }\n'); prepareOverride.impl = () => Promise.reject(new Error('EACCES: simulated cache failure')); try { - await expect(run(newCache(), [f])).resolves.toBeUndefined(); + await expect(run(newCache(), [f])).resolves.toBeDefined(); } finally { prepareOverride.impl = undefined; } @@ -379,6 +381,23 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { expect(fs.existsSync(markerPath)).toBe(false); // NO worker spawned on the warm hit }); + it('replays scope-extraction failures from a warm parse-cache hit', async () => { + const f = writeFile('src/broken.ts', 'export function broken() { return 1; }\n'); + const cache = newCache(); + + const cold = await run(cache, [f]); + expect(cold.scopeExtractionFailures).toEqual([f.path]); + await persistCaches(cache); + + const { loadParseCache } = await import('../../src/storage/parse-cache.js'); + const warm = await loadParseCache(storageDir); + fs.rmSync(markerPath, { force: true }); + + const replayed = await run(warm as ReturnType, [f]); + expect(fs.existsSync(markerPath)).toBe(false); + expect(replayed.scopeExtractionFailures).toEqual([f.path]); + }); + it('coherence gate: a parse-cache hit with NO durable shards re-dispatches the worker', async () => { const f = writeFile('src/cached.ts', 'export function cached() { return 1; }\n'); const cache = newCache(); diff --git a/gitnexus/test/unit/preprocess-source-parity.test.ts b/gitnexus/test/unit/preprocess-source-parity.test.ts index 6c3d04e6e..643d6cacd 100644 --- a/gitnexus/test/unit/preprocess-source-parity.test.ts +++ b/gitnexus/test/unit/preprocess-source-parity.test.ts @@ -4,6 +4,7 @@ import { providers, getProvider } from '../../src/core/ingestion/languages/index import { extractParsedFile } from '../../src/core/ingestion/scope-extractor-bridge.js'; import { isLanguageAvailable } from '../../src/core/tree-sitter/parser-loader.js'; import { ensureAndParse } from '../../src/core/embeddings/ast-utils.js'; +import type { LanguageProvider } from '../../src/core/ingestion/language-provider.js'; /** * Every provider that defines `preprocessSource` must produce the same @@ -55,6 +56,20 @@ const languagesWithHook = Object.entries(providers) .sort(); describe('LanguageProvider.preprocessSource parity', () => { + it('does not propagate an exception thrown by the warning callback', () => { + const provider = { + emitScopeCaptures: () => { + throw new Error('provider failed'); + }, + } as unknown as LanguageProvider; + + expect(() => + extractParsedFile(provider, 'const value = 1;', 'broken.ts', () => { + throw new Error('warning transport closed'); + }), + ).not.toThrow(); + }); + it('has a fixture for every provider defining the hook', () => { expect(Object.keys(FIXTURES).sort()).toEqual(languagesWithHook); }); diff --git a/gitnexus/test/unit/repo-manager.test.ts b/gitnexus/test/unit/repo-manager.test.ts index 1c6c20e5f..2080e2f45 100644 --- a/gitnexus/test/unit/repo-manager.test.ts +++ b/gitnexus/test/unit/repo-manager.test.ts @@ -230,6 +230,30 @@ describe('saveMeta dual-write', () => { expect(JSON.parse(legacy)).toEqual(meta); }); + it('round-trips scope extraction failure metadata through the production writer', async () => { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const withFailures: RepoMeta = { + ...meta, + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 3, + paths: ['src/a.ts', 'src/b.ts'], + truncated: true, + }, + }; + + await saveMeta(storagePath, withFailures); + + expect(await loadMeta(storagePath)).toMatchObject({ + scopeExtractionReceipt: 1, + scopeExtractionFailures: { + total: 3, + paths: ['src/a.ts', 'src/b.ts'], + truncated: true, + }, + }); + }); + it('leaves no stray tmp files behind after a successful write', async () => { const { storagePath } = getStoragePaths(tmpRepo.dbPath); await saveMeta(storagePath, meta); diff --git a/gitnexus/test/unit/resources.test.ts b/gitnexus/test/unit/resources.test.ts index aef0eac27..ff3894be1 100644 --- a/gitnexus/test/unit/resources.test.ts +++ b/gitnexus/test/unit/resources.test.ts @@ -539,6 +539,7 @@ describe('context resource freshness after out-of-process analyze (#2438)', () = lastCommit: 'current-head', indexedAt: '2026-07-18T12:00:00.000Z', incrementalInProgress: { startedAt: 1, toWriteCount: 2 }, + scopeExtractionReceipt: 1, embeddingCheckpoint: { at: '2026-07-18T12:00:00.000Z', nodesProcessed: 1, diff --git a/gitnexus/test/unit/result-merge.test.ts b/gitnexus/test/unit/result-merge.test.ts index dff56df3e..19d9b8b2e 100644 --- a/gitnexus/test/unit/result-merge.test.ts +++ b/gitnexus/test/unit/result-merge.test.ts @@ -58,6 +58,37 @@ describe('mergeResult', () => { expect(target.skippedPaths).toBeUndefined(); }); + it('unions scope-extraction failures across worker sub-batches', () => { + const target = emptyResult(); + mergeResult(target, { + ...emptyResult(), + scopeExtractionFailures: ['src/a.ts'], + }); + mergeResult(target, { + ...emptyResult(), + scopeExtractionFailures: ['src/b.ts'], + }); + expect(target.scopeExtractionFailures).toEqual(['src/a.ts', 'src/b.ts']); + }); + + it('leaves scope-extraction failures absent for backward-compatible results', () => { + const target = emptyResult(); + mergeResult(target, emptyResult()); + expect(target.scopeExtractionFailures).toBeUndefined(); + }); + + it('merges failure sets larger than the JavaScript argument limit', () => { + const target = emptyResult(); + const scopeExtractionFailures = Array.from( + { length: 70_000 }, + (_, index) => `src/failure-${index}.ts`, + ); + + expect(() => mergeResult(target, { ...emptyResult(), scopeExtractionFailures })).not.toThrow(); + expect(target.scopeExtractionFailures).toHaveLength(70_000); + expect(target.scopeExtractionFailures?.at(-1)).toBe('src/failure-69999.ts'); + }); + it('unions springTypes across sub-batch results, initializing the target when absent (#2288)', () => { const mkType = (name: string, filePath: string) => ({ filePath, diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index ea3914291..24d03844f 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -1516,6 +1516,8 @@ describe('runFullAnalysis Phase 5 embedding gate (#2790)', () => { runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, totalFileCount: 1, + scopeExtractionFailures: [], + unavailableScopeLanguageFiles: 0, graph: { forEachNode: (fn: (node: typeof stubNode) => void) => fn(stubNode), getNode: (id: string) => (id === GATE_NODE_ID ? stubNode : undefined), @@ -2122,6 +2124,8 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, totalFileCount: 1, + scopeExtractionFailures: [], + unavailableScopeLanguageFiles: 0, graph: { forEachNode: (fn: (node: typeof stubNode) => void) => fn(stubNode), getNode: (id: string) => (id === RESILIENCE_NODE_ID ? stubNode : undefined), diff --git a/gitnexus/test/unit/scope-extraction-failures.test.ts b/gitnexus/test/unit/scope-extraction-failures.test.ts new file mode 100644 index 000000000..54d589927 --- /dev/null +++ b/gitnexus/test/unit/scope-extraction-failures.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { + reconcileScopeExtractionFailures, + scopeExtractionFailureTotal, + summarizeScopeExtractionFailures, +} from '../../src/core/ingestion/scope-resolution/scope-extraction-failures.js'; + +describe('summarizeScopeExtractionFailures', () => { + it('deduplicates, sorts, and caps paths while retaining the exact total', () => { + expect(summarizeScopeExtractionFailures(['z.ts', 'a.ts', 'z.ts', 'b.ts'], 2)).toEqual({ + total: 3, + paths: ['a.ts', 'b.ts'], + truncated: true, + }); + }); + + it('returns undefined when no failure was recorded', () => { + expect(summarizeScopeExtractionFailures([])).toBeUndefined(); + expect(summarizeScopeExtractionFailures()).toBeUndefined(); + }); + + it('ignores malformed paths restored from a corrupt cache payload', () => { + expect( + summarizeScopeExtractionFailures(['valid.ts', null, undefined, 42] as unknown as string[]), + ).toEqual({ total: 1, paths: ['valid.ts'] }); + }); + + it('clears worker failures recovered by fallback and retains final omissions', () => { + const failures = new Set(['recovered.ts', 'still-broken.ts', 'untouched.ts']); + + reconcileScopeExtractionFailures( + failures, + ['recovered.ts', 'still-broken.ts', 'new-failure.ts'], + ['still-broken.ts', 'new-failure.ts'], + ); + + expect([...failures].sort()).toEqual(['new-failure.ts', 'still-broken.ts', 'untouched.ts']); + }); +}); + +describe('scopeExtractionFailureTotal', () => { + it.each([ + ['absent summary', undefined, 0], + ['clean summary', { total: 0, paths: [] }, 0], + ['failure summary', { total: 2, paths: ['a.ts', 'b.ts'] }, 2], + ['non-object', 'invalid', undefined], + ['null', null, undefined], + ['fractional count', { total: 1.5 }, undefined], + ['negative count', { total: -1 }, undefined], + ['missing count', {}, undefined], + ])('reads %s consistently', (_name, summary, expected) => { + expect(scopeExtractionFailureTotal(summary)).toBe(expected); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution-phase-failures.test.ts b/gitnexus/test/unit/scope-resolution-phase-failures.test.ts new file mode 100644 index 000000000..ca1816d58 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution-phase-failures.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const runScopeResolutionMock = vi.hoisted(() => vi.fn()); +vi.mock('../../src/core/ingestion/scope-resolution/pipeline/run.js', async (importOriginal) => { + const actual = + await importOriginal< + typeof import('../../src/core/ingestion/scope-resolution/pipeline/run.js') + >(); + return { ...actual, runScopeResolution: runScopeResolutionMock }; +}); + +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { createSemanticModel } from '../../src/core/ingestion/model/index.js'; +import { scopeResolutionPhase } from '../../src/core/ingestion/scope-resolution/pipeline/phase.js'; +import type { ParseOutput } from '../../src/core/ingestion/pipeline-phases/parse.js'; +import type { StructureOutput } from '../../src/core/ingestion/pipeline-phases/structure.js'; +import type { + PhaseResult, + PipelineContext, +} from '../../src/core/ingestion/pipeline-phases/types.js'; + +const phaseResult = (phaseName: string, output: T): PhaseResult => ({ + phaseName, + output, + durationMs: 0, +}); + +describe('scopeResolutionPhase failure reconciliation', () => { + let repoDir = ''; + + afterEach(() => { + runScopeResolutionMock.mockReset(); + if (repoDir) fs.rmSync(repoDir, { recursive: true, force: true }); + }); + + it('retains a parse failure when the main-thread provider fallback also fails', async () => { + repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scope-phase-failure-')); + fs.writeFileSync(path.join(repoDir, 'broken.py'), 'def broken(:\n'); + + runScopeResolutionMock.mockReturnValue({ + filesProcessed: 0, + filesSkipped: 1, + scopeExtractionFailedPaths: ['broken.py'], + importsEmitted: 0, + resolve: { unresolved: 0 }, + referenceEdgesEmitted: 0, + referenceSkipped: 0, + propertyDispatchSkippedKeys: 0, + importedValueRefEdges: 0, + uniqueNamePropertyEdges: 0, + uniqueNamePropertyAmbiguous: 0, + uniqueNamePropertyNarrowed: 0, + uniqueNamePropertyAmbiguousNames: [], + uniqueNamePropertyCrossLanguage: 0, + uniqueNamePropertyCrossLanguageNames: [], + resolutionOutcomes: [], + undecidedSatisfaction: [], + functionSummaries: [], + callSummaries: [], + }); + + const graph = createKnowledgeGraph(); + const ctx: PipelineContext = { + repoPath: repoDir, + graph, + onProgress: () => {}, + pipelineStart: Date.now(), + }; + const structure: StructureOutput = { + scannedFiles: [{ path: 'broken.py', size: 13 }], + allPaths: ['broken.py'], + allPathSet: new Set(['broken.py']), + totalFiles: 1, + }; + const parse = { + model: createSemanticModel(), + parsedFiles: [], + scopeExtractionFailures: ['broken.py'], + } as unknown as ParseOutput; + const deps = new Map>([ + ['structure', phaseResult('structure', structure)], + ['parse', phaseResult('parse', parse)], + ['crossFile', phaseResult('crossFile', {})], + ]); + + const output = await scopeResolutionPhase.execute(ctx, deps); + + expect(runScopeResolutionMock).toHaveBeenCalledOnce(); + expect(output.scopeExtractionFailures).toEqual(['broken.py']); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/run-progress.test.ts b/gitnexus/test/unit/scope-resolution/run-progress.test.ts index 45bbfe9d7..6894f96d8 100644 --- a/gitnexus/test/unit/scope-resolution/run-progress.test.ts +++ b/gitnexus/test/unit/scope-resolution/run-progress.test.ts @@ -7,6 +7,7 @@ import { import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { createSemanticModel } from '../../../src/core/ingestion/model/semantic-model.js'; import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; +import type { LanguageProvider } from '../../../src/core/ingestion/language-provider.js'; const mkScope = (id: ScopeId, filePath: string): Scope => ({ id, @@ -40,6 +41,12 @@ const stubProvider = { propagatesReturnTypesAcrossImports: false, } as unknown as ScopeResolver; +const providerWithEmitter = (emitScopeCaptures: LanguageProvider['emitScopeCaptures']) => + ({ + ...stubProvider, + languageProvider: { emitScopeCaptures } as LanguageProvider, + }) as ScopeResolver; + describe('runScopeResolution onProgress', () => { it('emits sub-phases in order for a 3-file input', () => { const files = [ @@ -104,4 +111,44 @@ describe('runScopeResolution onProgress', () => { expect(stats.filesProcessed).toBe(0); expect(calls).toEqual([{ subPhase: 'extracting', current: 0, total: 0 }]); }); + + it('counts empty migrated files as skipped without recording extraction failures', () => { + const stats = runScopeResolution( + { + graph: createKnowledgeGraph(), + model: createSemanticModel(), + files: [ + { path: 'empty.py', content: '' }, + { path: 'whitespace.py', content: ' \n\t' }, + ], + }, + providerWithEmitter(() => { + throw new Error('empty files must short-circuit before capture'); + }), + ); + + expect(stats.filesProcessed).toBe(0); + expect(stats.filesSkipped).toBe(2); + expect(stats.scopeExtractionFailedPaths).toEqual([]); + }); + + it('records a warned emitter failure as a final extraction omission', () => { + const warnings: string[] = []; + const stats = runScopeResolution( + { + graph: createKnowledgeGraph(), + model: createSemanticModel(), + files: [{ path: 'broken.py', content: 'value = 1' }], + onWarn: (warning) => warnings.push(warning), + }, + providerWithEmitter(() => { + throw new Error('capture failed'); + }), + ); + + expect(stats.filesProcessed).toBe(0); + expect(stats.filesSkipped).toBe(1); + expect(stats.scopeExtractionFailedPaths).toEqual(['broken.py']); + expect(warnings).toEqual([expect.stringContaining('capture failed')]); + }); }); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 390757b22..9d996279a 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -64,6 +64,8 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + 'test/integration/impact-epistemic-lower-bound.test.ts', + 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', @@ -136,6 +138,8 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + 'test/integration/impact-epistemic-lower-bound.test.ts', + 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', From f64cc8b7a86e48c1d1027d176677e88a510d0c22 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Sat, 29 Aug 2026 08:39:09 +0100 Subject: [PATCH 2/3] feat(group): add GraphQL cross-repo contracts (#3070) * feat(group): add GraphQL contract extraction * fix(group): tighten GraphQL contract guards * fix(group): complete GraphQL review hardening * fix(group): isolate bounded GraphQL reads * fix(group): harden GraphQL contract extraction --- gitnexus/README.md | 15 + gitnexus/package-lock.json | 10 + gitnexus/package.json | 1 + gitnexus/scripts/cross-platform-tests.ts | 2 + gitnexus/src/core/group/PIPELINE.md | 22 +- gitnexus/src/core/group/config-parser.ts | 28 +- .../src/core/group/extractors/fs-utils.ts | 91 +++ .../group/extractors/graphql-extractor.ts | 707 ++++++++++++++++++ .../group/extractors/manifest-extractor.ts | 11 +- gitnexus/src/core/group/matching.ts | 8 +- gitnexus/src/core/group/storage.ts | 1 + gitnexus/src/core/group/sync.ts | 13 + gitnexus/src/core/group/types.ts | 19 +- .../src/core/ingestion/utils/symbol-labels.ts | 4 +- .../group/graphql-resolve-symbol.test.ts | 145 ++++ .../test/unit/group/config-parser.test.ts | 45 ++ gitnexus/test/unit/group/fs-utils.test.ts | 89 +++ .../test/unit/group/graphql-extractor.test.ts | 502 +++++++++++++ .../unit/group/manifest-label-drift.test.ts | 18 + gitnexus/test/unit/group/matching.test.ts | 18 + gitnexus/test/unit/group/sync.test.ts | 28 + gitnexus/test/unit/group/types.test.ts | 4 +- gitnexus/vitest.config.ts | 2 + 23 files changed, 1764 insertions(+), 19 deletions(-) create mode 100644 gitnexus/src/core/group/extractors/graphql-extractor.ts create mode 100644 gitnexus/test/integration/group/graphql-resolve-symbol.test.ts create mode 100644 gitnexus/test/unit/group/fs-utils.test.ts create mode 100644 gitnexus/test/unit/group/graphql-extractor.test.ts diff --git a/gitnexus/README.md b/gitnexus/README.md index e9b16ac73..e524eaa19 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -281,6 +281,21 @@ gitnexus group status # Check staleness of repos in a group gitnexus group impact --target --repo # Cross-repo blast radius ``` +GraphQL contract matching is opt-in in the group's `group.yaml`: + +```yaml +detect: + graphql: true +``` + +The initial exact-only slice matches methods and properties on top-level NestJS `@Resolver` +classes using imported `@Query`, `@Mutation`, and `@Subscription` decorators. Named +`.graphql`/`.gql` operations are anchored by generated `Document` declarations; +object, static `gql` template, and `TypedDocumentString` initializers must prove the operation name +and root fields. Dynamic decorator names, anonymous operations, and ambiguous or missing graph +anchors are deliberately omitted. Add common infrastructure fields such as `/health` to +`matching.exclude_links_paths` to keep those GraphQL contracts visible without cross-linking them. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. ## Remote Embeddings diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 08aa3a863..7188c7e1c 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -23,6 +23,7 @@ "graphology": "^0.26.0", "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", + "graphql": "^16.14.2", "ignore": "^7.0.5", "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", @@ -3338,6 +3339,15 @@ "graphology-types": ">=0.23.0" } }, + "node_modules/graphql": { + "version": "16.14.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", + "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, "node_modules/guid-typescript": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 87d83a090..5fccc65d1 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -69,6 +69,7 @@ "graphology": "^0.26.0", "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", + "graphql": "^16.14.2", "ignore": "^7.0.5", "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index f5feeeed5..d38672724 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -90,6 +90,7 @@ const PLATFORM_LOGIC = [ 'test/unit/ignore-service.test.ts', 'test/unit/group/bridge-db.test.ts', 'test/unit/group/bridge-db-edge.test.ts', + 'test/unit/group/fs-utils.test.ts', 'test/unit/onnxruntime-node-resolver.test.ts', // Windows cmd.exe arg-quoting + compose-and-spawn for the npm install (#2372): // the quoting rules and win32 single-string spawn shape are OS-sensitive, so @@ -140,6 +141,7 @@ const LBUG_NATIVE = [ // opens them through the pool adapter (native addon + bridge file locking). // Windows is skipped in-file (describeReopen) due to the bridge reopen lock. 'test/integration/group/cross-trace-e2e.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/local-backend.test.ts', 'test/integration/local-backend-calltool.test.ts', 'test/integration/search-core.test.ts', diff --git a/gitnexus/src/core/group/PIPELINE.md b/gitnexus/src/core/group/PIPELINE.md index 9a8c4b972..72961cca3 100644 --- a/gitnexus/src/core/group/PIPELINE.md +++ b/gitnexus/src/core/group/PIPELINE.md @@ -16,10 +16,12 @@ flowchart TD D --> E1[TopicExtractor] D --> E2[HttpRouteExtractor] D --> E3[GrpcExtractor] + D --> E4[GraphqlExtractor] E1 --> F[ExtractedContract array
per repo] E2 --> F E3 --> F + E4 --> F B --> M[ManifestExtractor] M --> G[Manifest contracts
+ cross-links] @@ -59,14 +61,27 @@ flowchart TD **Strategy A** (graph-assisted) uses Cypher over edges already produced by the main ingestion pipeline: + - HTTP: `HANDLES_ROUTE` / `FETCHES` edges from `(File)-[]->(Route)` - topic: none (pipeline doesn't yet produce topic nodes — Strategy B only) - gRPC: none (Strategy B + proto map only) -**Strategy B** (source-scan) is 100% tree-sitter based after this PR. +**Strategy B** (source-scan) uses tree-sitter for language source and the +official GraphQL parser for `.graphql` / `.gql` operation documents. Each `*-patterns/.ts` plugin owns its grammar + S-expression queries; the top-level orchestrator imports neither. +GraphQL detection is opt-in with `detect.graphql: true`. The initial slice +recognizes imported NestJS `Query`, `Mutation`, and `Subscription` decorators on +top-level imported `Resolver` classes, plus named operation documents. Generated +object documents, static `gql` templates, and `TypedDocumentString` values are +verified against the operation and its resolved root fragments. Providers and +consumers must resolve to one exact, real graph symbol; anonymous operations, +dynamic decorator names, ambiguous generated symbols, malformed documents, +symlink escapes, and bounded-parser overflows are skipped rather than linked +approximately. `matching.exclude_links_paths` also suppresses configured GraphQL +root fields from exact cross-linking while retaining their registry entries. + ## Plugin architecture ```mermaid @@ -117,6 +132,7 @@ They use the `MATCH (n) WHERE labels(n) IN [...]` allowlist form, NOT the `MATCH (n:A|B)` disjunction — LadybugDB's parser rejects a disjunction that names a reserved keyword (e.g. `Macro`, `Union`), which is what broke the `custom` branch in #2325: + - `topic` → `labels(n) IN ['Function','Method','Class','Interface']` - `grpc`/`thrift` method → `labels(n) IN ['Function','Method']`, service → `labels(n) IN ['Class','Interface']` - `lib` → `labels(n) IN ['Module']` @@ -144,7 +160,7 @@ without coordinating through any shared state. ## Cross-repo trace (`cross-trace.ts`) A second consumer of the bridge. Where cross-impact fans a blast radius -*outward* from one symbol, cross-trace stitches a directed **path** between +_outward_ from one symbol, cross-trace stitches a directed **path** between two symbols that live in different repos: ```mermaid @@ -158,7 +174,7 @@ flowchart TD ``` It reuses the same `symbolUid` join as cross-impact, but issues its own -*pair* query (`listCrossingsBetween`) because a path needs BOTH endpoints of +_pair_ query (`listCrossingsBetween`) because a path needs BOTH endpoints of a crossing — the uid-filtered neighbor join (`resolveBridgeNeighbors`, shared with impact) returns only the far side. The crossing is clamped to one boundary (`MAX_SUPPORTED_CROSS_DEPTH`). With `pdg: true` the boundary-adjacent diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index 754f578f9..b831c6706 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -1,10 +1,15 @@ import { createRequire } from 'node:module'; -import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from './types.js'; +import type { + ContractRole, + GroupConfig, + GroupManifestLink, + ManifestContractType, +} from './types.js'; const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -const VALID_CONTRACT_TYPES: ContractType[] = [ +const VALID_CONTRACT_TYPES: ManifestContractType[] = [ 'http', 'grpc', 'thrift', @@ -26,6 +31,7 @@ const VALID_ROLES: ContractRole[] = ['provider', 'consumer']; // repos that need cross-repo header tracking. const DEFAULT_DETECT = { http: true, + graphql: false, grpc: true, thrift: true, topics: true, @@ -66,7 +72,7 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { if (!link.to || !repoPaths.has(link.to as string)) { throw new Error(`links[${i}].to "${link.to}" does not match any repo path in group`); } - if (!VALID_CONTRACT_TYPES.includes(link.type as ContractType)) { + if (!VALID_CONTRACT_TYPES.includes(link.type as ManifestContractType)) { throw new Error( `links[${i}].type "${link.type}" is invalid. Expected: ${VALID_CONTRACT_TYPES.join(', ')}`, ); @@ -84,13 +90,25 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { return { from: link.from as string, to: link.to as string, - type: link.type as ContractType, + type: link.type as ManifestContractType, contract: String(link.contract), role: link.role as ContractRole, }; }); - const detect = { ...DEFAULT_DETECT, ...((raw.detect as object) || {}) }; + const rawDetect = raw.detect; + if ( + rawDetect !== undefined && + (!rawDetect || typeof rawDetect !== 'object' || Array.isArray(rawDetect)) + ) { + throw new Error('detect must be a mapping of boolean flags'); + } + for (const [key, value] of Object.entries((rawDetect as Record) || {})) { + if (key in DEFAULT_DETECT && typeof value !== 'boolean') { + throw new Error(`detect.${key} must be true or false`); + } + } + const detect = { ...DEFAULT_DETECT, ...((rawDetect as object) || {}) }; const matching = { ...DEFAULT_MATCHING, ...((raw.matching as object) || {}) }; const packages = (raw.packages as Record>) || {}; diff --git a/gitnexus/src/core/group/extractors/fs-utils.ts b/gitnexus/src/core/group/extractors/fs-utils.ts index 384f63203..7f02bbd1d 100644 --- a/gitnexus/src/core/group/extractors/fs-utils.ts +++ b/gitnexus/src/core/group/extractors/fs-utils.ts @@ -21,3 +21,94 @@ export function readSafe(repoPath: string, rel: string): string | null { return null; } } + +/** Read a regular in-repo file without buffering more than `maxBytes`. */ +export async function readSafeBounded( + repoPath: string, + rel: string, + maxBytes: number, +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) return null; + const abs = path.resolve(repoPath, rel); + const base = path.resolve(repoPath); + const relToBase = path.relative(base, abs); + if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; + + try { + const canonicalBase = await fs.promises.realpath(base); + const canonicalFile = await fs.promises.realpath(abs); + const canonicalRelative = path.relative(canonicalBase, canonicalFile); + if (canonicalRelative.startsWith('..') || path.isAbsolute(canonicalRelative)) return null; + const beforeOpen = await fs.promises.lstat(canonicalFile); + if (!beforeOpen.isFile() || beforeOpen.size > maxBytes) return null; + if (maxBytes === 0) return beforeOpen.size === 0 ? '' : null; + + return await new Promise((resolve) => { + const stream = fs.createReadStream(canonicalFile, { + flags: 'r', + start: 0, + end: maxBytes, + autoClose: true, + }); + const chunks: Buffer[] = []; + let totalBytes = 0; + let validated = false; + let settled = false; + + const finish = (value: string | null): void => { + if (settled) return; + settled = true; + resolve(value); + }; + + stream.pause(); + stream.once('open', (fd) => { + try { + const opened = fs.fstatSync(fd); + if (!opened.isFile() || opened.size > maxBytes) { + finish(null); + stream.destroy(); + return; + } + + const currentCanonical = fs.realpathSync(canonicalFile); + const currentRelative = path.relative(canonicalBase, currentCanonical); + const current = fs.statSync(currentCanonical); + if ( + currentRelative.startsWith('..') || + path.isAbsolute(currentRelative) || + opened.dev !== current.dev || + opened.ino !== current.ino + ) { + finish(null); + stream.destroy(); + return; + } + + validated = true; + stream.resume(); + } catch { + finish(null); + stream.destroy(); + } + }); + stream.on('data', (chunk: Buffer | string) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += bytes.length; + if (totalBytes > maxBytes) { + finish(null); + stream.destroy(); + return; + } + chunks.push(bytes); + }); + stream.once('end', () => { + finish(validated ? Buffer.concat(chunks, totalBytes).toString('utf8') : null); + }); + stream.once('error', () => finish(null)); + stream.once('close', () => finish(null)); + }); + } catch { + return null; + } +} diff --git a/gitnexus/src/core/group/extractors/graphql-extractor.ts b/gitnexus/src/core/group/extractors/graphql-extractor.ts new file mode 100644 index 000000000..efafe4e2e --- /dev/null +++ b/gitnexus/src/core/group/extractors/graphql-extractor.ts @@ -0,0 +1,707 @@ +import { glob } from 'glob'; +import { + Kind, + parse, + type DocumentNode, + type FragmentDefinitionNode, + type OperationDefinitionNode, + type SelectionSetNode, +} from 'graphql'; +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import { createIgnoreFilter } from '../../../config/ignore-service.js'; +import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { logger } from '../../logger.js'; +import { ParseTimeoutError, parseSourceSafe } from '../../tree-sitter/safe-parse.js'; +import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; +import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafeBounded } from './fs-utils.js'; + +const PROVIDER_GLOB = '**/*.{ts,tsx,mts,cts}'; +const DOCUMENT_GLOB = '**/*.{graphql,gql}'; +const NEST_GRAPHQL_PACKAGE = '@nestjs/graphql'; +const MAX_GRAPHQL_TOKENS = 100_000; +const MAX_GRAPHQL_DEFINITIONS = 5_000; +const MAX_GRAPHQL_OPERATIONS = 500; +const MAX_GRAPHQL_SELECTIONS = 10_000; +const MAX_GRAPHQL_TRAVERSAL_DEPTH = 64; +const MAX_PROVIDER_AST_NODES = 100_000; +const MAX_PROVIDER_AST_DEPTH = 256; +const GRAPHQL_NAME = /^[_A-Za-z][_0-9A-Za-z]*$/; + +type GraphqlOperationKind = 'query' | 'mutation' | 'subscription'; + +interface ResolvedSymbol { + uid: string; + name: string; + filePath: string; +} + +interface DecoratorBindings { + operations: Map; + resolvers: Set; +} + +type DecoratorFieldName = + | { kind: 'absent' } + | { kind: 'literal'; value: string } + | { kind: 'dynamic' }; + +type GeneratedSymbolIndex = Map; +type GeneratedIndexCache = Map>; + +export const RESOLVE_METHOD_QUERY = ` +MATCH (n) +WHERE labels(n) IN ['Method','Function','Property','CodeElement'] + AND n.name = $name AND n.filePath = $filePath AND n.startLine = $startLine AND n.id <> '' +RETURN n.id AS uid, n.name AS name, n.filePath AS filePath +ORDER BY n.id ASC +LIMIT 2`; + +// LadybugDB returns labels(n) as a scalar string, not Neo4j's string array. +// The real-db integration test executes this exact query and guards that dialect contract. +export const RESOLVE_GENERATED_SYMBOL_QUERY = ` +MATCH (n) +WHERE labels(n) IN ['Const','Variable','Function','Method','CodeElement'] + AND n.name = $name AND n.filePath <> '' AND n.id <> '' +RETURN n.id AS uid, n.name AS name, n.filePath AS filePath +ORDER BY n.id ASC +LIMIT 2`; + +function rowValue(row: Record, key: string, position: number): string { + return String(row[key] ?? row[position] ?? ''); +} + +function uniqueRealSymbol(rows: Record[]): ResolvedSymbol | null { + if (rows.length !== 1) return null; + const row = rows[0]; + const symbol = { + uid: rowValue(row, 'uid', 0), + name: rowValue(row, 'name', 1), + filePath: rowValue(row, 'filePath', 2).replace(/\\/g, '/'), + }; + return symbol.uid && symbol.name && symbol.filePath ? symbol : null; +} + +function unquote(text: string): string | null { + const trimmed = text.trim(); + if (trimmed.length < 2) return null; + const quote = trimmed[0]; + if ((quote !== "'" && quote !== '"' && quote !== '`') || trimmed.at(-1) !== quote) return null; + const value = trimmed.slice(1, -1); + return value.includes('${') ? null : value; +} + +function unwrapExpression(node: Parser.SyntaxNode): Parser.SyntaxNode { + let current = node; + while ( + ['as_expression', 'satisfies_expression', 'parenthesized_expression'].includes(current.type) && + current.namedChildren[0] + ) { + current = current.namedChildren[0]; + } + return current; +} + +function objectPairValue(node: Parser.SyntaxNode, key: string): Parser.SyntaxNode | null { + const object = unwrapExpression(node); + if (object.type !== 'object') return null; + for (const pair of object.namedChildren) { + if (pair.type !== 'pair') continue; + const keyNode = pair.childForFieldName('key'); + const pairKey = keyNode ? (unquote(keyNode.text) ?? keyNode.text) : null; + if (pairKey === key) return pair.childForFieldName('value'); + } + return null; +} + +function literalValue(node: Parser.SyntaxNode | null): string | null { + return node ? unquote(unwrapExpression(node).text) : null; +} + +function graphqlNameValue(node: Parser.SyntaxNode | null): string | null { + return node ? literalValue(objectPairValue(node, 'value')) : null; +} + +function withinGeneratedAstBudget(root: Parser.SyntaxNode): boolean { + const pending: Array<{ node: Parser.SyntaxNode; depth: number }> = [{ node: root, depth: 0 }]; + let visited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + visited++; + if (visited > MAX_PROVIDER_AST_NODES || current.depth > MAX_PROVIDER_AST_DEPTH) return false; + for (let index = current.node.namedChildren.length - 1; index >= 0; index--) { + const child = current.node.namedChildren[index]; + if (child) pending.push({ node: child, depth: current.depth + 1 }); + } + } + return true; +} + +function generatedRootFields( + selectionSet: Parser.SyntaxNode | null, + fragments: ReadonlyMap, +): Set | null { + const fields = new Set(); + if (!selectionSet) return null; + const seenFragments = new Set(); + const pending: Array<{ selectionSet: Parser.SyntaxNode; depth: number }> = [ + { selectionSet, depth: 0 }, + ]; + let selectionsVisited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + const selections = objectPairValue(current.selectionSet, 'selections'); + const array = selections ? unwrapExpression(selections) : null; + if (!array || array.type !== 'array') return null; + for (const item of array.namedChildren) { + selectionsVisited++; + if (selectionsVisited > MAX_GRAPHQL_SELECTIONS) return null; + const selection = unwrapExpression(item); + const kind = literalValue(objectPairValue(selection, 'kind')); + if (kind === 'Field') { + const field = graphqlNameValue(objectPairValue(selection, 'name')); + if (field) fields.add(field); + continue; + } + if (kind === 'InlineFragment') { + const nested = objectPairValue(selection, 'selectionSet'); + if (nested) pending.push({ selectionSet: nested, depth: current.depth + 1 }); + continue; + } + if (kind !== 'FragmentSpread') continue; + const name = graphqlNameValue(objectPairValue(selection, 'name')); + if (!name || seenFragments.has(name)) continue; + const fragment = fragments.get(name); + if (!fragment) continue; + const nested = objectPairValue(fragment, 'selectionSet'); + if (!nested) continue; + seenFragments.add(name); + pending.push({ selectionSet: nested, depth: current.depth + 1 }); + } + } + return fields; +} + +function parsedDocumentProof( + source: string, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], +): boolean { + let document: DocumentNode; + try { + document = parse(source, { noLocation: true, maxTokens: MAX_GRAPHQL_TOKENS }); + } catch { + return false; + } + if (document.definitions.length > MAX_GRAPHQL_DEFINITIONS) return false; + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind === Kind.FRAGMENT_DEFINITION) + fragments.set(definition.name.value, definition); + } + for (const definition of document.definitions) { + if (definition.kind !== Kind.OPERATION_DEFINITION) continue; + if (definition.operation !== operationKind || definition.name?.value !== operationName) + continue; + const fields = rootFields(definition.selectionSet, fragments); + return fields !== null && requiredFields.every((field) => fields.includes(field)); + } + return false; +} + +function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { + const value = unwrapExpression(initializer); + if (value.type === 'string') { + if (value.text.startsWith('"')) { + try { + return JSON.parse(value.text) as string; + } catch { + return null; + } + } + return unquote(value.text); + } + if (value.type === 'template_string') return unquote(value.text); + + if (value.type === 'call_expression') { + const template = value.namedChildren.find((child) => child.type === 'template_string'); + return template ? unquote(template.text) : null; + } + + if (value.type !== 'new_expression') return null; + const constructor = value.childForFieldName('constructor') ?? value.namedChildren[0]; + if (!constructor || !constructor.text.endsWith('TypedDocumentString')) return null; + const args = value.childForFieldName('arguments'); + const first = args?.namedChildren[0]; + return first ? staticGraphqlSource(first) : null; +} + +export function hasGeneratedDocumentProof( + initializer: Parser.SyntaxNode, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], +): boolean { + if (!withinGeneratedAstBudget(initializer)) return false; + const staticSource = staticGraphqlSource(initializer); + if (staticSource !== null) { + return parsedDocumentProof(staticSource, operationKind, operationName, requiredFields); + } + const document = unwrapExpression(initializer); + if (literalValue(objectPairValue(document, 'kind')) !== 'Document') return false; + const definitions = objectPairValue(document, 'definitions'); + const array = definitions ? unwrapExpression(definitions) : null; + if (!array || array.type !== 'array') return false; + + const fragments = new Map(); + for (const item of array.namedChildren) { + const definition = unwrapExpression(item); + if (literalValue(objectPairValue(definition, 'kind')) !== 'FragmentDefinition') continue; + const name = graphqlNameValue(objectPairValue(definition, 'name')); + if (name) fragments.set(name, definition); + } + + for (const item of array.namedChildren) { + const definition = unwrapExpression(item); + if (literalValue(objectPairValue(definition, 'kind')) !== 'OperationDefinition') continue; + if (literalValue(objectPairValue(definition, 'operation')) !== operationKind) continue; + if (graphqlNameValue(objectPairValue(definition, 'name')) !== operationName) continue; + const fields = generatedRootFields(objectPairValue(definition, 'selectionSet'), fragments); + if (fields && requiredFields.every((field) => fields.has(field))) return true; + } + return false; +} + +function importedDecoratorBindings(root: Parser.SyntaxNode): DecoratorBindings { + const operations = new Map(); + const resolvers = new Set(); + for (const child of root.namedChildren) { + if (child.type !== 'import_statement') continue; + const source = child.childForFieldName('source'); + if (!source || unquote(source.text) !== NEST_GRAPHQL_PACKAGE) continue; + + const namedImports = child.namedChildren + .find((node) => node.type === 'import_clause') + ?.namedChildren.find((node) => node.type === 'named_imports'); + if (!namedImports) continue; + + for (const specifier of namedImports.namedChildren) { + if (specifier.type !== 'import_specifier') continue; + const imported = specifier.childForFieldName('name')?.text; + const local = specifier.childForFieldName('alias')?.text ?? imported; + if (!imported || !local) continue; + const kind = imported.toLowerCase(); + if (kind === 'query' || kind === 'mutation' || kind === 'subscription') { + operations.set(local, kind); + } else if (imported === 'Resolver') { + resolvers.add(local); + } + } + } + return { operations, resolvers }; +} + +function decoratorKind( + decorator: Parser.SyntaxNode, + bindings: Map, +): { kind: GraphqlOperationKind; argumentsNode?: Parser.SyntaxNode } | null { + const expression = decorator.namedChildren[0]; + if (!expression) return null; + if (expression.type === 'identifier') { + const kind = bindings.get(expression.text); + return kind ? { kind } : null; + } + if (expression.type !== 'call_expression') return null; + const callee = expression.childForFieldName('function'); + if (!callee || callee.type !== 'identifier') return null; + const kind = bindings.get(callee.text); + if (!kind) return null; + return { kind, argumentsNode: expression.childForFieldName('arguments') ?? undefined }; +} + +function decoratorFieldName(argumentsNode: Parser.SyntaxNode | undefined): DecoratorFieldName { + if (!argumentsNode || argumentsNode.namedChildren.length === 0) return { kind: 'absent' }; + const args = argumentsNode.namedChildren; + if (args[0] && ['string', 'template_string'].includes(args[0].type)) { + const direct = unquote(args[0].text); + return direct === null ? { kind: 'dynamic' } : { kind: 'literal', value: direct }; + } + + let sawOptions = false; + + for (const arg of args) { + if (arg.type !== 'object') continue; + sawOptions = true; + for (const pair of arg.namedChildren) { + if (pair.type === 'spread_element' || pair.type.startsWith('shorthand_property_identifier')) { + return { kind: 'dynamic' }; + } + if (pair.type !== 'pair') continue; + const key = pair.childForFieldName('key')?.text.replace(/^['"]|['"]$/g, ''); + if (key !== 'name') continue; + const value = pair.childForFieldName('value'); + if (!value || !['string', 'template_string'].includes(value.type)) { + return { kind: 'dynamic' }; + } + const literal = unquote(value.text); + return literal === null ? { kind: 'dynamic' } : { kind: 'literal', value: literal }; + } + } + if (sawOptions || args.length === 1) return { kind: 'absent' }; + return { kind: 'dynamic' }; +} + +function topLevelResolverClassBodies( + root: Parser.SyntaxNode, + resolverBindings: ReadonlySet, +): Parser.SyntaxNode[] | null { + if (!withinGeneratedAstBudget(root)) return null; + const bodies: Parser.SyntaxNode[] = []; + for (const statement of root.namedChildren) { + const classNode = + statement.type === 'class_declaration' + ? statement + : statement.type === 'export_statement' + ? statement.namedChildren.find((child) => child.type === 'class_declaration') + : undefined; + if (!classNode) continue; + const decorators = [ + ...new Set([ + ...statement.namedChildren.filter((child) => child.type === 'decorator'), + ...classNode.namedChildren.filter((child) => child.type === 'decorator'), + ]), + ]; + const isResolver = decorators.some((decorator) => { + const expression = decorator.namedChildren[0]; + if (!expression) return false; + const callee = + expression.type === 'call_expression' + ? expression.childForFieldName('function') + : expression; + return callee?.type === 'identifier' && resolverBindings.has(callee.text); + }); + if (!isResolver) continue; + const body = classNode.childForFieldName('body'); + if (body) bodies.push(body); + } + return bodies; +} + +function rootFields( + selectionSet: SelectionSetNode, + fragments: ReadonlyMap, +): string[] | null { + const fields: string[] = []; + const seenFragments = new Set(); + const pending: Array<{ selectionSet: SelectionSetNode; depth: number }> = [ + { selectionSet, depth: 0 }, + ]; + let selectionsVisited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + for (const selection of current.selectionSet.selections) { + selectionsVisited++; + if (selectionsVisited > MAX_GRAPHQL_SELECTIONS) return null; + if (selection.kind === Kind.FIELD) { + fields.push(selection.name.value); + continue; + } + if (selection.kind === Kind.INLINE_FRAGMENT) { + pending.push({ selectionSet: selection.selectionSet, depth: current.depth + 1 }); + continue; + } + const name = selection.name.value; + if (seenFragments.has(name)) continue; + const fragment = fragments.get(name); + if (!fragment) continue; + seenFragments.add(name); + pending.push({ selectionSet: fragment.selectionSet, depth: current.depth + 1 }); + } + } + return fields; +} + +function generatedCandidates(operation: OperationDefinitionNode): string[] { + const name = operation.name?.value; + return name ? [`${name}Document`] : []; +} + +async function generatedDocumentMatches( + repoPath: string, + symbol: ResolvedSymbol, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], + cache: GeneratedIndexCache, +): Promise { + const normalizedPath = symbol.filePath.replace(/\\/g, '/'); + let pendingIndex = cache.get(normalizedPath); + if (!pendingIndex) { + pendingIndex = buildGeneratedSymbolIndex(repoPath, normalizedPath); + cache.set(normalizedPath, pendingIndex); + } + const index = await pendingIndex; + const values = index?.get(symbol.name) ?? []; + return values.some((value) => + hasGeneratedDocumentProof(value, operationKind, operationName, requiredFields), + ); +} + +async function buildGeneratedSymbolIndex( + repoPath: string, + filePath: string, +): Promise { + const source = await readSafeBounded(repoPath, filePath, getMaxFileSizeBytes()); + if (source === null) return null; + const parser = new Parser(); + parser.setLanguage( + filePath.toLowerCase().endsWith('.tsx') ? TypeScript.tsx : TypeScript.typescript, + ); + let tree: Parser.Tree; + try { + tree = parseSourceSafe(parser, source, undefined, undefined, filePath); + } catch (error) { + if (error instanceof ParseTimeoutError) return null; + throw error; + } + + return indexGeneratedDeclarators(tree.rootNode); +} + +export function indexGeneratedDeclarators(root: Parser.SyntaxNode): GeneratedSymbolIndex { + const index: GeneratedSymbolIndex = new Map(); + const pending = [root]; + while (pending.length > 0) { + const node = pending.pop(); + if (!node) break; + if (node.type === 'variable_declarator') { + const name = node.childForFieldName('name')?.text; + const value = node.childForFieldName('value'); + if (name && value) { + const values = index.get(name) ?? []; + values.push(value); + index.set(name, values); + } + } + for (let child = node.namedChildren.length - 1; child >= 0; child--) { + pending.push(node.namedChildren[child]); + } + } + return index; +} + +function dedupe(contracts: ExtractedContract[]): ExtractedContract[] { + const seen = new Set(); + return contracts.filter((contract) => { + const key = `${contract.contractId}|${contract.role}|${contract.symbolUid}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +export class GraphqlExtractor implements ContractExtractor { + type = 'graphql' as const; + + async canExtract(_repo: RepoHandle): Promise { + return true; + } + + async extract( + dbExecutor: CypherExecutor | null, + repoPath: string, + _repo: RepoHandle, + ): Promise { + if (!dbExecutor) return []; + const ignore = await createIgnoreFilter(repoPath); + const [providerFiles, documentFiles] = await Promise.all([ + glob(PROVIDER_GLOB, { cwd: repoPath, ignore, nodir: true }), + glob(DOCUMENT_GLOB, { cwd: repoPath, ignore, nodir: true }), + ]); + const contracts = [ + ...(await this.extractProviders(dbExecutor, repoPath, providerFiles)), + ...(await this.extractConsumers(dbExecutor, repoPath, documentFiles)), + ]; + return dedupe(contracts); + } + + private async extractProviders( + dbExecutor: CypherExecutor, + repoPath: string, + files: string[], + ): Promise { + const parser = new Parser(); + const contracts: ExtractedContract[] = []; + const maxFileSizeBytes = getMaxFileSizeBytes(); + for (const rel of files) { + if (/\.(?:spec|test)\.[cm]?tsx?$/i.test(rel)) continue; + const source = await readSafeBounded(repoPath, rel, maxFileSizeBytes); + if (source === null || !source.includes(NEST_GRAPHQL_PACKAGE)) continue; + parser.setLanguage( + rel.toLowerCase().endsWith('.tsx') ? TypeScript.tsx : TypeScript.typescript, + ); + let tree: Parser.Tree; + try { + tree = parseSourceSafe(parser, source, undefined, undefined, rel); + } catch (error) { + if (error instanceof ParseTimeoutError) continue; + throw error; + } + const bindings = importedDecoratorBindings(tree.rootNode); + if (bindings.operations.size === 0 || bindings.resolvers.size === 0) continue; + const bodies = topLevelResolverClassBodies(tree.rootNode, bindings.resolvers); + if (bodies === null) continue; + for (const body of bodies) { + let decorators: Parser.SyntaxNode[] = []; + for (const member of body.namedChildren) { + if (member.type === 'comment') continue; + if (member.type === 'decorator') { + decorators.push(member); + continue; + } + if (member.type !== 'method_definition' && member.type !== 'public_field_definition') { + decorators = []; + continue; + } + const memberDecorators = [ + ...new Set([ + ...decorators, + ...member.namedChildren.filter((child) => child.type === 'decorator'), + ]), + ]; + const methodName = member.childForFieldName('name')?.text; + if (!methodName) { + decorators = []; + continue; + } + for (const decorator of memberDecorators) { + const operation = decoratorKind(decorator, bindings.operations); + if (!operation) continue; + const parsedField = decoratorFieldName(operation.argumentsNode); + if (parsedField.kind === 'dynamic') continue; + const field = parsedField.kind === 'literal' ? parsedField.value : methodName; + if (!GRAPHQL_NAME.test(field)) continue; + const filePath = rel.replace(/\\/g, '/'); + const symbol = uniqueRealSymbol( + await dbExecutor(RESOLVE_METHOD_QUERY, { + name: methodName, + filePath, + startLine: + member.type === 'public_field_definition' + ? (member.childForFieldName('value')?.startPosition.row ?? + member.startPosition.row) + 1 + : member.startPosition.row + 1, + }), + ); + if (!symbol) continue; + contracts.push({ + contractId: `graphql::${operation.kind}::${field}`, + type: 'graphql', + role: 'provider', + symbolUid: symbol.uid, + symbolRef: { filePath: symbol.filePath, name: symbol.name }, + symbolName: symbol.name, + confidence: 1, + meta: { + operationKind: operation.kind, + fieldName: field, + resolverPath: filePath, + extractionStrategy: 'nestjs_decorator', + }, + }); + } + decorators = []; + } + } + } + return contracts; + } + + private async extractConsumers( + dbExecutor: CypherExecutor, + repoPath: string, + files: string[], + ): Promise { + const contracts: ExtractedContract[] = []; + const generatedIndexCache: GeneratedIndexCache = new Map(); + const maxFileSizeBytes = getMaxFileSizeBytes(); + for (const rel of files) { + const source = await readSafeBounded(repoPath, rel, maxFileSizeBytes); + if (source === null) continue; + let document: DocumentNode; + try { + document = parse(source, { noLocation: true, maxTokens: MAX_GRAPHQL_TOKENS }); + } catch (error) { + logger.debug({ file: rel, error }, 'skipping invalid GraphQL document'); + continue; + } + if (document.definitions.length > MAX_GRAPHQL_DEFINITIONS) continue; + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind === Kind.FRAGMENT_DEFINITION) { + fragments.set(definition.name.value, definition); + } + } + const operations = document.definitions.filter( + (definition): definition is OperationDefinitionNode => + definition.kind === Kind.OPERATION_DEFINITION && definition.name !== undefined, + ); + if (operations.length > MAX_GRAPHQL_OPERATIONS) continue; + for (const definition of operations) { + const operationName = definition.name?.value; + if (!operationName) continue; + const documentPath = rel.replace(/\\/g, '/'); + const operationFields = rootFields(definition.selectionSet, fragments); + if (operationFields === null) continue; + const uniqueFields = [...new Set(operationFields)]; + let symbol: ResolvedSymbol | null = null; + for (const candidate of generatedCandidates(definition)) { + const resolved = uniqueRealSymbol( + await dbExecutor(RESOLVE_GENERATED_SYMBOL_QUERY, { name: candidate }), + ); + if ( + resolved && + (await generatedDocumentMatches( + repoPath, + resolved, + definition.operation, + operationName, + uniqueFields, + generatedIndexCache, + )) + ) { + symbol = resolved; + break; + } + } + if (!symbol) continue; + for (const field of uniqueFields) { + contracts.push({ + contractId: `graphql::${definition.operation}::${field}`, + type: 'graphql', + role: 'consumer', + symbolUid: symbol.uid, + symbolRef: { filePath: symbol.filePath, name: symbol.name }, + symbolName: symbol.name, + confidence: 1, + meta: { + operationKind: definition.operation, + operationName: definition.name.value, + fieldName: field, + documentPath, + extractionStrategy: 'graphql_ast', + }, + }); + } + } + } + return contracts; + } +} diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 272c27ce6..d4175442a 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -1,4 +1,9 @@ -import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js'; +import type { + CrossLink, + GroupManifestLink, + ManifestContractType, + StoredContract, +} from '../types.js'; import type { CypherExecutor } from '../contract-extractor.js'; import { logger } from '../../logger.js'; @@ -366,11 +371,11 @@ export class ManifestExtractor { * equality matching without requiring wildcard logic downstream. * * NOTE on exhaustiveness: the switch covers every current - * `ContractType` variant and falls through to a `never` assertion so + * manifest-declared contract type and falls through to a `never` assertion so * TypeScript fails the build if a new variant is added without a * corresponding case. */ - private buildContractId(type: ContractType, contract: string): string { + private buildContractId(type: ManifestContractType, contract: string): string { switch (type) { case 'http': { // Canonicalize method casing and path separators so logically diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index eea6fc102..2647cc009 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -37,7 +37,13 @@ function buildNoisyContractFilter( : new Set(); const excludeParamOnly = matchingConfig?.exclude_links_param_only_paths === true; - return function isNoisyHttpContract(contractId: string): boolean { + return function isNoisyContract(contractId: string): boolean { + if (contractId.startsWith('graphql::')) { + const parts = contractId.split('::'); + if (parts.length < 3) return false; + const field = parts.slice(2).join('::'); + return excludePaths.has(field) || excludePaths.has(`/${field}`); + } if (!contractId.startsWith('http::')) return false; const parts = contractId.split('::'); if (parts.length < 3) return false; diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index 6e82fbb0a..e196095ef 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -96,6 +96,7 @@ packages: {} detect: http: true + graphql: false grpc: true topics: true diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 7efd65ef7..d79cd4de5 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -22,6 +22,7 @@ import type { MatchType, } from './types.js'; import { HttpRouteExtractor } from './extractors/http-route-extractor.js'; +import { GraphqlExtractor } from './extractors/graphql-extractor.js'; import { GrpcExtractor } from './extractors/grpc-extractor.js'; import { ThriftExtractor } from './extractors/thrift-extractor.js'; import { TopicExtractor } from './extractors/topic-extractor.js'; @@ -295,6 +296,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis const entries = registryEntries; const resolve = opts?.resolveRepoHandle ?? defaultResolveHandle(entries); const httpEx = new HttpRouteExtractor(); + const graphqlEx = new GraphqlExtractor(); const grpcEx = new GrpcExtractor(); const thriftEx = new ThriftExtractor(); const topicEx = new TopicExtractor(); @@ -343,6 +345,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis } } + if (config.detect.graphql === true) { + const extracted = await graphqlEx.extract(executor, handle.repoPath, handle); + for (const c of extracted) { + repoContracts.push({ + ...c, + repo: groupPath, + service: assignService(c.symbolRef.filePath, boundaries), + }); + } + } + if (config.detect.grpc) { const extracted = await grpcEx.extract(executor, handle.repoPath, handle); for (const c of extracted) { diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index cf7191664..beeb6f053 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,4 +1,13 @@ -export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include'; +export type ContractType = + | 'http' + | 'graphql' + | 'grpc' + | 'thrift' + | 'topic' + | 'lib' + | 'custom' + | 'include'; +export type ManifestContractType = Exclude; export type MatchType = 'exact' | 'manifest' | 'wildcard'; export type ContractRole = 'provider' | 'consumer'; @@ -16,13 +25,14 @@ export interface GroupConfig { export interface GroupManifestLink { from: string; to: string; - type: ContractType; + type: ManifestContractType; contract: string; role: ContractRole; } export interface DetectConfig { http: boolean; + graphql?: boolean; grpc: boolean; thrift: boolean; topics: boolean; @@ -32,11 +42,12 @@ export interface DetectConfig { export interface MatchingConfig { /** - * HTTP paths to exclude from cross-link matching. Contracts at these paths + * HTTP paths or GraphQL root fields to exclude from cross-link matching. Contracts at these paths * are still extracted and visible in the registry, but they don't produce * cross-repo links. Useful for health-check endpoints (`/ping`, `/health`) * that every service exposes and would otherwise create N×M false links. - * Trailing slashes are normalized before comparison. + * Trailing slashes are normalized before comparison. GraphQL fields may be + * written as `health` or `/health`. * @default [] */ exclude_links_paths?: string[]; diff --git a/gitnexus/src/core/ingestion/utils/symbol-labels.ts b/gitnexus/src/core/ingestion/utils/symbol-labels.ts index a21df10b6..0751324dc 100644 --- a/gitnexus/src/core/ingestion/utils/symbol-labels.ts +++ b/gitnexus/src/core/ingestion/utils/symbol-labels.ts @@ -13,8 +13,8 @@ import type { NodeLabel } from 'gitnexus-shared'; * Single source of truth so the set can't silently drift the way the inline copy * did in #2379. * - * NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY` - * carries a near-identical hand-list that is intentionally a SUBSET — it excludes + * NOTE: group extractor queries in `manifest-extractor.ts` and `graphql-extractor.ts` + * carry near-identical hand-lists that are intentionally SUBSETS — they exclude * `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution * behavior check (would widen which nodes resolve as contract symbols), so it is * deliberately left separate for now. diff --git a/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts new file mode 100644 index 000000000..283f68842 --- /dev/null +++ b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts @@ -0,0 +1,145 @@ +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { afterAll, expect, it, vi } from 'vitest'; +import { GraphqlExtractor } from '../../../src/core/group/extractors/graphql-extractor.js'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import { closeLbug, executeParameterized } from '../../../src/core/lbug/pool-adapter.js'; +import type { GroupConfig, RepoHandle } from '../../../src/core/group/types.js'; +import { withTestLbugDB } from '../../helpers/test-indexed-db.js'; + +const SEED = [ + `CREATE (:Method {id:'method:health', name:'health', filePath:'src/health.resolver.ts', startLine:5, endLine:5, content:'', description:''})`, + `CREATE (:Const {id:'const:health-document', name:'HealthDocument', filePath:'src/generated.ts', startLine:1, endLine:1, content:'', description:''})`, +]; + +withTestLbugDB( + 'graphql-resolve-symbol', + (handle) => { + let providerRoot = ''; + let consumerRoot = ''; + + afterAll(async () => { + try { + await closeLbug(handle.repoId); + } catch { + /* best-effort */ + } + }); + + it('anchors provider and generated Document consumer through real LadybugDB queries', async () => { + providerRoot = path.join(handle.tmpHandle.dbPath, 'provider-repo'); + consumerRoot = path.join(handle.tmpHandle.dbPath, 'consumer-repo'); + await fs.mkdir(path.join(providerRoot, 'src'), { recursive: true }); + await fs.mkdir(path.join(consumerRoot, 'src'), { recursive: true }); + await fs.writeFile( + path.join(providerRoot, 'src/health.resolver.ts'), + `import { Query, Resolver } from '@nestjs/graphql';\n@Resolver()\nclass HealthResolver {\n @Query()\n health() { return 'ok'; }\n}`, + 'utf8', + ); + await fs.writeFile( + path.join(consumerRoot, 'src/health.graphql'), + 'query Health { health }', + 'utf8', + ); + await fs.writeFile( + path.join(consumerRoot, 'src/generated.ts'), + `export const HealthDocument = { + kind: 'Document', + definitions: [{ + kind: 'OperationDefinition', + operation: 'query', + name: { kind: 'Name', value: 'Health' }, + selectionSet: { + kind: 'SelectionSet', + selections: [{ kind: 'Field', name: { kind: 'Name', value: 'health' } }] + } + }] +};`, + 'utf8', + ); + const providerRepo: RepoHandle = { + id: handle.repoId, + path: 'api', + repoPath: providerRoot, + storagePath: handle.tmpHandle.dbPath, + }; + const consumerRepo: RepoHandle = { + id: handle.repoId, + path: 'web', + repoPath: consumerRoot, + storagePath: handle.tmpHandle.dbPath, + }; + + const execute = (query: string, params: Record = {}) => + executeParameterized(handle.repoId, query, params); + const contracts = [ + ...(await new GraphqlExtractor().extract(execute, providerRoot, providerRepo)), + ...(await new GraphqlExtractor().extract(execute, consumerRoot, consumerRepo)), + ]; + + expect(contracts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contractId: 'graphql::query::health', + role: 'provider', + symbolUid: 'method:health', + }), + expect.objectContaining({ + contractId: 'graphql::query::health', + role: 'consumer', + symbolUid: 'const:health-document', + }), + ]), + ); + + const repoManager = await import('../../../src/storage/repo-manager.js'); + const registrySpy = vi.spyOn(repoManager, 'readRegistryStrict').mockResolvedValue([]); + const config: GroupConfig = { + version: 1, + name: 'graphql-production-wiring', + description: '', + repos: { api: 'api', web: 'web' }, + links: [], + packages: {}, + detect: { + http: false, + graphql: true, + grpc: false, + thrift: false, + topics: false, + includes: false, + workspace_deps: false, + }, + matching: {}, + }; + try { + const synced = await syncGroup(config, { + resolveRepoHandle: async (_regName, groupPath) => + groupPath === 'api' ? providerRepo : consumerRepo, + skipWrite: true, + }); + expect( + synced.contracts.map((contract) => [ + contract.contractId, + contract.role, + contract.symbolUid, + ]), + ).toEqual([ + ['graphql::query::health', 'provider', 'method:health'], + ['graphql::query::health', 'consumer', 'const:health-document'], + ]); + expect(synced.crossLinks).toEqual([ + expect.objectContaining({ + contractId: 'graphql::query::health', + matchType: 'exact', + from: expect.objectContaining({ repo: 'web', symbolUid: 'const:health-document' }), + to: expect.objectContaining({ repo: 'api', symbolUid: 'method:health' }), + }), + ]); + } finally { + registrySpy.mockRestore(); + } + }); + }, + { seed: SEED, poolAdapter: true }, +); diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index 2ede066f4..9644b304b 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -45,6 +45,7 @@ describe('parseGroupConfig', () => { expect(config.packages['hr/common'].npm).toBe('@hr/common'); expect(config.detect.http).toBe(true); expect(config.detect.grpc).toBe(false); + expect(config.detect.graphql).toBe(false); }); it('applies defaults for missing optional fields', () => { @@ -175,6 +176,50 @@ detect: }); }); + describe('detect.graphql opt-in default', () => { + it('defaults GraphQL extraction to false', () => { + const config = parseGroupConfig(`version: 1\nname: test\nrepos: { app: my-app }\n`); + expect(config.detect.graphql).toBe(false); + }); + + it('honors explicit GraphQL extraction', () => { + const config = parseGroupConfig( + `version: 1\nname: test\nrepos: { app: my-app }\ndetect:\n graphql: true\n`, + ); + expect(config.detect.graphql).toBe(true); + }); + + it('rejects string-like detect booleans instead of silently changing behavior', () => { + expect(() => + parseGroupConfig( + `version: 1\nname: test\nrepos: { app: my-app }\ndetect:\n graphql: yes\n`, + ), + ).toThrow(/detect\.graphql must be true or false/i); + expect(() => + parseGroupConfig( + `version: 1\nname: test\nrepos: { app: my-app }\ndetect:\n http: "false"\n`, + ), + ).toThrow(/detect\.http must be true or false/i); + }); + + it('rejects GraphQL manifest links until they can resolve real endpoint symbols', () => { + const yaml = ` +version: 1 +name: test +repos: + web: web-repo + api: api-repo +links: + - from: web + to: api + type: graphql + contract: query::health + role: consumer +`; + expect(() => parseGroupConfig(yaml)).toThrow(/type "graphql" is invalid/i); + }); + }); + it('parses thrift manifest links', () => { const yaml = ` version: 1 diff --git a/gitnexus/test/unit/group/fs-utils.test.ts b/gitnexus/test/unit/group/fs-utils.test.ts new file mode 100644 index 000000000..7f2f45e88 --- /dev/null +++ b/gitnexus/test/unit/group/fs-utils.test.ts @@ -0,0 +1,89 @@ +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { readSafe, readSafeBounded } from '../../../src/core/group/extractors/fs-utils.js'; +import { cleanupTempDir } from '../../helpers/test-db.js'; + +const tempDirs: string[] = []; + +describe('group extractor readSafe', () => { + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => cleanupTempDir(dir))); + }); + + it('rejects a path whose canonical target escapes through a directory symlink', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-outside-')); + tempDirs.push(repo, outside); + await fs.writeFile(path.join(outside, 'secret.graphql'), 'query Secret { secret }', 'utf8'); + await fs.symlink( + outside, + path.join(repo, 'linked'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + + await expect(readSafeBounded(repo, 'linked/secret.graphql', 1024)).resolves.toBeNull(); + }); + + it('reads a regular file within the canonical repository root', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + await fs.writeFile(path.join(repo, 'schema.graphql'), 'query Health { health }', 'utf8'); + + expect(readSafe(repo, 'schema.graphql')).toBe('query Health { health }'); + await expect(readSafeBounded(repo, 'schema.graphql', 1024)).resolves.toBe( + 'query Health { health }', + ); + }); + + it('rejects an oversized sparse file before reading its contents', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + const file = await fs.open(path.join(repo, 'oversized.graphql'), 'w'); + try { + await file.truncate(16 * 1024 * 1024); + } finally { + await file.close(); + } + + await expect(readSafeBounded(repo, 'oversized.graphql', 1024)).resolves.toBeNull(); + }); + + it('accepts a regular file whose size is exactly maxBytes', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + await fs.writeFile(path.join(repo, 'exact.graphql'), '12345678', 'utf8'); + + await expect(readSafeBounded(repo, 'exact.graphql', 8)).resolves.toBe('12345678'); + await expect(readSafeBounded(repo, 'exact.graphql', 7)).resolves.toBeNull(); + }); + + it.skipIf(process.platform === 'win32')( + 'reads a final-file symlink whose canonical target stays inside the repository', + async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + tempDirs.push(repo); + await fs.writeFile(path.join(repo, 'schema.graphql'), 'query Health { health }', 'utf8'); + await fs.symlink('schema.graphql', path.join(repo, 'schema-link.graphql'), 'file'); + + await expect(readSafeBounded(repo, 'schema-link.graphql', 1024)).resolves.toBe( + 'query Health { health }', + ); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'rejects a final-file symlink whose canonical target escapes the repository', + async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-repo-')); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-readsafe-outside-')); + tempDirs.push(repo, outside); + const secret = path.join(outside, 'secret.graphql'); + await fs.writeFile(secret, 'query Secret { secret }', 'utf8'); + await fs.symlink(secret, path.join(repo, 'secret-link.graphql'), 'file'); + + await expect(readSafeBounded(repo, 'secret-link.graphql', 1024)).resolves.toBeNull(); + }, + ); +}); diff --git a/gitnexus/test/unit/group/graphql-extractor.test.ts b/gitnexus/test/unit/group/graphql-extractor.test.ts new file mode 100644 index 000000000..5e6d69505 --- /dev/null +++ b/gitnexus/test/unit/group/graphql-extractor.test.ts @@ -0,0 +1,502 @@ +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { parse } from 'graphql'; +import Parser from 'tree-sitter'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CypherExecutor } from '../../../src/core/group/contract-extractor.js'; +import { + GraphqlExtractor, + indexGeneratedDeclarators, +} from '../../../src/core/group/extractors/graphql-extractor.js'; +import type { RepoHandle } from '../../../src/core/group/types.js'; +import { cleanupTempDir } from '../../helpers/test-db.js'; + +const tempDirs: string[] = []; + +async function makeRepo( + files: Record, +): Promise<{ root: string; repo: RepoHandle }> { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-graphql-')); + tempDirs.push(root); + for (const [relative, content] of Object.entries(files)) { + const absolute = path.join(root, relative); + await fs.mkdir(path.dirname(absolute), { recursive: true }); + await fs.writeFile(absolute, content, 'utf8'); + } + return { + root, + repo: { id: 'test', path: 'app', repoPath: root, storagePath: path.join(root, '.gitnexus') }, + }; +} + +function executor(symbols: Record>>): CypherExecutor { + return async (_query, params = {}) => { + const name = String(params.name ?? ''); + const filePath = params.filePath ? `@${String(params.filePath)}` : ''; + return symbols[`${name}${filePath}`] ?? symbols[name] ?? []; + }; +} + +function generatedDocument( + operation: 'query' | 'mutation' | 'subscription', + name: string, + fields: string[], +): string { + const selections = fields + .map((field) => `{ kind: 'Field', name: { kind: 'Name', value: '${field}' } }`) + .join(', '); + return `{ kind: 'Document', definitions: [{ + kind: 'OperationDefinition', + operation: '${operation}', + name: { kind: 'Name', value: '${name}' }, + selectionSet: { kind: 'SelectionSet', selections: [${selections}] } + }] }`; +} + +describe('GraphqlExtractor', () => { + afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(tempDirs.splice(0).map((dir) => cleanupTempDir(dir))); + }); + + it('anchors NestJS providers and document consumers to exact real symbols', async () => { + const { root, repo } = await makeRepo({ + 'src/widget.resolver.ts': ` +import { Resolver, Query as GqlQuery, Mutation, Subscription } from '@nestjs/graphql'; +@Resolver() +class WidgetResolver { + @GqlQuery(() => Widget, { name: 'widget' }) + fetchWidget() { return null; } + @Mutation('saveWidget') + save() { return null; } + @Subscription() + widgetChanged() { return null; } +}`, + 'src/widget.graphql': ` +fragment WidgetRoot on Query { widget } +query GetWidget { alias: widget ...WidgetRoot } +mutation SaveWidget { saveWidget } +subscription WatchWidget { widgetChanged } +`, + 'src/generated.ts': ` +export const GetWidgetDocument = ${generatedDocument('query', 'GetWidget', ['widget'])}; +export const SaveWidgetDocument = ${generatedDocument('mutation', 'SaveWidget', ['saveWidget'])}; +export const WatchWidgetDocument = ${generatedDocument('subscription', 'WatchWidget', ['widgetChanged'])}; +`, + }); + const run = executor({ + 'fetchWidget@src/widget.resolver.ts': [ + { uid: 'method:fetch', name: 'fetchWidget', filePath: 'src/widget.resolver.ts' }, + ], + 'save@src/widget.resolver.ts': [ + { uid: 'method:save', name: 'save', filePath: 'src/widget.resolver.ts' }, + ], + 'widgetChanged@src/widget.resolver.ts': [ + { uid: 'method:watch', name: 'widgetChanged', filePath: 'src/widget.resolver.ts' }, + ], + GetWidgetDocument: [ + { uid: 'const:get', name: 'GetWidgetDocument', filePath: 'src/generated.ts' }, + ], + SaveWidgetDocument: [ + { uid: 'const:save', name: 'SaveWidgetDocument', filePath: 'src/generated.ts' }, + ], + WatchWidgetDocument: [ + { uid: 'const:watch', name: 'WatchWidgetDocument', filePath: 'src/generated.ts' }, + ], + }); + + const contracts = await new GraphqlExtractor().extract(run, root, repo); + + expect( + contracts.map((contract) => [contract.contractId, contract.role, contract.symbolUid]), + ).toEqual([ + ['graphql::query::widget', 'provider', 'method:fetch'], + ['graphql::mutation::saveWidget', 'provider', 'method:save'], + ['graphql::subscription::widgetChanged', 'provider', 'method:watch'], + ['graphql::query::widget', 'consumer', 'const:get'], + ['graphql::mutation::saveWidget', 'consumer', 'const:save'], + ['graphql::subscription::widgetChanged', 'consumer', 'const:watch'], + ]); + }); + + it('skips unproven decorators, ambiguous anchors, anonymous operations, and invalid documents', async () => { + const { root, repo } = await makeRepo({ + 'src/not-nest.ts': ` +function Query(): MethodDecorator { return () => undefined; } +class LocalResolver { @Query() localOnly() {} }`, + 'src/ambiguous.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +@Resolver() +class AmbiguousResolver { @Query() widget() {} }`, + 'src/anonymous.graphql': `query { widget }`, + 'src/invalid.graphql': `query Broken {`, + 'src/missing.graphql': `query MissingGenerated { widget }`, + }); + const ambiguous = [ + { uid: 'method:a', name: 'widget', filePath: 'src/ambiguous.resolver.ts' }, + { uid: 'method:b', name: 'widget', filePath: 'src/ambiguous.resolver.ts' }, + ]; + + const contracts = await new GraphqlExtractor().extract( + executor({ 'widget@src/ambiguous.resolver.ts': ambiguous }), + root, + repo, + ); + + expect(contracts).toEqual([]); + }); + + it('rejects provider fields that are not GraphQL Names', async () => { + const { root, repo } = await makeRepo({ + 'src/invalid.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +@Resolver() +class InvalidResolver { + @Query('bad::field') separator() {} + @Query('line\\nfeed') newline() {} + @Query('right\\u202etoLeft') bidi() {} + @Query('9startsWithDigit') digit() {} +}`, + }); + let lookups = 0; + const run: CypherExecutor = async () => { + lookups++; + return [{ uid: 'method:invalid', name: 'invalid', filePath: 'src/invalid.resolver.ts' }]; + }; + + expect(await new GraphqlExtractor().extract(run, root, repo)).toEqual([]); + expect(lookups).toBe(0); + }); + + it('skips a provider file whose AST exceeds the traversal depth cap', async () => { + const nested = `${'['.repeat(300)}0${']'.repeat(300)}`; + const { root, repo } = await makeRepo({ + 'src/deep.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +const nested = ${nested}; +@Resolver() +class DeepResolver { @Query() health() {} }`, + }); + let lookups = 0; + + expect( + await new GraphqlExtractor().extract( + async () => { + lookups++; + return [{ uid: 'method:health', name: 'health', filePath: 'src/deep.resolver.ts' }]; + }, + root, + repo, + ), + ).toEqual([]); + expect(lookups).toBe(0); + }); + + it('uses an exact unique Document symbol when no generated hook exists', async () => { + const { root, repo } = await makeRepo({ + 'src/health.graphql': `query Health { health }`, + 'src/generated/graphql.ts': `export const HealthDocument = ${generatedDocument('query', 'Health', ['health'])};`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + HealthDocument: [ + { uid: 'var:health', name: 'HealthDocument', filePath: 'src/generated/graphql.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts).toEqual([ + expect.objectContaining({ + contractId: 'graphql::query::health', + role: 'consumer', + symbolUid: 'var:health', + symbolRef: { filePath: 'src/generated/graphql.ts', name: 'HealthDocument' }, + }), + ]); + }); + + it('skips matching tokens that occur only in unrelated initializer metadata', async () => { + const { root, repo } = await makeRepo({ + 'src/health.graphql': `query Health { health }`, + 'src/generated/graphql.ts': `export const HealthDocument = { + metadata: { operation: 'Health', field: 'health' }, + kind: 'NotADocument' + };`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + HealthDocument: [ + { uid: 'var:health', name: 'HealthDocument', filePath: 'src/generated/graphql.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts).toEqual([]); + }); + + it('fails closed when a generated Document initializer exceeds the AST depth budget', async () => { + const { root, repo } = await makeRepo({ + 'src/deep.graphql': `query Deep { health }`, + 'src/generated.ts': `export const DeepDocument = ${'('.repeat(300)}${generatedDocument( + 'query', + 'Deep', + ['health'], + )}${')'.repeat(300)};`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + DeepDocument: [{ uid: 'const:deep', name: 'DeepDocument', filePath: 'src/generated.ts' }], + }), + root, + repo, + ); + + expect(contracts).toEqual([]); + }); + + it('fails closed for oversized, deeply nested, and excessive-operation documents', async () => { + const nested = `${'... on Query { '.repeat(65)}health${' }'.repeat(65)}`; + const manyOperations = Array.from( + { length: 501 }, + (_, index) => `query Op${index} { field${index} }`, + ).join('\n'); + const { root, repo } = await makeRepo({ + 'src/oversized.graphql': `${' '.repeat(1_000_001)}query Huge { huge }`, + 'src/deep.graphql': `query Deep { ${nested} }`, + 'src/many.graphql': manyOperations, + }); + let lookups = 0; + const run: CypherExecutor = async (_query, params = {}) => { + lookups++; + const name = String(params.name ?? ''); + return name.startsWith('useOp') + ? [{ uid: `fn:${name}`, name, filePath: 'src/generated.ts' }] + : name === 'useDeepQuery' + ? [{ uid: 'fn:deep', name, filePath: 'src/generated.ts' }] + : []; + }; + + const contracts = await new GraphqlExtractor().extract(run, root, repo); + + expect(contracts).toEqual([]); + expect(contracts).not.toContainEqual(expect.objectContaining({ symbolUid: 'fn:deep' })); + expect(lookups).toBe(0); + }); + + it('keeps decorators across comments and supports decorated resolver properties', async () => { + const { root, repo } = await makeRepo({ + 'src/commented.resolver.ts': ` +import { Query, Mutation, Resolver } from '@nestjs/graphql'; +@Resolver() +export class CommentedResolver { + @Query() + /** Public schema description. */ + health() { return true; } + + @Mutation() + // The comment is not an ownership boundary. + save = async () => true; +}`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + 'health@src/commented.resolver.ts': [ + { uid: 'method:health', name: 'health', filePath: 'src/commented.resolver.ts' }, + ], + 'save@src/commented.resolver.ts': [ + { uid: 'property:save', name: 'save', filePath: 'src/commented.resolver.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.contractId)).toEqual([ + 'graphql::query::health', + 'graphql::mutation::save', + ]); + }); + + it('requires a top-level imported Resolver and skips dynamic field names', async () => { + const { root, repo } = await makeRepo({ + 'src/scoped.resolver.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +const FIELD = 'viewer'; +const NAMES = { viewer: FIELD }; +const name = FIELD; +const opts = { name: FIELD }; +class Helper { @Query() helperOnly() {} } +function factory() { + @Resolver() + class NestedResolver { @Query() nestedOnly() {} } + return NestedResolver; +} +@Resolver() +class RealResolver { + @Query(() => String, { name: FIELD }) dynamicName() {} + @Query(() => String, { name: NAMES.viewer }) memberName() {} + @Query(() => String, { name }) shorthandName() {} + @Query(() => String, { ...opts }) spreadOptions() {} + @Query(() => String, { name: \`get\${FIELD}\` }) interpolatedName() {} + @Query(() => String, { name: 'get' + 'Widget' }) concatenatedName() {} + @Query(() => String) stableName() {} +}`, + }); + const lookedUp: string[] = []; + const run: CypherExecutor = async (_query, params = {}) => { + lookedUp.push(String(params.name)); + return [ + { + uid: `method:${String(params.name)}`, + name: String(params.name), + filePath: 'src/scoped.resolver.ts', + }, + ]; + }; + + const contracts = await new GraphqlExtractor().extract(run, root, repo); + + expect(lookedUp).toEqual(['stableName']); + expect(contracts).toEqual([ + expect.objectContaining({ contractId: 'graphql::query::stableName' }), + ]); + }); + + it('does not extract co-located spec resolvers', async () => { + const { root, repo } = await makeRepo({ + 'src/widget.resolver.spec.ts': ` +import { Query, Resolver } from '@nestjs/graphql'; +@Resolver() +class MockResolver { @Query() widget() {} }`, + }); + let lookups = 0; + + const contracts = await new GraphqlExtractor().extract( + async () => { + lookups++; + return [{ uid: 'method:mock', name: 'widget', filePath: 'src/widget.resolver.spec.ts' }]; + }, + root, + repo, + ); + + expect(contracts).toEqual([]); + expect(lookups).toBe(0); + }); + + it('indexes a generated declaration after more than 100000 earlier AST nodes', () => { + const filler = { type: 'identifier', namedChildren: [] } as unknown as Parser.SyntaxNode; + const name = { text: 'LateDocument' } as Parser.SyntaxNode; + const value = { type: 'object', namedChildren: [] } as unknown as Parser.SyntaxNode; + const declaration = { + type: 'variable_declarator', + namedChildren: [], + childForFieldName: (field: string) => + field === 'name' ? name : field === 'value' ? value : null, + } as unknown as Parser.SyntaxNode; + const root = { + type: 'program', + namedChildren: [...Array(100_001).fill(filler), declaration], + } as unknown as Parser.SyntaxNode; + + expect(indexGeneratedDeclarators(root).get('LateDocument')).toEqual([value]); + }); + + it('proves generated root fields through fragment spreads and inline fragments', async () => { + const { root, repo } = await makeRepo({ + 'src/widgets.graphql': ` +query GetWidgets { widget ...MoreRoots ... on Query { inlineRoot } } +fragment MoreRoots on Query { gadget } +`, + 'src/generated.ts': ` +export const GetWidgetsDocument = ${JSON.stringify( + parse(` +query GetWidgets { widget ...MoreRoots ... on Query { inlineRoot } } +fragment MoreRoots on Query { gadget } +`), + )};`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + GetWidgetsDocument: [ + { uid: 'const:widgets', name: 'GetWidgetsDocument', filePath: 'src/generated.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.contractId).sort()).toEqual([ + 'graphql::query::gadget', + 'graphql::query::inlineRoot', + 'graphql::query::widget', + ]); + }); + + it('accepts static gql tags and TypedDocumentString initializers', async () => { + const { root, repo } = await makeRepo({ + 'src/tagged.graphql': `query Tagged { tagged }`, + 'src/string.graphql': `query StringMode { stringMode }`, + 'src/generated.ts': ` +export const TaggedDocument = gql\`query Tagged { tagged }\`; +export const StringModeDocument = new TypedDocumentString("query StringMode {\\n stringMode\\n}"); +`, + }); + + const contracts = await new GraphqlExtractor().extract( + executor({ + TaggedDocument: [ + { uid: 'const:tagged', name: 'TaggedDocument', filePath: 'src/generated.ts' }, + ], + StringModeDocument: [ + { uid: 'const:string', name: 'StringModeDocument', filePath: 'src/generated.ts' }, + ], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.symbolUid).sort()).toEqual([ + 'const:string', + 'const:tagged', + ]); + }); + + it('parses one generated module once and continues past a shadowed candidate', async () => { + const { root, repo } = await makeRepo({ + 'src/one.graphql': `query One { one }`, + 'src/two.graphql': `query Two { two }`, + 'src/generated.ts': ` +function shadow() { const OneDocument = { kind: 'NotADocument' }; } +export const OneDocument = ${generatedDocument('query', 'One', ['one'])}; +export const TwoDocument = ${generatedDocument('query', 'Two', ['two'])}; +`, + }); + const parseSpy = vi.spyOn(Parser.prototype, 'parse'); + + const contracts = await new GraphqlExtractor().extract( + executor({ + OneDocument: [{ uid: 'const:one', name: 'OneDocument', filePath: 'src/generated.ts' }], + TwoDocument: [{ uid: 'const:two', name: 'TwoDocument', filePath: 'src/generated.ts' }], + }), + root, + repo, + ); + + expect(contracts.map((contract) => contract.symbolUid).sort()).toEqual([ + 'const:one', + 'const:two', + ]); + expect(parseSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/test/unit/group/manifest-label-drift.test.ts b/gitnexus/test/unit/group/manifest-label-drift.test.ts index 18802e8c5..dea8553c6 100644 --- a/gitnexus/test/unit/group/manifest-label-drift.test.ts +++ b/gitnexus/test/unit/group/manifest-label-drift.test.ts @@ -9,6 +9,10 @@ */ import { describe, it, expect } from 'vitest'; import { CUSTOM_CONTRACT_RESOLVE_QUERY } from '../../../src/core/group/extractors/manifest-extractor.js'; +import { + RESOLVE_GENERATED_SYMBOL_QUERY, + RESOLVE_METHOD_QUERY, +} from '../../../src/core/group/extractors/graphql-extractor.js'; import { SYMBOL_NODE_LABELS } from '../../../src/core/ingestion/utils/symbol-labels.js'; describe('manifest contract-resolve label list vs SYMBOL_NODE_LABELS (#2380)', () => { @@ -32,3 +36,17 @@ describe('manifest contract-resolve label list vs SYMBOL_NODE_LABELS (#2380)', ( expect(diff).toEqual(['Module', 'Namespace', 'Variable']); }); }); + +describe.each([ + ['GraphQL provider', RESOLVE_METHOD_QUERY], + ['GraphQL generated symbol', RESOLVE_GENERATED_SYMBOL_QUERY], +])('%s query label list vs SYMBOL_NODE_LABELS', (_name, query) => { + const match = query.match(/labels\(n\) IN \[([^\]]+)\]/); + const queryLabels = (match?.[1]?.match(/'([^']+)'/g) ?? []).map((label) => label.slice(1, -1)); + const symbolLabels = new Set(SYMBOL_NODE_LABELS); + + it('keeps every hand-listed label in the shared symbol label set', () => { + expect(queryLabels.length).toBeGreaterThan(0); + for (const label of queryLabels) expect(symbolLabels.has(label)).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/group/matching.test.ts b/gitnexus/test/unit/group/matching.test.ts index a10f1d017..bc9937e85 100644 --- a/gitnexus/test/unit/group/matching.test.ts +++ b/gitnexus/test/unit/group/matching.test.ts @@ -183,6 +183,24 @@ describe('runExactMatch', () => { expect(matched).toHaveLength(0); }); + it('suppresses configured GraphQL root fields from exact matching', () => { + const provider = { + ...makeContract('graphql::query::health', 'provider', 'api'), + type: 'graphql' as const, + }; + const consumer = { + ...makeContract('graphql::query::health', 'consumer', 'web'), + type: 'graphql' as const, + }; + + const { matched, unmatched } = runExactMatch([provider, consumer], undefined, { + exclude_links_paths: ['/health'], + }); + + expect(matched).toEqual([]); + expect(unmatched).toEqual([]); + }); + it('does not match same-repo when only one has service', () => { const contracts: StoredContract[] = [ { diff --git a/gitnexus/test/unit/group/sync.test.ts b/gitnexus/test/unit/group/sync.test.ts index 68fe13c50..95bdd96d7 100644 --- a/gitnexus/test/unit/group/sync.test.ts +++ b/gitnexus/test/unit/group/sync.test.ts @@ -23,6 +23,7 @@ describe('syncGroup', () => { packages: {}, detect: { http: true, + graphql: false, grpc: false, thrift: false, topics: false, @@ -72,6 +73,33 @@ describe('syncGroup', () => { expect(result.unmatched).toHaveLength(0); }); + it('exact-matches GraphQL root fields across repositories', async () => { + const config = makeConfig({ api: 'api-repo', web: 'web-repo' }); + const contracts: StoredContract[] = [ + { + ...makeContract('graphql::query::widget', 'provider', 'api'), + type: 'graphql', + }, + { + ...makeContract('graphql::query::widget', 'consumer', 'web'), + type: 'graphql', + }, + ]; + + const result = await syncGroup(config, { + extractorOverride: async () => contracts, + skipWrite: true, + }); + + expect(result.crossLinks).toEqual([ + expect.objectContaining({ + type: 'graphql', + contractId: 'graphql::query::widget', + matchType: 'exact', + }), + ]); + }); + it('reports missing repos', async () => { const config = makeConfig({ 'app/backend': 'nonexistent-repo' }); diff --git a/gitnexus/test/unit/group/types.test.ts b/gitnexus/test/unit/group/types.test.ts index cfa9abba2..94b0b556c 100644 --- a/gitnexus/test/unit/group/types.test.ts +++ b/gitnexus/test/unit/group/types.test.ts @@ -20,6 +20,7 @@ describe('Group types', () => { packages: {}, detect: { http: true, + graphql: true, grpc: true, thrift: true, topics: true, @@ -48,7 +49,7 @@ describe('Group types', () => { }); it('ExtractedContract accepts all contract types', () => { - const types: ContractType[] = ['http', 'grpc', 'topic', 'lib', 'custom']; + const types: ContractType[] = ['http', 'graphql', 'grpc', 'topic', 'lib', 'custom']; types.forEach((t) => { const contract: ExtractedContract = { contractId: `${t}::test`, @@ -88,6 +89,7 @@ describe('Group types', () => { packages: {}, detect: { http: true, + graphql: true, grpc: true, thrift: true, topics: true, diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 9d996279a..4f9d73256 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -95,6 +95,7 @@ export default defineConfig({ 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/manifest-synthetic-impact-lbug.test.ts', 'test/integration/group/http-route-resolve-symbol.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/fts-stemmer-sweep.test.ts', 'test/integration/lbug-multiwriter-deadlock.test.ts', 'test/integration/extension-binary-real.test.ts', @@ -169,6 +170,7 @@ export default defineConfig({ 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/manifest-synthetic-impact-lbug.test.ts', 'test/integration/group/http-route-resolve-symbol.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/skills-e2e.test.ts', 'test/integration/fts-extension-e2e.test.ts', 'test/integration/fts-stemmer-sweep.test.ts', From 38a0837e4b3295eb3c916329a9ba572694cde259 Mon Sep 17 00:00:00 2001 From: "John R. Eakin" Date: Sat, 29 Aug 2026 02:46:30 -0500 Subject: [PATCH 3/3] feat(wiki): add grok local CLI provider (#3069) * feat(wiki): add grok local CLI provider Wiki generation can use `gitnexus wiki --provider grok` to spawn the authenticated Grok Build CLI (`grok --prompt-file`) instead of an HTTP API key. * style(wiki): prettier grok-client for CI format check CI quality/format failed on grok-client.ts. Auto-format matches repo prettier so the GitNexus /autofix comment is applied locally. * Update Grok CLI configuration to use empty allowlist and increase max tu * Replace Grok tool allowlist with explicit denylist and strict sandbox * Increase Grok max turns to 15 to accommodate prompt variance * chore(wiki): drop Unreleased CHANGELOG hunk and restore lockfile libc selectors Feature PRs do not own CHANGELOG.md. Restore the 16 libc platform selectors deleted from package-lock.json with no dependency change. * fix(wiki): resolve grok CLI through Windows cmd.exe shims Extract resolveWindowsCliCommand from the local CLI client and use it for grok detect/spawn so npm .cmd installs work without a shell. Keep detectGrokCLI() returning the display name for the wiki menu. * fix(wiki): wait for grok child close before timeout cleanup Do not reject the grok spawn promise on the timeout timer. Kill the child, escalate SIGKILL after 2s, and reject only on close (or a second 2s hard deadline) so callGrokLLM cannot rm the sandbox while the process is alive. * fix(wiki): reject incomplete grok stopReason and distinct parse errors Honor JSON stopReason (end_turn or omitted succeeds; anything else throws). Split empty-output / non-JSON / missing-text messages and include a truncated stdout excerpt. Drop unused GrokConfig.workingDirectory. * fix(wiki): keep grok temp dir on hung timeout and ignore stdin Hard-deadline reject no longer removes --cwd while the child may still be running. Spawn stdin is ignored so grok's unused pipe cannot EPIPE the wiki process. Co-Authored-By: Grok 4.6 * fix(wiki): require grok stopReason=end_turn for a finished page Live grok 1.0.5 with wiki spawn flags returns stopReason end_turn. Omitted, null, or empty stopReason is no longer treated as success, so generateLeafPage cannot write a page that never completed. Co-Authored-By: Grok 4.6 * style(wiki): deslop grok parse nesting and extra comments Flatten parseGrokOutput with early returns and drop narrative comments that restated the timeout/stdin/stopReason constraints. Behavior unchanged. Co-Authored-By: Grok 4.6 * test(wiki): make grok Windows spawn tests match real cmd.exe On Windows CI, detectGrokCLI also calls where.exe, ComSpec is an absolute cmd.exe path, and waitForSpawn must wait for real fs I/O. Co-Authored-By: Grok 4.6 * test(wiki): expect taskkill on Windows grok timeout, not child.kill killChildTree uses taskkill /T /F on win32 and only falls back to child.kill() if that fails. Co-Authored-By: Grok 4.6 * test(wiki): remove grok temp dir after hard-deadline leak assertion The hard-deadline test must keep the dir until close, then emit close so late cleanup runs and the temp directory is not left behind. Co-Authored-By: Grok 4.6 * test(wiki): wait for grok temp dir rm after late close Windows CI failed the hard-deadline test because 30 setImmediate ticks cannot observe fire-and-forget fs.rm. Poll with real timers after close. --------- Co-authored-by: Grok 4.6 --- .claude/skills/gitnexus-cli/SKILL.md | 8 +- README.md | 1 + .../skills/gitnexus-cli/SKILL.md | 40 +- gitnexus/README.md | 1 + gitnexus/skills/gitnexus-cli.md | 8 +- gitnexus/src/cli/i18n/en.ts | 2 +- gitnexus/src/cli/i18n/zh-CN.ts | 2 +- gitnexus/src/cli/index.ts | 2 +- gitnexus/src/cli/wiki.ts | 24 +- gitnexus/src/core/wiki/generator.ts | 8 + gitnexus/src/core/wiki/grok-client.ts | 359 ++++++++++ gitnexus/src/core/wiki/llm-client.ts | 8 +- gitnexus/src/core/wiki/local-cli-client.ts | 38 +- gitnexus/src/storage/repo-manager.ts | 2 + gitnexus/test/integration/cli-e2e.test.ts | 9 + gitnexus/test/unit/cli-index-help.test.ts | 1 + .../test/unit/local-cli-subprocess.test.ts | 622 +++++++++++++++++- gitnexus/test/unit/wiki-flags.test.ts | 118 ++++ 18 files changed, 1207 insertions(+), 46 deletions(-) create mode 100644 gitnexus/src/core/wiki/grok-client.ts diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index 853d44860..e993c38f8 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -55,15 +55,19 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. | Flag | Effect | | ------------------- | ----------------------------------------- | -| `--force` | Force full regeneration | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | | `--model ` | LLM model (default: MiniMax-M3) | | `--base-url ` | LLM API base URL | | `--api-key ` | LLM API key | | `--concurrency ` | Parallel LLM calls (default: 3) | +| `--timeout ` | LLM request timeout in seconds (default: disabled) | +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | ### list — Show all indexed repos diff --git a/README.md b/README.md index e26373aff..e89ddcdb3 100644 --- a/README.md +++ b/README.md @@ -767,6 +767,7 @@ gitnexus wiki # Use a custom model or provider (default model: minimax/minimax-m2.5) gitnexus wiki --model gpt-4o gitnexus wiki --base-url https://api.anthropic.com/v1 +gitnexus wiki --provider grok # local Grok Build CLI (uses `grok login`, no API key) # Force full regeneration gitnexus wiki --force diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index 9c7a1b599..e993c38f8 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -19,14 +19,14 @@ node .gitnexus/run.cjs analyze Run from the project root. This parses all source files, builds the knowledge graph, writes it to `.gitnexus/`, and generates CLAUDE.md / AGENTS.md context files. -| Flag | Effect | -|------|--------| -| `--force` | Force full re-index even if up to date | +| Flag | Effect | +| -------------- | ---------------------------------------------------------------- | +| `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | -**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. +**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. ### status — Check index freshness @@ -44,10 +44,10 @@ node .gitnexus/run.cjs clean Deletes the `.gitnexus/` directory and unregisters the repo from the global registry. Use before re-indexing if the index is corrupt or after removing GitNexus from a project. -| Flag | Effect | -|------|--------| -| `--force` | Skip confirmation prompt | -| `--all` | Clean all indexed repos, not just the current one | +| Flag | Effect | +| --------- | ------------------------------------------------- | +| `--force` | Skip confirmation prompt | +| `--all` | Clean all indexed repos, not just the current one | ### wiki — Generate documentation from the graph @@ -55,19 +55,21 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. -| Flag | Effect | -|------|--------| -| `--force` | Force full regeneration, also required to re-gerenate an existing wiki in a different language | -| `--model ` | LLM model (default: MiniMax-M3) | -| `--base-url ` | LLM API base URL | -| `--api-key ` | LLM API key | -| `--concurrency ` | Parallel LLM calls (default: 3) | -| `--gist` | Publish wiki as a public GitHub Gist | +| Flag | Effect | +| ------------------- | ----------------------------------------- | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | +| `--model ` | LLM model (default: MiniMax-M3) | +| `--base-url ` | LLM API base URL | +| `--api-key ` | LLM API key | +| `--concurrency ` | Parallel LLM calls (default: 3) | | `--timeout ` | LLM request timeout in seconds (default: disabled) | -| `--retries ` | Max LLM retry attempts per request (default: 3) | -| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese)| +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | +| `--gist` | Publish wiki as a public GitHub Gist | + ### list — Show all indexed repos ```bash diff --git a/gitnexus/README.md b/gitnexus/README.md index e524eaa19..8013026f2 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -256,6 +256,7 @@ gitnexus clean # Delete index for current repo gitnexus clean --all --force # Delete all indexes gitnexus wiki [path] # Generate LLM-powered docs from knowledge graph gitnexus wiki --model # Wiki with custom LLM model (default: minimax/minimax-m2.5) +gitnexus wiki --provider grok # Local Grok Build CLI (uses `grok login`, no API key) gitnexus wiki --base-url http://llama-box.local:8080/v1 --allow-insecure-connection llama-box.local # Allow an exact LAN/self-hosted HTTP LLM host; env: GITNEXUS_ALLOW_INSECURE_CONNECTION gitnexus doctor # Show runtime platform capabilities and embedding configuration diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index 853d44860..e993c38f8 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -55,15 +55,19 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. | Flag | Effect | | ------------------- | ----------------------------------------- | -| `--force` | Force full regeneration | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | | `--model ` | LLM model (default: MiniMax-M3) | | `--base-url ` | LLM API base URL | | `--api-key ` | LLM API key | | `--concurrency ` | Parallel LLM calls (default: 3) | +| `--timeout ` | LLM request timeout in seconds (default: disabled) | +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | ### list — Show all indexed repos diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index e893e61ac..c22da5211 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -234,7 +234,7 @@ export const en = { 'Clean parked LadybugDB recovery sidecars (missing-shadow WAL quarantines and dirty-recovery parks)', 'help.option.wiki.force': 'Force full regeneration even if up to date', 'help.option.wiki.provider': - 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: minimax)', + 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax)', 'help.option.wiki.model': 'LLM model or deployment name (default: MiniMax-M3)', 'help.option.wiki.baseUrl': 'LLM API base URL. Azure v1: https://{resource}.openai.azure.com/openai/v1', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 2c066f010..7ef2d244b 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -220,7 +220,7 @@ export const zhCN = { '清理已暂存的 LadybugDB 恢复 sidecar（missing-shadow WAL 隔离文件与 dirty-recovery 暂存文件）', 'help.option.wiki.force': '即使已是最新也强制完整重新生成', 'help.option.wiki.provider': - 'LLM 提供商：minimax、openai、openrouter、azure、custom、cursor、claude、codex 或 opencode（默认：minimax）', + 'LLM 提供商：minimax、openai、openrouter、azure、custom、cursor、claude、codex、opencode 或 grok（默认：minimax）', 'help.option.wiki.model': 'LLM 模型或 deployment 名称（默认：MiniMax-M3）', 'help.option.wiki.baseUrl': 'LLM API base URL。Azure v1：https://{resource}.openai.azure.com/openai/v1', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 1ccf75c2f..a1edf0b53 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -303,7 +303,7 @@ program .option('-f, --force', 'Force full regeneration even if up to date') .option( '--provider ', - 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: minimax)', + 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax)', ) .option('--model ', 'LLM model or deployment name (default: MiniMax-M3)') .option( diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index d65d130a7..007451ef5 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -25,6 +25,7 @@ import { type LLMProvider, } from '../core/wiki/llm-client.js'; import { detectCursorCLI } from '../core/wiki/cursor-client.js'; +import { detectGrokCLI } from '../core/wiki/grok-client.js'; import { detectLocalCLI } from '../core/wiki/local-cli-client.js'; import { logger } from '../core/logger.js'; @@ -65,20 +66,22 @@ function parsePositiveIntegerOption( function isLocalProvider( provider: LLMProvider | undefined, -): provider is 'cursor' | 'claude' | 'codex' | 'opencode' { +): provider is 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok' { return ( provider === 'cursor' || provider === 'claude' || provider === 'codex' || - provider === 'opencode' + provider === 'opencode' || + provider === 'grok' ); } -function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode') { +function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok') { if (provider === 'cursor') return 'cursorModel'; if (provider === 'claude') return 'claudeModel'; if (provider === 'codex') return 'codexModel'; if (provider === 'opencode') return 'opencodeModel'; + if (provider === 'grok') return 'grokModel'; throw new Error(`Unsupported local provider: ${provider satisfies never}`); } @@ -287,7 +290,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) if (!llmConfig.apiKey && !isLocalProvider(llmConfig.provider)) { console.log(' Error: No LLM API key found.'); console.log(' Set MINIMAX_API_KEY, GITNEXUS_API_KEY, or OPENAI_API_KEY,'); - console.log(' or pass --api-key , or use --provider cursor|claude|codex|opencode.\n'); + console.log( + ' or pass --api-key , or use --provider cursor|claude|codex|opencode|grok.\n', + ); process.exitCode = 1; return; } @@ -301,9 +306,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) const hasClaude = detectLocalCLI('claude'); const hasCodex = detectLocalCLI('codex'); const hasOpenCode = detectLocalCLI('opencode'); + const hasGrok = detectGrokCLI(); const localChoices: Array<{ choice: string; - provider: 'cursor' | 'claude' | 'codex' | 'opencode'; + provider: 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok'; }> = []; // Provider selection @@ -346,6 +352,14 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) }); console.log(` [${choice}] OpenCode CLI (local, uses your OpenCode login/config)`); } + if (hasGrok) { + const choice = String(nextChoice++); + localChoices.push({ + choice, + provider: 'grok', + }); + console.log(` [${choice}] Grok CLI (local, uses your Grok Build login)`); + } console.log(''); const maxChoice = String(nextChoice - 1); diff --git a/gitnexus/src/core/wiki/generator.ts b/gitnexus/src/core/wiki/generator.ts index 2e6180731..d1f3aaabc 100644 --- a/gitnexus/src/core/wiki/generator.ts +++ b/gitnexus/src/core/wiki/generator.ts @@ -40,6 +40,7 @@ import { } from './llm-client.js'; import { callCursorLLM, resolveCursorConfig } from './cursor-client.js'; +import { callGrokLLM, resolveGrokConfig } from './grok-client.js'; import { callClaudeLLM, callCodexLLM, @@ -225,6 +226,13 @@ export class WikiGenerator { }); return callCursorLLM(prompt, cursorConfig, systemPrompt, options); } + if (this.llmConfig.provider === 'grok') { + const grokConfig = resolveGrokConfig({ + model: this.llmConfig.model, + requestTimeoutMs: this.llmConfig.requestTimeoutMs, + }); + return callGrokLLM(prompt, grokConfig, systemPrompt, options); + } if ( this.llmConfig.provider === 'claude' || this.llmConfig.provider === 'codex' || diff --git a/gitnexus/src/core/wiki/grok-client.ts b/gitnexus/src/core/wiki/grok-client.ts new file mode 100644 index 000000000..43a2e0160 --- /dev/null +++ b/gitnexus/src/core/wiki/grok-client.ts @@ -0,0 +1,359 @@ +/** + * Grok Build CLI client for wiki generation. + * + * Uses headless `grok --prompt-file` so large wiki prompts are not placed on + * argv or stdin (Grok does not read stdin as the prompt). + */ + +import { spawn, execFileSync } from 'child_process'; +import { StringDecoder } from 'string_decoder'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import type { CallLLMOptions, LLMResponse } from './llm-client.js'; +import { resolveWindowsCliCommand, type ResolvedCliCommand } from './local-cli-client.js'; +import { logger } from '../logger.js'; + +export interface GrokConfig { + model?: string; + requestTimeoutMs?: number; +} + +// Verified live: an empty --tools allowlist does NOT disable the shell tool +// (`run_terminal_cmd`) — Grok still ran `find` across the user's entire home +// directory looking for project context, taking 10+ minutes per call and +// then hitting --max-turns anyway. A denylist naming the tool explicitly is +// what actually blocks it; internal tool IDs, not the CLI-facing names +// (shell is `run_terminal_cmd`, not `bash`). +const GROK_DISALLOWED_TOOLS = 'run_terminal_cmd,search_replace,web_search,web_fetch,spawn_subagent'; + +// Defense in depth beyond the tool denylist: a kernel-enforced (Landlock/ +// Seatbelt) sandbox so reads/writes stay confined to --cwd (our empty temp +// dir) + system paths even if a future tool slips past the denylist. +const GROK_SANDBOX_PROFILE = 'strict'; + +// Verified live with the denylist + sandbox above: turn counts vary run to +// run (observed 3-10 across identical prompts, including the large overview +// prompt that aggregates every module's summary). 15 gives real headroom +// over that variance without leaving a runaway session effectively uncapped. +const GROK_MAX_TURNS = '15'; + +let cachedGrokCommand: ResolvedCliCommand | null | undefined; + +function isVerbose(): boolean { + return process.env.GITNEXUS_VERBOSE === '1'; +} + +function verboseLog(...args: unknown[]): void { + if (isVerbose()) { + logger.info({ args }, '[grok-cli]'); + } +} + +function killChildTree(child: import('child_process').ChildProcess): void { + if (process.platform === 'win32' && child.pid !== undefined) { + try { + execFileSync('taskkill', ['/T', '/F', '/PID', String(child.pid)], { + stdio: 'ignore', + windowsHide: true, + }); + return; + } catch { + // Process may have already exited — fall through to child.kill() + } + } + child.kill(); +} + +/** Returns `'grok'` when the CLI is on PATH, else null. Cached. */ +export function detectGrokCLI(): string | null { + if (cachedGrokCommand !== undefined) return cachedGrokCommand?.displayName ?? null; + const resolved = resolveWindowsCliCommand('grok'); + try { + execFileSync(resolved.command, [...resolved.argsPrefix, '--version'], { + stdio: 'ignore', + windowsHide: true, + }); + cachedGrokCommand = resolved; + } catch (err: unknown) { + const isNotFound = + err instanceof Error && 'code' in err && (err as NodeJS.ErrnoException).code === 'ENOENT'; + if (!isNotFound && err instanceof Error) { + logger.warn( + `grok CLI found but --version failed (exit ${(err as { status?: number }).status ?? '?'}). ` + + 'Ensure it is authenticated: run `grok --version` manually.', + ); + } + cachedGrokCommand = null; + } + return cachedGrokCommand?.displayName ?? null; +} + +function getDetectedGrokCommand(): ResolvedCliCommand | null { + detectGrokCLI(); + return cachedGrokCommand ?? null; +} + +export function resolveGrokConfig(overrides?: Partial): GrokConfig { + return { + model: overrides?.model, + requestTimeoutMs: overrides?.requestTimeoutMs, + }; +} + +function excerpt(raw: string, max = 200): string { + const trimmed = raw.trim(); + if (trimmed.length <= max) return trimmed; + return `${trimmed.slice(0, max)}…`; +} + +function parseGrokOutput(stdout: string): string { + const trimmed = stdout.trim(); + if (!trimmed) { + throw new Error('grok CLI returned empty output'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + throw new Error(`grok CLI returned non-JSON output: ${excerpt(trimmed)}`); + } + + if (!parsed || typeof parsed !== 'object') { + throw new Error(`grok CLI JSON has no text field: ${excerpt(trimmed)}`); + } + + const record = parsed as { + type?: string; + message?: string; + text?: unknown; + stopReason?: unknown; + }; + if (record.type === 'error') { + throw new Error(record.message || 'grok CLI returned an error'); + } + if (typeof record.text !== 'string') { + throw new Error(`grok CLI JSON has no text field: ${excerpt(trimmed)}`); + } + + const content = record.text.trim(); + if (!content) { + throw new Error('grok CLI returned empty text'); + } + + const rawReason = + record.stopReason === undefined || record.stopReason === null + ? '' + : String(record.stopReason).trim(); + if (rawReason.toLowerCase() !== 'end_turn') { + throw new Error( + rawReason + ? `grok CLI stopped with stopReason=${rawReason}` + : 'grok CLI JSON is missing stopReason=end_turn', + ); + } + return content; +} + +/** + * Call Grok Build in headless mode and return the assistant text. + * + * `--cwd` is an empty temp directory (not the repo) so project AGENTS.md + * files are not injected into wiki generation. + */ +export async function callGrokLLM( + prompt: string, + config: GrokConfig, + systemPrompt?: string, + options?: CallLLMOptions, +): Promise { + const grokCmd = getDetectedGrokCommand(); + if (!grokCmd) { + throw new Error( + 'Grok CLI not found. Install Grok Build and ensure `grok` is on PATH. Run `grok login` if unauthenticated.', + ); + } + + const fullPrompt = systemPrompt ? `${systemPrompt}\n\n---\n\n${prompt}` : prompt; + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-wiki-grok-')); + const promptPath = path.join(tempDir, 'prompt.txt'); + const childLifecycle = { spawned: false, closed: false }; + + try { + await fs.writeFile(promptPath, fullPrompt, 'utf-8'); + + const args = [ + '--prompt-file', + promptPath, + '--output-format', + 'json', + '--max-turns', + GROK_MAX_TURNS, + '--no-plan', + '--no-subagents', + '--disable-web-search', + '--disallowed-tools', + GROK_DISALLOWED_TOOLS, + '--sandbox', + GROK_SANDBOX_PROFILE, + '--cwd', + tempDir, + ]; + if (config.model) { + args.push('--model', config.model); + } + + verboseLog( + 'Spawning:', + grokCmd.command, + [...grokCmd.argsPrefix, ...args].join(' ').replace(promptPath, '[prompt-file]'), + ); + if (config.model) { + verboseLog('Model:', config.model); + } + + const content = await runGrok( + grokCmd.command, + [...grokCmd.argsPrefix, ...args], + tempDir, + config, + options, + childLifecycle, + ); + return { content }; + } finally { + // Skip rm while a live child may still be using cwd/sandbox/prompt-file. + if (!childLifecycle.spawned || childLifecycle.closed) { + await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined); + } + } +} + +function runGrok( + command: string, + args: string[], + cwd: string, + config: GrokConfig, + options: CallLLMOptions | undefined, + lifecycle: { spawned: boolean; closed: boolean }, +): Promise { + const startTime = Date.now(); + + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + // Prompt is --prompt-file; an unused stdin pipe can EPIPE the wiki process. + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + env: { + ...process.env, + CI: '1', + }, + }); + lifecycle.spawned = true; + + verboseLog('Process spawned with PID:', child.pid); + + let stdout = ''; + let stderr = ''; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + let settled = false; + let timedOut = false; + let killTimer: ReturnType | undefined; + let killEscalate: ReturnType | undefined; + let hardDeadline: ReturnType | undefined; + const timeoutMs = config.requestTimeoutMs; + const timeoutError = + timeoutMs !== undefined && timeoutMs > 0 + ? new Error( + `grok CLI timed out after ${ + timeoutMs >= 60_000 + ? `${Math.round(timeoutMs / 60_000)}m` + : `${Math.round(timeoutMs / 1_000)}s` + }. Increase --timeout or omit it to disable the request timeout.`, + ) + : undefined; + + const clearKillTimers = () => { + if (killTimer !== undefined) clearTimeout(killTimer); + if (killEscalate !== undefined) clearTimeout(killEscalate); + if (hardDeadline !== undefined) clearTimeout(hardDeadline); + }; + + const rejectOnce = (error: Error) => { + if (settled) return; + settled = true; + clearKillTimers(); + reject(error); + }; + + const resolveOnce = (value: string) => { + if (settled) return; + settled = true; + clearKillTimers(); + resolve(value); + }; + + const KILL_GRACE_MS = 2000; + if (timeoutMs !== undefined && timeoutMs > 0 && timeoutError) { + killTimer = setTimeout(() => { + timedOut = true; + killChildTree(child); + killEscalate = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch { + // Process may have already exited. + } + hardDeadline = setTimeout(() => { + rejectOnce(timeoutError); + }, KILL_GRACE_MS); + }, KILL_GRACE_MS); + }, timeoutMs); + } + + child.stdout.on('data', (chunk: Buffer) => { + const chunkStr = stdoutDecoder.write(chunk); + stdout += chunkStr; + options?.onChunk?.(stdout.length); + }); + + child.stderr.on('data', (chunk: Buffer) => { + stderr += stderrDecoder.write(chunk); + }); + + child.on('close', (code) => { + const alreadySettled = settled; + lifecycle.closed = true; + stdout += stdoutDecoder.end(); + stderr += stderrDecoder.end(); + verboseLog( + `Process exited with code ${code} after ${((Date.now() - startTime) / 1000).toFixed(1)}s`, + ); + + if (timedOut && timeoutError) { + rejectOnce(timeoutError); + } else if (code !== 0) { + const details = stderr.trim() || stdout.trim(); + rejectOnce(new Error(`grok CLI exited with code ${code}: ${details}`)); + } else { + try { + resolveOnce(parseGrokOutput(stdout)); + } catch (err) { + rejectOnce(err instanceof Error ? err : new Error(String(err))); + } + } + + if (alreadySettled) { + void fs.rm(cwd, { recursive: true, force: true }).catch(() => undefined); + } + }); + + child.on('error', (err) => { + lifecycle.closed = true; + rejectOnce(new Error(`Failed to spawn grok CLI: ${err.message}`)); + }); + }); +} diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts index 9e5988550..b8af62099 100644 --- a/gitnexus/src/core/wiki/llm-client.ts +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -18,6 +18,7 @@ export type LLMProvider = | 'claude' | 'codex' | 'opencode' + | 'grok' | 'minimax'; export const MINIMAX_OPENAI_BASE_URLS = { @@ -112,12 +113,15 @@ export async function resolveLLMConfig(overrides?: Partial): Promise< ? savedConfig.codexModel : savedProvider === 'opencode' ? savedConfig.opencodeModel - : undefined; + : savedProvider === 'grok' + ? savedConfig.grokModel + : undefined; const localProvider = savedProvider === 'cursor' || savedProvider === 'claude' || savedProvider === 'codex' || - savedProvider === 'opencode'; + savedProvider === 'opencode' || + savedProvider === 'grok'; const apiKey = overrides?.apiKey || diff --git a/gitnexus/src/core/wiki/local-cli-client.ts b/gitnexus/src/core/wiki/local-cli-client.ts index 4edfe57ec..30ffa86ac 100644 --- a/gitnexus/src/core/wiki/local-cli-client.ts +++ b/gitnexus/src/core/wiki/local-cli-client.ts @@ -29,12 +29,14 @@ const COMMANDS: Record = { opencode: 'opencode', }; -interface LocalCommand { +export interface ResolvedCliCommand { displayName: string; command: string; argsPrefix: string[]; } +type LocalCommand = ResolvedCliCommand; + function killChildTree(child: import('child_process').ChildProcess): void { if (process.platform === 'win32' && child.pid !== undefined) { try { @@ -387,10 +389,33 @@ function getDetectedCommand(provider: LocalAgentProvider): LocalCommand | null { return cachedCommands.get(provider) ?? null; } +/** + * Resolve a PATH command for spawn/execFile without a shell. + * On Windows, npm shims are `.cmd` files that Node cannot execFile/spawn + * directly — prefer a native `.exe` from `where.exe`, else `cmd.exe /d /s /c`. + */ +export function resolveWindowsCliCommand(displayName: string): ResolvedCliCommand { + if (process.platform !== 'win32') { + return { displayName, command: displayName, argsPrefix: [] }; + } + + const located = findWindowsCommand(`${displayName}.cmd`) || findWindowsCommand(displayName); + if (located && /\.exe$/i.test(located)) { + return { displayName, command: located, argsPrefix: [] }; + } + + // Last-resort fallback for installations that only expose a .cmd shim. + return { + displayName, + command: process.env.ComSpec || 'cmd.exe', + argsPrefix: ['/d', '/s', '/c', displayName], + }; +} + function resolveLocalCommand(provider: LocalAgentProvider): LocalCommand { const displayName = COMMANDS[provider]; if (process.platform !== 'win32') { - return { displayName, command: displayName, argsPrefix: [] }; + return resolveWindowsCliCommand(displayName); } const npmBin = findWindowsCommand(`${displayName}.cmd`) || findWindowsCommand(displayName); @@ -418,14 +443,7 @@ function resolveLocalCommand(provider: LocalAgentProvider): LocalCommand { } } - // Last-resort fallback for non-npm Windows installations that only expose a - // .cmd shim. Prompts are passed via stdin, so repo content is not placed on - // the command line. - return { - displayName, - command: process.env.ComSpec || 'cmd.exe', - argsPrefix: ['/d', '/s', '/c', displayName], - }; + return resolveWindowsCliCommand(displayName); } function findWindowsCommand(command: string): string | null { diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index ce2e594cf..b223d5f69 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -1582,11 +1582,13 @@ export interface CLIConfig { | 'claude' | 'codex' | 'opencode' + | 'grok' | 'minimax'; cursorModel?: string; claudeModel?: string; codexModel?: string; opencodeModel?: string; + grokModel?: string; /** Azure api-version query param (e.g. '2024-10-21'). Only used when provider is 'azure'. */ apiVersion?: string; /** Set true when the deployment is a reasoning model (o1, o3, o4-mini). Auto-detected for OpenAI; must be set for Azure deployments. */ diff --git a/gitnexus/test/integration/cli-e2e.test.ts b/gitnexus/test/integration/cli-e2e.test.ts index 957a3f4e7..d44b5c4a8 100644 --- a/gitnexus/test/integration/cli-e2e.test.ts +++ b/gitnexus/test/integration/cli-e2e.test.ts @@ -1150,6 +1150,7 @@ describe('CLI end-to-end', () => { expect(result.stdout).toContain('--provider '); expect(result.stdout).toContain('claude'); expect(result.stdout).toContain('codex'); + expect(result.stdout).toContain('grok'); expect(result.stdout).toContain('--review'); expect(result.stdout).toContain('-v, --verbose'); expect(result.stdout).toContain('--model '); @@ -1230,6 +1231,14 @@ describe('CLI end-to-end', () => { expect(combined).not.toMatch(/API key:/); }); + it('wiki --provider grok without API key does not prompt for key in non-TTY', () => { + const result = runCliRaw(['wiki', MINI_REPO, '--provider', 'grok'], repoRoot, 15000); + if (result.status === null) return; + + const combined = result.stdout + result.stderr; + expect(combined).not.toMatch(/API key:/); + }); + it('wiki --help includes --verbose flag description', () => { const result = runCliRaw(['wiki', '--help'], repoRoot); if (result.status === null) return; diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index b0c276bc2..a35541d18 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -249,6 +249,7 @@ describe('CLI help surface', () => { expect(result.stdout).toContain('--provider '); expect(result.stdout).toContain('claude'); expect(result.stdout).toContain('codex'); + expect(result.stdout).toContain('grok'); expect(result.stdout).toContain('--review'); expect(result.stdout).toContain('-v, --verbose'); expect(result.stdout).toContain('--model '); diff --git a/gitnexus/test/unit/local-cli-subprocess.test.ts b/gitnexus/test/unit/local-cli-subprocess.test.ts index ce0e0a2f8..9e46f75f4 100644 --- a/gitnexus/test/unit/local-cli-subprocess.test.ts +++ b/gitnexus/test/unit/local-cli-subprocess.test.ts @@ -14,6 +14,7 @@ function makeFakeChild(opts?: { stdout?: string; stderr?: string; stdinEndBehavior?: 'normal' | 'epipe'; + closeOnSpawn?: boolean; }) { const child = new EventEmitter() as any; child.stdout = new EventEmitter(); @@ -23,8 +24,7 @@ function makeFakeChild(opts?: { child.kill = vi.fn(); let stdinContent = ''; - child.stdin.end = vi.fn((data?: string) => { - if (data) stdinContent += data; + const finish = () => { queueMicrotask(() => { if (opts?.stdinEndBehavior === 'epipe') { child.stdin.emit('error', new Error('write EPIPE')); @@ -37,9 +37,14 @@ function makeFakeChild(opts?: { } child.emit('close', opts?.exitCode ?? 0); }); + }; + + child.stdin.end = vi.fn((data?: string) => { + if (data) stdinContent += data; + if (!opts?.closeOnSpawn) finish(); }); - return { child, getStdin: () => stdinContent }; + return { child, getStdin: () => stdinContent, complete: finish }; } // ─── Claude CLI argv contract ───────────────────────────────────────── @@ -623,3 +628,614 @@ describe('Codex CLI flag contract snapshot', () => { expect(args[args.length - 1]).toBe('-'); }); }); + +// ─── Grok CLI subprocess contract ───────────────────────────────────── + +describe('Grok CLI subprocess contract', () => { + let spawnSpy: ReturnType; + let fakeChild: ReturnType; + + function spawnGrokChild(onSpawn?: (...args: unknown[]) => void) { + spawnSpy = vi.fn((...args: unknown[]) => { + onSpawn?.(...args); + fakeChild.complete(); + return fakeChild.child; + }); + } + + function grokFake( + opts: Parameters[0] = { + stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }), + }, + onSpawn?: (...args: unknown[]) => void, + ) { + fakeChild = makeFakeChild({ ...opts, closeOnSpawn: true }); + spawnGrokChild(onSpawn); + } + + beforeEach(() => { + vi.resetModules(); + grokFake(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + async function loadGrokClient() { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockReturnValue('grok 1.0.5'), + execSync: vi.fn().mockReturnValue('grok 1.0.5'), + spawn: spawnSpy, + })); + return import('../../src/core/wiki/grok-client.js'); + } + + it('detectGrokCLI returns grok when grok --version succeeds and caches the result', async () => { + const execFileSync = vi.fn().mockReturnValue('grok 1.0.5'); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync, + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { detectGrokCLI } = await import('../../src/core/wiki/grok-client.js'); + + expect(detectGrokCLI()).toBe('grok'); + expect(detectGrokCLI()).toBe('grok'); + const versionCalls = execFileSync.mock.calls.filter( + (call: unknown[]) => Array.isArray(call[1]) && (call[1] as string[]).includes('--version'), + ); + expect(versionCalls).toHaveLength(1); + }); + + it('detectGrokCLI returns null on ENOENT', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockImplementation(() => { + const err = new Error('not found') as NodeJS.ErrnoException; + err.code = 'ENOENT'; + throw err; + }), + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { detectGrokCLI } = await import('../../src/core/wiki/grok-client.js'); + + expect(detectGrokCLI()).toBeNull(); + }); + + it('spawns grok with prompt-file, json output, max-turns 15, a tool denylist, and a strict sandbox', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('user prompt', {}); + + const args = spawnSpy.mock.calls[0][1] as string[]; + expect(args).toContain('--prompt-file'); + expect(args).toContain('--output-format'); + expect(args).toContain('json'); + const turnsIdx = args.indexOf('--max-turns'); + expect(turnsIdx).toBeGreaterThanOrEqual(0); + expect(args[turnsIdx + 1]).toBe('15'); + expect(args).toContain('--no-plan'); + expect(args).toContain('--no-subagents'); + expect(args).toContain('--disable-web-search'); + expect(args).toContain('--disallowed-tools'); + const denyIdx = args.indexOf('--disallowed-tools'); + expect(args[denyIdx + 1]).toBe( + 'run_terminal_cmd,search_replace,web_search,web_fetch,spawn_subagent', + ); + expect(args).toContain('--sandbox'); + const sandboxIdx = args.indexOf('--sandbox'); + expect(args[sandboxIdx + 1]).toBe('strict'); + expect(args).not.toContain('--tools'); + expect(args).not.toContain('--yolo'); + expect(args).not.toContain('--always-approve'); + expect(args.some((arg) => arg.includes('user prompt'))).toBe(false); + }); + + it('writes system + separator + user prompt to --prompt-file, not stdin', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('user prompt', {}, 'system prompt'); + + const args = spawnSpy.mock.calls[0][1] as string[]; + const fileIdx = args.indexOf('--prompt-file'); + const promptPath = args[fileIdx + 1]; + // File is removed after the call; capture contents via spawn-time read. + // The implementation writes before spawn, so the spy can read it if we hook spawn. + expect(fakeChild.getStdin()).toBe(''); + expect(promptPath).toContain('gitnexus-wiki-grok-'); + }); + + it('writes the concatenated prompt before spawn', async () => { + let promptContents = ''; + const fs = await import('fs'); + grokFake( + { stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }) }, + (..._spawnArgs: unknown[]) => { + const args = _spawnArgs[1] as string[]; + const fileIdx = args.indexOf('--prompt-file'); + promptContents = fs.readFileSync(args[fileIdx + 1], 'utf-8'); + }, + ); + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('user prompt', {}, 'system prompt'); + + expect(promptContents).toBe('system prompt\n\n---\n\nuser prompt'); + }); + + it('passes --cwd to an empty temp dir, not a repo path', async () => { + grokFake( + { stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }) }, + (_cmd, args, opts) => { + const argv = args as string[]; + const spawnOpts = opts as { cwd?: string }; + expect(argv).toContain('--cwd'); + const cwdIdx = argv.indexOf('--cwd'); + expect(argv[cwdIdx + 1]).toContain('gitnexus-wiki-grok-'); + expect(argv[cwdIdx + 1]).toBe(spawnOpts.cwd); + }, + ); + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + }); + + it('appends --model only when model is set', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', { model: 'grok-build' }); + + const args = spawnSpy.mock.calls[0][1] as string[]; + expect(args).toContain('--model'); + expect(args).toContain('grok-build'); + }); + + it('does not include --model when model is empty', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + + const args = spawnSpy.mock.calls[0][1] as string[]; + expect(args).not.toContain('--model'); + }); + + it('parses JSON text field as the LLM content', async () => { + const { callGrokLLM } = await loadGrokClient(); + + const result = await callGrokLLM('prompt', {}); + expect(result.content).toBe('wiki page'); + }); + + it('rejects with exit code and stderr on non-zero exit', async () => { + grokFake({ exitCode: 1, stderr: 'auth required' }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI exited with code 1: auth required', + ); + }); + + it('rejects when grok returns a JSON error object', async () => { + grokFake({ + stdout: JSON.stringify({ type: 'error', message: 'session failed' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow('session failed'); + }); + + it('rejects when JSON text is empty', async () => { + grokFake({ stdout: JSON.stringify({ text: ' ' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow('grok CLI returned empty text'); + }); + + it('rejects incomplete stopReason even when text is present', async () => { + grokFake({ + stdout: JSON.stringify({ text: 'cut off mid-sent', stopReason: 'max_tokens' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/stopReason=max_tokens/); + }); + + it('rejects PascalCase incomplete stopReason', async () => { + grokFake({ + stdout: JSON.stringify({ text: 'partial', stopReason: 'MaxTokens' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/stopReason=MaxTokens/); + }); + + it('accepts end_turn stopReason with text', async () => { + grokFake({ + stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }), + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).resolves.toEqual({ content: 'wiki page' }); + }); + + it('rejects omitted stopReason even when text is non-empty', async () => { + grokFake({ stdout: JSON.stringify({ text: 'wiki page' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI JSON is missing stopReason=end_turn', + ); + }); + + it('rejects null stopReason even when text is non-empty', async () => { + grokFake({ stdout: JSON.stringify({ text: 'wiki page', stopReason: null }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI JSON is missing stopReason=end_turn', + ); + }); + + it('rejects empty stopReason even when text is non-empty', async () => { + grokFake({ stdout: JSON.stringify({ text: 'wiki page', stopReason: '' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow( + 'grok CLI JSON is missing stopReason=end_turn', + ); + }); + + it('rejects empty stdout with a dedicated message', async () => { + grokFake({ stdout: ' ' }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow('grok CLI returned empty output'); + }); + + it('rejects non-JSON stdout with an excerpt', async () => { + grokFake({ stdout: 'Update available\nnot-json' }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/non-JSON output:.*Update available/s); + }); + + it('rejects JSON without a text field with an excerpt', async () => { + grokFake({ stdout: JSON.stringify({ sessionId: 'abc' }) }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/no text field:.*"sessionId"/s); + }); + + it('removes the temp prompt file and cwd after success', async () => { + const fs = await import('fs'); + let promptPath = ''; + let cwdPath = ''; + grokFake( + { stdout: JSON.stringify({ text: 'wiki page', stopReason: 'end_turn' }) }, + (_cmd, args) => { + const argv = args as string[]; + const fileIdx = argv.indexOf('--prompt-file'); + promptPath = argv[fileIdx + 1]; + cwdPath = argv[argv.indexOf('--cwd') + 1]; + expect(fs.existsSync(promptPath)).toBe(true); + }, + ); + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + + expect(fs.existsSync(promptPath)).toBe(false); + expect(fs.existsSync(cwdPath)).toBe(false); + }); + + it('removes the temp dir after failure', async () => { + const fs = await import('fs'); + let cwdPath = ''; + grokFake({ exitCode: 1, stderr: 'nope' }, (_cmd, args) => { + const argv = args as string[]; + cwdPath = argv[argv.indexOf('--cwd') + 1]; + }); + const { callGrokLLM } = await loadGrokClient(); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/exited with code 1/); + expect(fs.existsSync(cwdPath)).toBe(false); + }); + + it('throws when grok CLI is not on PATH', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockImplementation(() => { + const err = new Error('not found') as NodeJS.ErrnoException; + err.code = 'ENOENT'; + throw err; + }), + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { callGrokLLM } = await import('../../src/core/wiki/grok-client.js'); + + await expect(callGrokLLM('prompt', {})).rejects.toThrow(/Grok CLI not found/); + }); + + it('sets CI=1 and windowsHide=true in spawn options', async () => { + const { callGrokLLM } = await loadGrokClient(); + + await callGrokLLM('prompt', {}); + + const spawnOpts = spawnSpy.mock.calls[0][2]; + expect(spawnOpts.env.CI).toBe('1'); + expect(spawnOpts.windowsHide).toBe(true); + }); + + it('on Windows detects grok and spawns via cmd.exe /d /s /c, not a .cmd path', async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, 'platform', { value: 'win32' }); + + try { + const execFileSync = vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'where.exe') return 'C:\\Users\\me\\AppData\\Roaming\\npm\\grok.cmd\r\n'; + return 'grok 1.0.5'; + }); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync, + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { detectGrokCLI, callGrokLLM } = await import('../../src/core/wiki/grok-client.js'); + + expect(detectGrokCLI()).toBe('grok'); + await callGrokLLM('prompt', {}); + + const spawnCmd = spawnSpy.mock.calls[0][0] as string; + const spawnArgs = spawnSpy.mock.calls[0][1] as string[]; + expect(spawnCmd.toLowerCase()).not.toMatch(/\.cmd$/); + expect(spawnCmd).toBe(process.env.ComSpec || 'cmd.exe'); + expect(spawnArgs.slice(0, 4)).toEqual(['/d', '/s', '/c', 'grok']); + expect(spawnArgs).toContain('--prompt-file'); + expect(spawnArgs).toContain('--sandbox'); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + } + }); + + it('on Windows uses ComSpec when set instead of cmd.exe', async () => { + const originalPlatform = process.platform; + const originalComSpec = process.env.ComSpec; + Object.defineProperty(process, 'platform', { value: 'win32' }); + process.env.ComSpec = 'C:\\Windows\\System32\\cmd.exe'; + + try { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync: vi.fn().mockReturnValue('grok 1.0.5'), + execSync: vi.fn(), + spawn: spawnSpy, + })); + const { callGrokLLM } = await import('../../src/core/wiki/grok-client.js'); + + await callGrokLLM('prompt', {}); + + expect(spawnSpy.mock.calls[0][0]).toBe('C:\\Windows\\System32\\cmd.exe'); + expect((spawnSpy.mock.calls[0][1] as string[]).slice(0, 4)).toEqual([ + '/d', + '/s', + '/c', + 'grok', + ]); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + if (originalComSpec === undefined) delete process.env.ComSpec; + else process.env.ComSpec = originalComSpec; + } + }); +}); + +describe('Grok CLI timeout', () => { + beforeEach(() => { + vi.resetModules(); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + function hangingChild() { + const child = new EventEmitter() as any; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.stdin = new EventEmitter() as any; + child.pid = 99; + child.kill = vi.fn(); + child.stdin.end = vi.fn(); + return child; + } + + async function loadGrokWithChild(child: any, execFileSyncImpl?: (...args: any[]) => unknown) { + const spawnSpy = vi.fn(() => child); + const execFileSync = + execFileSyncImpl ?? + vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'taskkill') return ''; + return 'grok 1.0.5'; + }); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('child_process', () => ({ + execFileSync, + execSync: vi.fn(), + spawn: spawnSpy, + })); + const mod = await import('../../src/core/wiki/grok-client.js'); + return { ...mod, spawnSpy, execFileSync }; + } + + async function waitForSpawn(spawnSpy: ReturnType) { + const deadline = Date.now() + 2000; + while (spawnSpy.mock.calls.length === 0 && Date.now() < deadline) { + await Promise.resolve(); + await new Promise((r) => setImmediate(r)); + } + expect(spawnSpy).toHaveBeenCalled(); + } + + it('kills child process after requestTimeoutMs and rejects with timeout error', async () => { + const child = hangingChild(); + const { callGrokLLM, spawnSpy, execFileSync } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 5000 }); + await waitForSpawn(spawnSpy); + vi.advanceTimersByTime(5000); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 5s'); + if (process.platform === 'win32') { + const taskkillCalls = execFileSync.mock.calls.filter((c: unknown[]) => c[0] === 'taskkill'); + expect(taskkillCalls.length).toBeGreaterThan(0); + } else { + expect(child.kill).toHaveBeenCalled(); + } + }); + + it('uses taskkill /T /F /PID on Windows for process-tree kill', async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, 'platform', { value: 'win32' }); + try { + const child = hangingChild(); + child.pid = 42; + const execFileSync = vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'taskkill') return ''; + if (cmd === 'where.exe') return 'C:\\npm\\grok.cmd\n'; + return 'grok 1.0.5'; + }); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child, execFileSync); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 3000 }); + await waitForSpawn(spawnSpy); + vi.advanceTimersByTime(3000); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 3s'); + const taskkillCalls = execFileSync.mock.calls.filter((c: unknown[]) => c[0] === 'taskkill'); + expect(taskkillCalls.length).toBe(1); + expect(taskkillCalls[0][1]).toEqual(['/T', '/F', '/PID', '42']); + expect(child.kill).not.toHaveBeenCalled(); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + } + }); + + it('falls back to child.kill() when taskkill fails on Windows', async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, 'platform', { value: 'win32' }); + try { + const child = hangingChild(); + child.pid = 42; + const execFileSync = vi.fn().mockImplementation((cmd: string) => { + if (cmd === 'taskkill') throw new Error('taskkill: process not found'); + if (cmd === 'where.exe') return 'C:\\npm\\grok.cmd\n'; + return 'grok 1.0.5'; + }); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child, execFileSync); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 2000 }); + await waitForSpawn(spawnSpy); + vi.advanceTimersByTime(2000); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 2s'); + expect(child.kill).toHaveBeenCalled(); + } finally { + Object.defineProperty(process, 'platform', { value: originalPlatform }); + } + }); + + it('does not set a kill timer when requestTimeoutMs is undefined', async () => { + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', {}); + await waitForSpawn(spawnSpy); + child.stdout.emit('data', Buffer.from(JSON.stringify({ text: 'ok', stopReason: 'end_turn' }))); + child.emit('close', 0); + await expect(promise).resolves.toEqual({ content: 'ok' }); + expect(child.kill).not.toHaveBeenCalled(); + }); + + it('ignores grok stdin so an unused pipe cannot EPIPE', async () => { + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', {}); + await waitForSpawn(spawnSpy); + const spawnOpts = spawnSpy.mock.calls[0][2] as { stdio?: unknown }; + expect(spawnOpts.stdio).toEqual(['ignore', 'pipe', 'pipe']); + expect(child.stdin.end).not.toHaveBeenCalled(); + child.stdout.emit('data', Buffer.from(JSON.stringify({ text: 'ok', stopReason: 'end_turn' }))); + child.emit('close', 0); + await expect(promise).resolves.toEqual({ content: 'ok' }); + }); + + it('does not remove the temp dir until the child closes after timeout', async () => { + const fs = await import('fs'); + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 5000 }); + await waitForSpawn(spawnSpy); + const cwd = (spawnSpy.mock.calls[0][2] as { cwd: string }).cwd; + expect(fs.existsSync(cwd)).toBe(true); + let state: 'pending' | 'ok' | 'err' = 'pending'; + void promise.then( + () => { + state = 'ok'; + }, + () => { + state = 'err'; + }, + ); + vi.advanceTimersByTime(5000); + for (let i = 0; i < 30; i++) { + await new Promise((r) => setImmediate(r)); + } + expect(state).toBe('pending'); + expect(fs.existsSync(cwd)).toBe(true); + child.emit('close', null); + await expect(promise).rejects.toThrow('grok CLI timed out after 5s'); + expect(fs.existsSync(cwd)).toBe(false); + }); + + it('does not remove the temp dir when the hard deadline rejects without a close event', async () => { + const fs = await import('fs'); + const child = hangingChild(); + const { callGrokLLM, spawnSpy } = await loadGrokWithChild(child); + const promise = callGrokLLM('prompt', { requestTimeoutMs: 5000 }); + await waitForSpawn(spawnSpy); + const cwd = (spawnSpy.mock.calls[0][2] as { cwd: string }).cwd; + expect(fs.existsSync(cwd)).toBe(true); + + vi.advanceTimersByTime(5000 + 2000 + 2000); + await expect(promise).rejects.toThrow('grok CLI timed out after 5s'); + expect(fs.existsSync(cwd)).toBe(true); + + // Late close rms via fire-and-forget fs.rm. Fake setTimeout would starve + // that I/O; switch to real timers and poll until the dir is gone. + vi.useRealTimers(); + child.emit('close', null); + const goneDeadline = Date.now() + 2000; + while (fs.existsSync(cwd) && Date.now() < goneDeadline) { + await new Promise((r) => setTimeout(r, 10)); + } + expect(fs.existsSync(cwd)).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/wiki-flags.test.ts b/gitnexus/test/unit/wiki-flags.test.ts index 69ae6762a..49f4cc08e 100644 --- a/gitnexus/test/unit/wiki-flags.test.ts +++ b/gitnexus/test/unit/wiki-flags.test.ts @@ -223,6 +223,42 @@ describe('resolveLLMConfig', () => { expect(config.model).toBe(''); }); + it('uses grokModel when provider is grok', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('../../src/storage/repo-manager.js', () => ({ + loadCLIConfig: vi.fn().mockResolvedValue({ + provider: 'grok', + grokModel: 'grok-build', + }), + })); + + const { resolveLLMConfig } = await import('../../src/core/wiki/llm-client.js'); + const config = await resolveLLMConfig({ provider: 'grok' }); + + expect(config.provider).toBe('grok'); + expect(config.model).toBe('grok-build'); + }); + + it('does not inherit HTTP model defaults for grok local provider', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + vi.doMock('../../src/storage/repo-manager.js', () => ({ + loadCLIConfig: vi.fn().mockResolvedValue({ + provider: 'openai', + model: 'legacy-http-model', + }), + })); + + const { resolveLLMConfig } = await import('../../src/core/wiki/llm-client.js'); + const config = await resolveLLMConfig({ provider: 'grok' }); + + expect(config.provider).toBe('grok'); + expect(config.model).toBe(''); + }); + it('does not inherit HTTP model defaults for local CLI providers', async () => { vi.doMock('../../src/storage/repo-manager.js', () => ({ loadCLIConfig: vi.fn().mockResolvedValue({ @@ -344,6 +380,9 @@ describe('wikiCommand provider switch persistence', () => { ) { const saveCLIConfig = vi.fn(); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); vi.doMock('../../src/storage/git.js', () => ({ getGitRoot: vi.fn(), isGitRepo: vi.fn().mockReturnValue(true), @@ -458,6 +497,26 @@ describe('wikiCommand provider switch persistence', () => { isReasoningModel: false, }); }); + + it('saves grok model to grokModel instead of the HTTP model field', async () => { + const saveCLIConfig = await saveProviderSwitch( + { + provider: 'minimax', + apiKey: 'old-minimax-key', + baseUrl: 'https://api.minimax.io/v1', + model: 'MiniMax-M3', + }, + { + provider: 'grok', + model: 'grok-build', + }, + ); + + const saved = saveCLIConfig.mock.calls[0][0] as Record; + expect(saved.provider).toBe('grok'); + expect(saved.grokModel).toBe('grok-build'); + expect(saved.model).toBeUndefined(); + }); }); // ─── --verbose flag ────────────────────────────────────────────────── @@ -1015,6 +1074,16 @@ describe('CLI config round-trip with cursor provider', () => { expect(loaded.apiKey).toBeUndefined(); }); + it('saves and loads grok provider config correctly', async () => { + const config = { provider: 'grok', grokModel: 'grok-build' }; + await fs.writeFile(configPath, JSON.stringify(config, null, 2)); + + const loaded = JSON.parse(await fs.readFile(configPath, 'utf-8')); + expect(loaded.provider).toBe('grok'); + expect(loaded.grokModel).toBe('grok-build'); + expect(loaded.apiKey).toBeUndefined(); + }); + it('saves openai provider config with model and apiKey', async () => { const config = { provider: 'openai', @@ -1245,6 +1314,55 @@ describe('WikiGenerator invokeLLM routing', () => { expect(result.content).toBe('opencode response'); }); + it('routes to callGrokLLM when provider is grok', async () => { + vi.doMock('../../src/core/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn() }, + })); + const cursorClient = await import('../../src/core/wiki/cursor-client.js'); + const localClient = await import('../../src/core/wiki/local-cli-client.js'); + const grokClient = await import('../../src/core/wiki/grok-client.js'); + const llmClient = await import('../../src/core/wiki/llm-client.js'); + + const cursorSpy = vi + .spyOn(cursorClient, 'callCursorLLM') + .mockResolvedValue({ content: 'cursor response' }); + const claudeSpy = vi + .spyOn(localClient, 'callClaudeLLM') + .mockResolvedValue({ content: 'claude response' }); + const grokSpy = vi + .spyOn(grokClient, 'callGrokLLM') + .mockResolvedValue({ content: 'grok response' }); + const openaiSpy = vi + .spyOn(llmClient, 'callLLM') + .mockResolvedValue({ content: 'openai response' }); + + const { WikiGenerator } = await import('../../src/core/wiki/generator.js'); + + const storagePath = path.join(tmpDir, 'storage'); + const wikiDir = path.join(storagePath, 'wiki'); + await fs.mkdir(wikiDir, { recursive: true }); + + const repoPath = path.join(tmpDir, 'repo'); + await fs.mkdir(repoPath, { recursive: true }); + + const generator = new WikiGenerator(repoPath, storagePath, path.join(storagePath, 'lbug'), { + apiKey: '', + baseUrl: '', + model: 'grok-build', + maxTokens: 1000, + temperature: 0, + provider: 'grok', + }); + + const result = await (generator as any).invokeLLM('test prompt', 'system prompt'); + + expect(grokSpy).toHaveBeenCalledTimes(1); + expect(claudeSpy).not.toHaveBeenCalled(); + expect(cursorSpy).not.toHaveBeenCalled(); + expect(openaiSpy).not.toHaveBeenCalled(); + expect(result.content).toBe('grok response'); + }); + it('routes to callLLM when provider is openai', async () => { const cursorClient = await import('../../src/core/wiki/cursor-client.js'); const localClient = await import('../../src/core/wiki/local-cli-client.js');