From 568ebdc1d9e6fce80a4da306bcb1ecebb20b60e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sun, 1 Nov 2026 11:08:20 +0000 Subject: [PATCH] perf(analyze): skip unchanged dirty snapshots (#3796) * perf(analyze): skip unchanged dirty snapshots * fix(analyze): harden dirty snapshot freshness Capture freshness hashes before parsing so edits during analysis remain detectable on the next run. Exercise the dirty snapshot and linked-worktree contracts in the native cross-platform test subset. * test: preserve checkout bytes on Windows --------- Co-authored-by: Gergo Magyar --- gitnexus/bench/analyze-phase-breakdown.md | 340 +++++++++++++++++- gitnexus/scripts/cross-platform-tests.ts | 3 + gitnexus/src/core/run-analyze.ts | 101 +++++- .../analyze-dirty-snapshot.test.ts | 286 +++++++++++++++ .../incremental-worktree-context.test.ts | 19 +- gitnexus/vitest.config.ts | 2 + 6 files changed, 720 insertions(+), 31 deletions(-) create mode 100644 gitnexus/test/integration/analyze-dirty-snapshot.test.ts diff --git a/gitnexus/bench/analyze-phase-breakdown.md b/gitnexus/bench/analyze-phase-breakdown.md index e04d48628..cfd4dd93e 100644 --- a/gitnexus/bench/analyze-phase-breakdown.md +++ b/gitnexus/bench/analyze-phase-breakdown.md @@ -1,8 +1,12 @@ # Analyze phase breakdown — where the time actually goes -Measured 2026-09-06/07 while landing #3194, #3196 and #3200. Records what the -`analyze` pipeline costs per phase, and — as importantly — the optimizations -that were measured and **rejected**, so the next person does not re-derive them. +Updated 2026-11-01 with production hardening and measurements from main on +LadybugDB **0.22.0** and an experimental fast path for unchanged dirty checkouts. +Records what the `analyze` +pipeline costs per phase and the results of optimization experiments. +The September measurements remain below as historical +evidence; the corpus and analyzer have changed, so they are not a controlled +before/after comparison. Unlike `parse-throughput.md` (a synthetic-fixture scaffold), these numbers come from a real repo corpus. They are not a CI gate; the gate for the parse @@ -10,7 +14,315 @@ dispatch path is `bench/parse-dispatch-rounds/`. --- -## Method, and one trap that invalidates everything +## Current measurements — 2026-10-31 + +### Environment and timing boundaries + +- Analyzer and corpus: main at `82966afa64ec0cbd0e09084cf7af89067ccf0c2d`. + Built once with `npm ci`; no changes to the build between samples. +- Dependencies: `@ladybugdb/core` **0.22.0**, packaged FTS **0.22.0**, + `tree-sitter` **0.25.1**. Node **v22.23.3**, Linux x64, Intel Core i9-12900K, + 24 logical CPUs, about 31.2 GiB RAM. This is a shared development host. +- Corpus: a detached git checkout of that commit, **7,726 tracked paths**, + **3,353 indexed files**, **3,090 parseable files** in **276 chunks**. + Both runtime and corpus are on local `/tmp` overlay storage, outside the + workspace's 9p mount. Index storage and GitNexus home are disposable and + isolated; shared-store reuse is disabled. +- Options: 16 workers, 8 GiB parent V8 heap limit, full content retention and + FTS, no embeddings or PDG, `skipAgentsMd: true`, `skipSkills: true`, + `verbose: true`. No `force` flag. Other analysis limits retain their defaults. +- Total is wall time around `await runFullAnalysis(...)`, after importing the + analyzer. It excludes CLI startup, the analyzer import, post-run graph + hashing, and process shutdown. Each sample runs in a new Node process. +- "Fresh" means no index or parse cache. OS filesystem caches were not flushed; + an earlier setup run had already read the corpus. That run disabled FTS with + `GITNEXUS_LBUG_EXTENSION_INSTALL=never` and is excluded from the results. + Valid runs use `load-only`, which successfully loads the packaged extension + without downloading it. + +Pipeline timings use the existing `NODE_ENV=development` phase logs. +`PROF_SCOPE_RESOLUTION=1` supplies per-language internals and `PROF_LBUG_LOAD=1` +supplies the graph-load interval. FTS is timed from the `fts` progress callback +at 85% to `Search indexes ready` at 90%; verbose callbacks also time each index. +All samples use this same instrumentation. The residual is total time minus +these intervals and the remaining pipeline phases; it is not a CPU profile. +The parse interval includes cache hydration and worker startup; graph loading +includes CSV emission and native COPY. Neither isolates JavaScript CPU time. + +### Fresh index, one-file edits, and unchanged controls + +Run order: one fresh index, an unchanged clean-checkout control, three successive +one-file edits, then a repeat with **no further edit** to the dirty checkout. +The edit replaces `gitnexus/src/cli/update-notice.ts` with its pristine contents +plus `\n// phase-breakdown trial N\n`, for `N = 1, 2, 3`. These are uncommitted, +comment-only changes; they measure the smallest edit path, not API changes, +deletions, or large change sets. + +All values below are seconds. Each column is one complete sample; rounding can +make the components differ from the total by 1ms. + +| Step | Fresh | Edit 1 | Edit 2 | Edit 3 | Dirty repeat | +| -------------------------------- | ----------: | ----------: | ----------: | ----------: | -----------: | +| **Total** | **166.015** | **120.361** | **122.634** | **126.342** | **110.820** | +| Parsing | 62.823 | 6.598 | 6.995 | 6.750 | 6.748 | +| Scope resolution | 70.335 | 73.987 | 75.941 | 77.268 | 87.054 | +| Graph load | 10.145 | 5.141 | 5.251 | 4.813 | 0.007 | +| FTS | 16.072 | 12.792 | 13.095 | 15.731 | 0.510 | +| Other pipeline phases | 3.143 | 3.307 | 3.255 | 3.308 | 1.025 | +| Residual outside those intervals | 3.498 | 18.536 | 18.097 | 18.472 | 15.476 | + +The edit median is **122.634s**, range **120.361–126.342s**, with three samples +on a shared host. This sequence measures the unmodified main baseline. + +The graph contains **117,437 nodes / 352,781 relationships / 3,463 communities / +767 processes**. These reported counts matched across runs. The fresh index +built all **23** configured FTS indexes. Every edit reported: + +- `changedFiles: 1`, `deletedFiles: 0`, `affectedDependents: 6`, + `writeMode: "incremental"`, and no rebuild reasons. +- **22 files reparsed / 3,068 cached**; 273 chunk hits and 3 misses. +- A row-removal set of **71 paths**, followed by a load of **6,941 nodes / + 59,204 relationships**. The write set extends beyond the changed file and its + six importer dependents. +- **9 FTS indexes rebuilt**: File, Function, Class, Method, Interface, TypeAlias, + Const, Property, Route. In Edit 2, Function took **4.885s**, File **4.423s**, + and the remaining seven indexes plus orchestration **3.786s**. + +**The two unchanged controls behave differently.** Immediately after the fresh +index, the clean checkout returned `alreadyUpToDate: true` in **1.245s**, without +running the pipeline. Repeating analysis after Edit 3, with the same uncommitted +contents, took **110.820s** and did not return `alreadyUpToDate`. It reported +**zero changed files**, two reparsed files and 3,088 cache hits, but still spent +**87.054s in scope resolution**. Community/process phases were skipped, the +graph loader received zero nodes and relationships, and only `Class.class_fts` +was rebuilt. This is a measured dirty-worktree repeat, not a full-rebuild +fallback or another edit sample. + +The fresh run and three edits also matched a filtered in-memory graph hash +(excluding node content, Folder/Community/Process nodes, and CONTAINS, +MEMBER_OF, STEP_IN_PROCESS edges). The dirty repeat skips derived graph phases +and is excluded from that comparison. This is not a persisted-graph or query +equivalence test. Runs emitted warnings for two dropped scope sites and for +callable-flow/process limits; the timings describe that default bounded +configuration, not complete graph coverage. + +### Where the edit time goes now + +In Edit 2, the sample with the median total, scope resolution is **61.9%** of +wall time. Summing the existing per-language profiler gives: + +| Scope-resolution work | Seconds | +| ---------------------------------------- | ---------: | +| `emit` | **49.604** | +| `resolve` | 7.043 | +| Other work inside the internal timers | 4.482 | +| Phase time outside those internal timers | 14.812 | +| **Whole phase** | **75.941** | + +`emit` alone is **40.4% of the entire edit run**. TypeScript reports **2,865 / +2,865 pre-extracted hits** in every edit sample, yet resolution and emission +still process the corpus. The 14.812s outside the internal timers includes +ParsedFile loading and other phase work; it was not separately instrumented, +so it must not all be labeled rehydration. + +The separate **18.097s run residual** includes work outside the pipeline, +graph-load and FTS timers, such as incremental deletion, reconciliation, +checkpoints, and publication. Those costs were not individually timed. The +graph-load profiler reports `mode=overlap`; its CSV and COPY subfields must not +be interpreted as separate CPU costs. + +These measurements identified two targets: an early exit for an unchanged dirty +snapshot and reducing repeated scope emission. The early-exit experiment below +demonstrates a speedup for unchanged snapshots. Repeated scope emission remains +the target for actual edits. No scriptc build or alternate native-linking +strategy was measured; these results do not establish a compiler migration's +benefit. + +### Experimental unchanged-dirty fast path — 2026-10-31 + +Candidate: `experiment/unchanged-dirty-fastpath`, based on the same main commit +`82966afa64ec0cbd0e09084cf7af89067ccf0c2d`. The production change is confined to +`src/core/run-analyze.ts`; native dependencies and the storage schema are +unchanged. This is an experimental result, not a released optimization. + +Each variant used its own runtime, corpus checkout, index, and GitNexus home. +Both corpora started with the identical comment-only edit from Edit 3 above. +Both indexes were seeded before timing, with stable runner identities throughout +the measured sequence. Setup runs, including the candidate's rebuild after a +code revision, are excluded. Hardware, options, instrumentation, and timing +boundaries match the baseline section. Our regression tests started after the +timing runs; other workloads on the shared host were not isolated. + +Run order was candidate 1, baseline 1, baseline 2, candidate 2, candidate 3, +baseline 3. Each run used a new Node process and made no source change. + +| Unchanged dirty snapshot | Main baseline (s) | Candidate (s) | +| ------------------------ | ----------------: | ------------: | +| Sample 1 | 102.494 | 1.657 | +| Sample 2 | 101.905 | 1.640 | +| Sample 3 | 102.688 | 1.552 | +| **Median** | **102.494** | **1.640** | + +The ratio of medians is **62.50×**, a **98.40% reduction** in elapsed analysis +time for this workload. All three baseline runs reported zero changed files, +but ran the pipeline and rewrote the database. All three candidate runs returned +`alreadyUpToDate: true` without a pipeline result. SHA-256 of the complete +LadybugDB file was identical before and after each candidate run. Reported graph +counts remained **117,437 nodes / 352,781 relationships** in both variants. + +The candidate retains the existing rebuild and configuration gates. For an +eligible dirty checkout, it verifies the recorded dirty files, enumerates the +current covered paths, and compares their content hashes with the indexed +snapshot. Missing or changed bytes, coverage changes, scan failures, incomplete +scope receipts, and unknown dirty paths fall through to analysis. The full +coverage check also catches edits hidden by Git's `assume-unchanged` flag. +This deliberately conservative path does not accelerate every dirty checkout, +including snapshots containing deleted or unindexed dirty paths. + +For a functional control, both corpora then received the same new exported +function and unique search token in `gitnexus/src/cli/update-notice.ts`. Baseline +took **168.291s**, candidate **149.265s**. Each reported one changed file, +22 reparsed files, 3,068 cached files, an incremental write, and no rebuild +reasons. Both produced **117,438 nodes / 352,782 relationships**, with matching +filtered in-memory graph hashes under the exclusions documented above. +Read-only queries against the persisted databases also returned identical +results: the new function and its location, one File FTS hit for the token, and +all **34** Function-to-Function call edges touching the edited file, including +confidence and reason. These are targeted checks, not exhaustive graph equality. + +This real-edit pair is **not evidence of an edit speedup**: it has one sample per +variant and the shared host's one-minute load rose from 5.62 to 13.85 during the +baseline run. Repeating the candidate after that edit took **1.821s**, returned +already up to date, and again preserved the database bytes. + +Validation covered **158 targeted tests across seven files**, including **13 new +fast-path cases** for native FTS/database preservation, real edits, additions, +deletions, renames, partial reverts, hidden edits, scan-policy changes, recovery, +and scan failures. The initial run passed 136 tests; 22 existing cases failed +with changing-runtime guards or incomplete reads of compiled JavaScript. All +22 passed when rerun in a fresh runtime copy with identical source. Hashes of +all **2,340 compiled build files** stayed unchanged throughout that rerun. +The build, `tsc --noEmit`, formatting, and diff checks passed; targeted ESLint +reported no errors. This was targeted validation, not the full repository suite. + +To reproduce, seed each variant independently with the same dirty source, then +use the harness below in separate processes with no further edits. Keep the +runtime fixed after seeding. Record `alreadyUpToDate`, rebuild reasons, graph +counts, and a database-file hash before/after each repeat. Follow with the same +real edit in both corpora and verify persisted symbol, call-edge, and FTS results. +Do not combine these no-op timings with the actual-edit baseline. + +### Production hardening — 2026-11-01 + +Review found a freshness race in the experiment: per-file hashes were read from +disk after parsing. A file edited between parsing and hashing could therefore +have a new hash beside an older graph; the next dirty repeat incorrectly returned +already up to date. A regression test reproduced that behavior before the fix. + +Git repositories now capture hashes before parsing and retain only paths that +the pipeline actually indexed. An edit during analysis differs from that receipt +on the next run, which reconciles the graph before allowing another unchanged +repeat. This moves the existing content-hash pass and adds a path-only scan; it +does not introduce another full-content hashing pass. The analyzer still reads a +live working tree, not an atomic filesystem snapshot. + +The dedicated native integration suite, `test/integration/analyze-dirty-snapshot.test.ts`, +covers both fresh and incremental versions of the race, as well as the original +fast-path and fallback cases. It is registered in the sequential LadybugDB +project and in the Windows/macOS CI subset. Database preservation uses an exact +byte comparison; the FTS check fails rather than skips when CI requires FTS. + +The hardened build was reseeded and measured against the same pinned main +runtime, with three unchanged dirty repeats per variant. The corpora retained +the identical exported function from the October functional control, so both +contained **117,438 nodes / 352,782 relationships**. The seed rebuild took +150.305s and is excluded. Options and timing boundaries are unchanged. + +| Unchanged dirty snapshot | Main baseline (s) | Hardened candidate (s) | +| ------------------------ | ----------------: | ---------------------: | +| Sample 1 | 134.649 | 2.200 | +| Sample 2 | 127.837 | 1.868 | +| Sample 3 | 101.765 | 1.960 | +| **Median** | **127.837** | **1.960** | + +This is **65.21×** by ratio of medians (**98.47% less elapsed time**). Every +candidate repeat returned already up to date without running the pipeline and +preserved the complete database file's SHA-256. All six runs retained native +FTS and matched graph counts. Run order was candidate, baseline, baseline, +candidate, candidate, baseline. Other validation ran on the same host; observed +one-minute load ranged from 9.23 to 16.69. These results confirm the benefit for +unchanged dirty repeats, not a speedup for real edits or an improvement over the +October speedup ratio. + +A subsequent real-edit control added the same exported function and search +token to both corpora. Main took **121.055s**, candidate **140.828s**. Both +reported one changed file, 22 reparsed files, six importer dependents, an +incremental write and no rebuild reasons. Filtered in-memory graph hashes and +the persisted new function, its File FTS hit, and all 34 call edges touching the +edited file matched. Counts were **117,439 nodes / 352,783 relationships**. +The next candidate repeat returned already up to date in **1.545s**. + +The candidate was slower in this single real-edit pair. Scope-resolution time +was 73.939s on main and 91.838s on the candidate, accounting for most of the +19.773s difference; the first pipeline progress event arrived at 0.923s and +1.569s respectively. Those phase timings locate the difference, but do not +establish its cause. Host load changed and this pair has one sample per +variant: neither a real-edit speedup nor a stable regression is established. + +### Reproducing the timing boundaries + +Use separate runtime and corpus checkouts of the pinned commit on local storage. +Build the runtime once with `npm ci`, then leave `dist` and dependencies unchanged. +Use a fresh corpus index for the first sample; retain that index and its parse +cache for the unchanged and edit samples. Set `BENCH_ROOT` to an isolated scratch +directory and `CORPUS` to the git checkout's absolute path. From the runtime's +`gitnexus/` directory, this collects the same timing boundaries: + +```bash +mkdir -p "$BENCH_ROOT/home" "$BENCH_ROOT/xdg" +NODE_ENV=development PROF_SCOPE_RESOLUTION=1 PROF_LBUG_LOAD=1 \ +GITNEXUS_HOME="$BENCH_ROOT/home" XDG_RUNTIME_DIR="$BENCH_ROOT/xdg" \ +GITNEXUS_SHARED_STORE=off GITNEXUS_LBUG_EXTENSION_INSTALL=load-only \ +node --max-old-space-size=8192 --input-type=module - "$CORPUS" <<'JS' +import { performance } from 'node:perf_hooks'; +import { runFullAnalysis } from './dist/core/run-analyze.js'; +const start = performance.now(); +const events = []; +const result = await runFullAnalysis(process.argv[2], { + skipAgentsMd: true, skipSkills: true, workerPoolSize: 16, verbose: true, +}, { + onProgress(phase, percent, message) { + events.push({ ms: performance.now() - start, phase, percent, message }); + }, + onLog(message) { + events.push({ ms: performance.now() - start, message }); + }, +}); +const wallMs = performance.now() - start; +console.log(JSON.stringify({ + wallMs, events, stats: result.stats, incrementalStats: result.incrementalStats, + alreadyUpToDate: result.alreadyUpToDate, rebuildReasons: result.rebuildReasons, + ftsSkipped: result.ftsSkipped, + reparsedFiles: result.pipelineResult?.reparsedFileCount, + cachedFiles: result.pipelineResult?.parseCacheHitFileCount, +})); +JS +``` + +Capture stdout and stderr for each sample. Reject an edit sample if it has +rebuild reasons, reports a full write, or skips FTS. The clean unchanged control +must return `alreadyUpToDate: true`; record the dirty repeat separately. Do not +reuse `--force` to measure edits, change the runtime between samples, or compare +a no-op with an actual edit. + +## Historical measurements — 2026-09-06/07 + +The following measurements were taken while landing #3194, #3196 and #3200. +Their optimization experiments have not been repeated against the October +corpus or LadybugDB 0.22.0. Claims about limits and rejected approaches below +describe that September baseline. + +### Method, and one trap that invalidates everything **The corpus must be a git repository.** A non-git checkout cannot record a schema fingerprint, so the tool forces a full rebuild on every run and prints: @@ -53,7 +365,7 @@ measured by injecting timestamp marks into a disposable copy of the built --- -## Cold analyze +### Cold analyze | | before #3194 | after #3196 | | ---------------- | -----------: | ----------: | @@ -66,7 +378,7 @@ measured by injecting timestamp marks into a disposable copy of the built Graph output identical throughout: 51,286 nodes / 163,092 edges / 2106 clusters / 759 flows. -## Re-analyze after a one-file edit — the developer loop +### Re-analyze after a one-file edit — the developer loop Leaf file (`cli/update-notice.ts`, 2 importers), stable runner identity, so the incremental path is genuinely taken. **31.7s total.** @@ -94,7 +406,7 @@ this.** A FULL-rebuild run of the same repo is 36-39s, with the graph write at ~6.3s and FTS at ~9.7s. Do not quote those as edit-loop numbers. -## FTS index rebuild — the largest non-resolution cost, and it is a floor +### FTS index rebuild — the largest non-resolution cost, and it is a floor A true incremental leaf edit rebuilds **8** of the 20 configured indexes — the tables the writeback actually DMLs. Per-index cost (measured against a copy of @@ -118,7 +430,7 @@ trap above), not a leaf edit. There is nothing to fix there. `File.file_fts` dominates because File rows carry whole-file content: 2442 rows, 32.9 MB, and Ladybug tokenizes at ~9.8 MB/s. -### Four ways out, all measured, all closed +#### Four ways out, all measured, all closed **Narrow further — no.** The 8 tables are exactly the ones holding rows for the 6 files in the write set (1 changed + 5 importers). There is no fat. @@ -144,7 +456,7 @@ is deleting full-file keyword search (#2317/#2323), not optimizing it. Capping the indexed content is a bad trade — the size distribution is flat, so a 64 KB cap still indexes 90% of the bytes while truncating the 72 largest files. -### The one lever left: overlap +#### The one lever left: overlap The FTS build runs on a libuv thread, not the main thread, and fully overlaps blocking JS: @@ -169,7 +481,7 @@ Two things block it today, and neither is small: would then leave the live index with fresh File content and stale symbols, instead of untouched. -## scopeResolution — 14.7s, and it is two different problems +### scopeResolution — 14.7s, and it is two different problems `PROF_SCOPE_RESOLUTION=1` already exists and reports the internal split. Marks injected around the phase supply the rest. On a one-file edit: @@ -186,7 +498,7 @@ injected around the phase supply the rest. On a one-file edit: `extract` is small because the parse cache works: 2121/2121 pre-extracted hits on the TypeScript pass. Nothing here re-parses. -### Rehydration: three full store scans, one per language +#### Rehydration: three full store scans, one per language The corpus resolves three languages — python (54 files), typescript (2121), javascript (59) — and `loadParsedFilesForPaths` walks **every** shard on each @@ -213,7 +525,7 @@ CRC-32 would buy ~90 ms and cost an envelope-format bump. The remainder is (~1085 ms), and the intern walk is not optional — dropping it regresses retained heap ~59%, which is #2649's constraint. -### `emit` is the real target +#### `emit` is the real target 7.2s, 49% of the phase and ~21% of the whole edit loop. It is a fan of passes, each walking every reference site in the repo: @@ -239,7 +551,7 @@ is a design, not a patch, and #2649 rules out "just cache the emitted edges". --- -## Measured and rejected +### Measured and rejected Recorded because each cost real time to establish and each looks attractive from the armchair. @@ -267,7 +579,7 @@ run (73.0s vs 69.6s on overlay). Measure on a local filesystem. --- -## Open +### Open at the September baseline The post-pipeline tail is fully accounted (99.7%): FTS rebuild, the graph write, and 0.8s of post-FTS event-loop drain between them. diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 07dde400f..72e5a83c0 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -195,6 +195,9 @@ const LBUG_NATIVE = [ 'test/integration/rename-provider-zig.test.ts', 'test/integration/pdg-explicit-owner.test.ts', 'test/integration/skip-fts.test.ts', + // Dirty snapshot freshness depends on native FTS, Git paths and file reads. + 'test/integration/analyze-dirty-snapshot.test.ts', + 'test/integration/incremental-worktree-context.test.ts', 'test/integration/lbug-core-adapter.test.ts', // Native COPY, cached vectors and cold shared-store readers vary by platform. 'test/integration/embedding-identity-integrity.test.ts', diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 289946659..e4a2de895 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -9,7 +9,11 @@ * wrapper or server worker) is responsible for process lifecycle. */ -import { detectGraphWriteCollapse, type GraphWriteCollapseVerdict } from './index-freshness.js'; +import { + detectGraphWriteCollapse, + getIndexIncompleteReasons, + type GraphWriteCollapseVerdict, +} from './index-freshness.js'; import { resolveFtsDisableReason, getFtsDisabledReason, @@ -278,6 +282,7 @@ import { resolveRepoIdentityRoot, } from '../storage/git.js'; import { isGitNexusManagedPath } from '../storage/gitnexus-managed-paths.js'; +import { walkRepositoryPaths } from './ingestion/filesystem-walker.js'; import { getMaxFileSizeBytes } from './ingestion/utils/max-file-size.js'; import { getIncludeDotDirs, includeDotDirsCoverage } from './ingestion/utils/include-dot-dirs.js'; import { @@ -1566,6 +1571,64 @@ export async function runFullAnalysis( }); } +/** Prove that a previously indexed dirty snapshot still covers the same readable bytes. */ +async function matchesIndexedDirtyWorktree(repoPath: string, meta: RepoMeta): Promise { + const { fileHashes, indexCoverage } = meta; + const indexedDirtyPaths = indexCoverage?.dirtyPaths; + if ( + !fileHashes || + !Array.isArray(indexedDirtyPaths) || + indexedDirtyPaths.length === 0 || + indexedDirtyPaths.some((rel) => typeof rel !== 'string') || + getIndexIncompleteReasons(meta).length > 0 || + indexCoverage?.maxFileSizeBytes !== getMaxFileSizeBytes() + ) { + return false; + } + + const liveDirtyPaths = listWorkingTreeDirtyPaths(repoPath); + const indexedDirtySet = new Set(indexedDirtyPaths); + if (!liveDirtyPaths?.length || liveDirtyPaths.some((rel) => !indexedDirtySet.has(rel))) { + return false; + } + + // Reject ordinary edits before scanning the repo. Include previously dirty + // paths so reverting one file while another stays dirty cannot look current. + const dirtyHashes = await computeFileHashes(repoPath, indexedDirtyPaths); + if ( + indexedDirtyPaths.some( + (rel) => !dirtyHashes.has(rel) || dirtyHashes.get(rel) !== fileHashes[rel], + ) + ) { + return false; + } + + try { + const files = await walkRepositoryPaths(repoPath, undefined, { + quiet: true, + // Enable the walker's enumeration-error reporting: an unreadable subtree + // must not be mistaken for an unchanged input set. + onPathsDiscovered: async () => {}, + }); + const currentPaths = files + .map((file) => file.path) + .filter((rel) => !isGitNexusManagedPath(rel)); + const recordedPaths = Object.keys(fileHashes).filter((rel) => !isGitNexusManagedPath(rel)); + if ( + currentPaths.length !== recordedPaths.length || + currentPaths.some((rel) => fileHashes[rel] === undefined) + ) { + return false; + } + // Check all covered files, including changes hidden from Git's status and + // changes to the scan's coverage through ignore rules outside the worktree. + const hashes = await computeFileHashes(repoPath, currentPaths); + return currentPaths.every((rel) => hashes.has(rel) && hashes.get(rel) === fileHashes[rel]); + } catch { + return false; + } +} + async function runFullAnalysisInner( repoPath: string, options: AnalyzeOptions, @@ -2591,9 +2654,8 @@ async function runFullAnalysisInner( // Non-git folders have currentCommit = '' — always rebuild since we can't detect changes if (currentCommit !== '') { // For git repos, even if HEAD matches lastCommit, the working tree - // may have uncommitted changes. Only short-circuit when the working - // tree is also clean — otherwise fall through to the incremental - // path which will hash-diff and update only changed files. + // may have uncommitted changes. A dirty checkout can short-circuit only + // after proving it still matches a successfully indexed dirty snapshot. // // We exclude paths that GitNexus itself writes during analyze: // .gitnexus/ — db / parse cache / meta.json @@ -2603,6 +2665,10 @@ async function runFullAnalysisInner( // fast path because the previous analyze just wrote them // (regression vs PR #1233 behavior). const dirty = isWorkingTreeDirty(repoPath); + const dirtySnapshotMatches = + dirty && + existsSync(placement.branch ? lbugPath : resolveGraphPath(storagePath)) && + (await matchesIndexedDirtyWorktree(repoPath, existingMeta)); // A clean porcelain status is not enough when the previous index captured // uncommitted content at this same HEAD: assume-unchanged and skip-worktree // paths are deliberately absent from porcelain. Re-hash only the paths @@ -2678,7 +2744,7 @@ async function runFullAnalysisInner( // re-analysis whenever an index authored where FTS was unavailable was // later read on a host where it loads — which is a legitimate, common // state, and the invariant `analyzer-identity-cli.test.ts` pins. - if (!dirty && !indexedContentChanged && !healUnregistered) { + if (((!dirty && !indexedContentChanged) || dirtySnapshotMatches) && !healUnregistered) { const processDetectionStamp = existingMeta.processDetection ?? toProcessDetectionStamp(processDetectionBudget); if (options.registryName) { @@ -3055,7 +3121,21 @@ async function runFullAnalysisInner( // ── Phase 1: Full Pipeline (0–60%) ──────────────────────────────── let pipelineResult; + let inputFileHashes: Map | undefined; try { + if (repoHasGit) { + // Capture freshness before parsing. Hashing afterward can certify an edit + // the graph never saw and let the dirty fast path preserve stale results. + // A concurrent edit now differs from this receipt on the next analysis. + const inputs = await walkRepositoryPaths(repoPath, undefined, { + quiet: true, + onPathsDiscovered: async () => {}, + }); + inputFileHashes = await computeFileHashes( + repoPath, + inputs.map((file) => file.path), + ); + } pipelineResult = await runPipelineFromRepo( repoPath, (p) => { @@ -3142,7 +3222,7 @@ async function runFullAnalysisInner( // before any graph rows, FTS values, or embeddings are persisted. applyContentRetention(pipelineResult.graph, contentRetention); - // Compute current per-file content hashes from the pipeline's File nodes. + // Restrict the pre-parse hashes to the pipeline's actual File nodes. // Used both to drive the incremental DB writeback (when eligible) and to // populate meta.json.fileHashes for the next run. const allFilePaths: string[] = []; @@ -3152,7 +3232,14 @@ async function runFullAnalysisInner( if (fp) allFilePaths.push(fp); } }); - const newFileHashes = await computeFileHashes(repoPath, allFilePaths); + const newFileHashes = inputFileHashes + ? new Map( + allFilePaths.flatMap((rel) => { + const hash = inputFileHashes.get(rel); + return hash === undefined ? [] : [[rel, hash] as const]; + }), + ) + : await computeFileHashes(repoPath, allFilePaths); const currentAnalysisFeatures = resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, allFilePaths); const currentAnalysisFeatureMismatches = existingMeta ? findAnalysisFeatureMismatches(existingMeta.analysisFeatures, currentAnalysisFeatures) diff --git a/gitnexus/test/integration/analyze-dirty-snapshot.test.ts b/gitnexus/test/integration/analyze-dirty-snapshot.test.ts new file mode 100644 index 000000000..9069e9944 --- /dev/null +++ b/gitnexus/test/integration/analyze-dirty-snapshot.test.ts @@ -0,0 +1,286 @@ +/** Real filesystem, graph and native FTS coverage for unchanged dirty analysis. */ +import { execSync } from 'node:child_process'; +import { writeFile, readFile, rm } from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, it, expect, vi } from 'vitest'; +import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { setupMiniRepo as setupSharedMiniRepo } from '../helpers/mini-repo.js'; + +const setupMiniRepo = () => setupSharedMiniRepo('gitnexus-dirty-snapshot-'); + +describe('unchanged dirty snapshot analysis', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('dirty snapshot fast path preserves the database bytes, call edges, and native FTS', async (ctx) => { + const repo = await setupMiniRepo(); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const target = path.join(repo.dbPath, 'src/logger.ts'); + const dirtySource = `${await readFile(target, 'utf8')}\n// dirtysnapshotneedle\n`; + await writeFile(target, dirtySource); + const options = { skipAgentsMd: true, skipSkills: true }; + const indexed = await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + if (indexed.ftsSkipped) { + if (process.env.GITNEXUS_REQUIRE_FTS === '1') throw new Error('Native FTS is required'); + ctx.skip('Packaged FTS is unavailable on this platform'); + } + const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); + const readEvidence = async () => { + await adapter.initLbug(lbugPath); + try { + return { + calls: await adapter.executeQuery( + "MATCH (a:Function)-[r:CodeRelation]->(b:Function) WHERE r.type = 'CALLS' " + + 'RETURN a.id AS source, b.id AS target ORDER BY source, target', + ), + search: await adapter.executeQuery( + "CALL QUERY_FTS_INDEX('File', 'file_fts', 'dirtysnapshotneedle') " + + 'RETURN node.filePath AS filePath ORDER BY filePath', + ), + }; + } finally { + await adapter.closeLbug(); + } + }; + const evidence = await readEvidence(); + expect(evidence.calls.length).toBeGreaterThan(0); + expect(evidence.search).toEqual([{ filePath: 'src/logger.ts' }]); + const database = await readFile(lbugPath); + const meta = await loadMeta(storagePath); + const onProgress = vi.fn(); + + const repeated = await runFullAnalysis(repo.dbPath, options, { onProgress }); + expect(repeated.alreadyUpToDate).toBe(true); + expect(repeated.pipelineResult).toBeUndefined(); + expect(repeated.rebuildReasons).toEqual([]); + expect(onProgress).not.toHaveBeenCalled(); + expect((await readFile(lbugPath)).equals(database)).toBe(true); + expect(await loadMeta(storagePath)).toEqual(meta); + expect(await readEvidence()).toEqual(evidence); + + // Same Git dirty path, different bytes: this must still run the pipeline. + await writeFile( + target, + `${dirtySource}\nexport function nextDirtyVersion() { return 42; }\n`, + ); + const edited = await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + expect(edited.alreadyUpToDate).toBeUndefined(); + expect(edited.incrementalStats).toMatchObject({ changedFiles: 1, writeMode: 'incremental' }); + await adapter.initLbug(lbugPath); + try { + expect( + await adapter.executeQuery( + "MATCH (f:Function {name: 'nextDirtyVersion'}) RETURN f.filePath AS filePath", + ), + ).toEqual([{ filePath: 'src/logger.ts' }]); + } finally { + await adapter.closeLbug(); + } + } finally { + await repo.cleanup(); + } + }, 300_000); + + it.each([ + 'addition', + 'deletion', + 'rename', + 'restore-one-dirty-file', + 'hidden-edit', + 'ignored-scan-policy', + 'file-size-policy', + 'force', + 'interrupted-write', + 'scope-incomplete', + 'scope-unverified', + ] as const)( + 'dirty snapshot fast path falls back for %s', + async (change) => { + const repo = await setupMiniRepo(); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + // This policy file can change the scan without appearing in Git status. + await writeFile(path.join(repo.dbPath, '.gitignore'), '.gitnexusignore\n'); + execSync('git add .gitignore', { cwd: repo.dbPath, stdio: 'pipe' }); + execSync( + 'git -c user.name=test -c user.email=t@t -c commit.gpgsign=false commit -qm policy', + { + cwd: repo.dbPath, + stdio: 'pipe', + }, + ); + const target = path.join(repo.dbPath, 'src/logger.ts'); + const original = await readFile(target, 'utf8'); + await writeFile(target, `${original}\n// indexed dirty snapshot\n`); + if (change === 'restore-one-dirty-file') { + const handler = path.join(repo.dbPath, 'src/handler.ts'); + await writeFile(handler, `${await readFile(handler, 'utf8')}\n// also dirty\n`); + } + const options = { skipAgentsMd: true, skipSkills: true, skipFts: true }; + await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + expect( + (await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} })).alreadyUpToDate, + ).toBe(true); + const { storagePath } = getStoragePaths(repo.dbPath); + + switch (change) { + case 'addition': + await writeFile( + path.join(repo.dbPath, 'src/added.ts'), + 'export const addedValue = 1;\n', + ); + break; + case 'deletion': + await rm(path.join(repo.dbPath, 'src/formatter.ts')); + break; + case 'rename': + execSync('git mv src/formatter.ts src/renamed.ts', { cwd: repo.dbPath, stdio: 'pipe' }); + break; + case 'restore-one-dirty-file': + await writeFile(target, original); + break; + case 'hidden-edit': { + execSync('git update-index --assume-unchanged src/formatter.ts', { + cwd: repo.dbPath, + stdio: 'pipe', + }); + const formatter = path.join(repo.dbPath, 'src/formatter.ts'); + await writeFile(formatter, `${await readFile(formatter, 'utf8')}\n// hidden edit\n`); + break; + } + case 'ignored-scan-policy': + await writeFile(path.join(repo.dbPath, '.gitnexusignore'), 'src/formatter.ts\n'); + break; + case 'file-size-policy': + vi.stubEnv('GITNEXUS_MAX_FILE_SIZE', '1024'); + break; + case 'interrupted-write': { + const meta = await loadMeta(storagePath); + await saveMeta(storagePath, { + ...meta!, + incrementalInProgress: { startedAt: Date.now(), toWriteCount: 0 }, + }); + break; + } + case 'scope-incomplete': + case 'scope-unverified': { + const meta = await loadMeta(storagePath); + await saveMeta(storagePath, { + ...meta!, + ...(change === 'scope-incomplete' + ? { scopeExtractionFailures: { total: 1, paths: ['src/logger.ts'] } } + : { scopeExtractionReceipt: undefined }), + }); + break; + } + case 'force': + break; + } + + const refreshed = await runFullAnalysis( + repo.dbPath, + { ...options, force: change === 'force' }, + { onProgress: () => {} }, + ); + expect(refreshed.alreadyUpToDate).toBeUndefined(); + expect(refreshed.pipelineResult).toBeDefined(); + const hashes = (await loadMeta(storagePath))!.fileHashes!; + if (change === 'addition') expect(hashes).toHaveProperty('src/added.ts'); + if (change === 'deletion' || change === 'rename' || change === 'ignored-scan-policy') { + expect(hashes).not.toHaveProperty('src/formatter.ts'); + } + if (change === 'rename') expect(hashes).toHaveProperty('src/renamed.ts'); + if (change === 'force') expect(refreshed.rebuildReasons).toContain('user-force'); + if (change === 'interrupted-write') + expect(refreshed.rebuildReasons).toContain('interrupted-rebuild'); + } finally { + await repo.cleanup(); + } + }, + 300_000, + ); + + it.each(['fresh', 'incremental'] as const)( + 'reconciles a file edited after %s parsing before accepting a dirty snapshot', + async (mode) => { + const repo = await setupMiniRepo(); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const pipeline = await import('../../src/core/ingestion/pipeline.js'); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const target = path.join(repo.dbPath, 'src/logger.ts'); + const options = { skipAgentsMd: true, skipSkills: true, skipFts: true }; + if (mode === 'incremental') { + await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + } + const dirtySource = `${await readFile(target, 'utf8')}\n// indexed dirty snapshot\n`; + await writeFile(target, dirtySource); + const runPipeline = pipeline.runPipelineFromRepo; + const editDuringAnalysis = vi + .spyOn(pipeline, 'runPipelineFromRepo') + .mockImplementationOnce(async (...args) => { + const result = await runPipeline(...args); + await writeFile(target, `${dirtySource}\nexport function editedDuringAnalysis() {}\n`); + return result; + }); + try { + await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + } finally { + editDuringAnalysis.mockRestore(); + } + + const refreshed = await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + expect(refreshed.alreadyUpToDate).toBeUndefined(); + const { lbugPath } = getStoragePaths(repo.dbPath); + await adapter.initLbug(lbugPath); + try { + expect( + await adapter.executeQuery( + "MATCH (f:Function {name: 'editedDuringAnalysis'}) RETURN f.filePath AS filePath", + ), + ).toEqual([{ filePath: 'src/logger.ts' }]); + } finally { + await adapter.closeLbug(); + } + expect( + (await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} })).alreadyUpToDate, + ).toBe(true); + } finally { + await repo.cleanup(); + } + }, + 300_000, + ); + + it('dirty snapshot fast path falls back when the coverage scan fails', async () => { + const repo = await setupMiniRepo(); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const walker = await import('../../src/core/ingestion/filesystem-walker.js'); + const target = path.join(repo.dbPath, 'src/logger.ts'); + await writeFile(target, `${await readFile(target, 'utf8')}\n// indexed dirty snapshot\n`); + const options = { skipAgentsMd: true, skipSkills: true, skipFts: true }; + await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + const scan = vi + .spyOn(walker, 'walkRepositoryPaths') + .mockRejectedValueOnce(new Error('unreadable directory')); + try { + const refreshed = await runFullAnalysis(repo.dbPath, options, { onProgress: () => {} }); + expect(refreshed.alreadyUpToDate).toBeUndefined(); + expect(refreshed.pipelineResult).toBeDefined(); + } finally { + scan.mockRestore(); + } + } finally { + await repo.cleanup(); + } + }, 300_000); +}); diff --git a/gitnexus/test/integration/incremental-worktree-context.test.ts b/gitnexus/test/integration/incremental-worktree-context.test.ts index d8292813b..25274cb32 100644 --- a/gitnexus/test/integration/incremental-worktree-context.test.ts +++ b/gitnexus/test/integration/incremental-worktree-context.test.ts @@ -150,6 +150,8 @@ describe.skipIf(!isLanguageAvailable(SupportedLanguages.Swift))( git(main, 'worktree', 'add', '-q', '-b', 'lookup-refresh', worktree); expect((await fs.stat(path.join(worktree, '.git'))).isFile()).toBe(true); expect(git(worktree, 'rev-parse', 'HEAD')).toBe(git(main, 'rev-parse', 'HEAD')); + // Preserve the checkout's actual bytes, including CRLF on Windows. + const swiftBefore = await fs.readFile(path.join(worktree, SWIFT_PATH)); const { storagePath, lbugPath } = getStoragePaths(worktree); expect(storagePath).toBe(path.join(worktree, '.gitnexus')); @@ -256,22 +258,19 @@ describe.skipIf(!isLanguageAvailable(SupportedLanguages.Swift))( ); expect(writePaths).not.toContain(SWIFT_PATH); expect((await loadMeta(storagePath))?.runnerIdentity).toEqual(baselineMeta?.runnerIdentity); - expect(await fs.readFile(path.join(worktree, SWIFT_PATH), 'utf8')).toBe(SWIFT_SOURCE); + expect(await fs.readFile(path.join(worktree, SWIFT_PATH))).toEqual(swiftBefore); await readPublicContexts(); expectCurrentStatus(); expect(await readPropertyFts()).toEqual(propertyBefore); deleteNodes.mockClear(); const noop = await runFullAnalysis(worktree, options, callbacks); - // The edit remains uncommitted, so clean-porcelain early return is - // unavailable. Identical content instead takes a zero-file refresh. - expect(noop.incrementalStats).toMatchObject({ - changedFiles: 0, - affectedDependents: 0, - deletedFiles: 0, - writeMode: 'incremental', - }); - expect(deleteNodes.mock.calls.flatMap(([filePaths]) => filePaths)).toEqual([]); + // The uncommitted edit is already indexed: verify freshness without + // running the pipeline or touching the persisted graph. + expect(noop.alreadyUpToDate).toBe(true); + expect(noop.pipelineResult).toBeUndefined(); + expect(noop.incrementalStats).toBeUndefined(); + expect(deleteNodes).not.toHaveBeenCalled(); expect((await loadMeta(storagePath))?.runnerIdentity).toEqual(baselineMeta?.runnerIdentity); await readPublicContexts(); expectCurrentStatus(); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index e70c94929..b31f9c10a 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -72,6 +72,7 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', 'test/integration/rename-provider-*.test.ts', @@ -187,6 +188,7 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + 'test/integration/analyze-dirty-snapshot.test.ts', 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', 'test/integration/rename-provider-*.test.ts',